diff --git "a/eval_results/base_bench_1.1/a_only/report.json" "b/eval_results/base_bench_1.1/a_only/report.json" new file mode 100644--- /dev/null +++ "b/eval_results/base_bench_1.1/a_only/report.json" @@ -0,0 +1,19763 @@ +{ + "created_at": "2026-08-19T11:28:59.082145+00:00", + "benchmark": "BananaMind Base Bench 1.1", + "benchmark_version": "1.1", + "dataset_id": "BananaMind/BananaMind-Base-Bench-1.1", + "dataset_revision": "main", + "dataset_file": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-Base-Bench-1.1/data/test.jsonl", + "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21", + "model": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-2.1-Unified", + "model_revision": "main", + "tokenizer": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-2.1-Unified", + "device": "cuda:0", + "dtype": "bfloat16", + "model_context_length": 4096, + "scoring": { + "task": "four-choice base-text continuation", + "choice_score": "mean conditional token log-probability", + "chat_template": false, + "generation": false, + "add_special_tokens": false, + "add_bos": false + }, + "elo": { + "scale": 400.0, + "prior_rating": 1000.0, + "prior_weight": 4.0, + "method": "weighted fixed-item logistic maximum-likelihood rating" + }, + "summary": { + "official_complete_run": true, + "cases": 350, + "passed": 117, + "accuracy": 0.3342857142857143, + "weighted_points": 210.95, + "possible_weighted_points": 668.1875, + "weighted_accuracy": 0.3157047984285848, + "overall_elo": 866, + "overall_elo_unrounded": 866.4554710473171, + "categories": { + "language_completion": { + "cases": 50, + "passed": 27, + "accuracy": 0.54, + "weighted_points": 43.5, + "possible_weighted_points": 78.5, + "weighted_accuracy": 0.554140127388535, + "elo": 942, + "elo_unrounded": 941.8854958589034 + }, + "commonsense": { + "cases": 50, + "passed": 17, + "accuracy": 0.34, + "weighted_points": 31.35, + "possible_weighted_points": 87.17500000000001, + "weighted_accuracy": 0.35962145110410093, + "elo": 850, + "elo_unrounded": 850.4897303751222 + }, + "world_knowledge": { + "cases": 50, + "passed": 12, + "accuracy": 0.24, + "weighted_points": 19.25, + "possible_weighted_points": 87.72500000000001, + "weighted_accuracy": 0.21943573667711597, + "elo": 731, + "elo_unrounded": 730.9378172569038 + }, + "context_tracking": { + "cases": 50, + "passed": 19, + "accuracy": 0.38, + "weighted_points": 32.099999999999994, + "possible_weighted_points": 94.19999999999999, + "weighted_accuracy": 0.34076433121019106, + "elo": 879, + "elo_unrounded": 879.4053503580149 + }, + "quantitative": { + "cases": 50, + "passed": 10, + "accuracy": 0.2, + "weighted_points": 19.175, + "possible_weighted_points": 103.025, + "weighted_accuracy": 0.1861198738170347, + "elo": 788, + "elo_unrounded": 788.4764543219264 + }, + "logical_reasoning": { + "cases": 50, + "passed": 17, + "accuracy": 0.34, + "weighted_points": 34.425000000000004, + "possible_weighted_points": 107.66250000000001, + "weighted_accuracy": 0.31974921630094044, + "elo": 962, + "elo_unrounded": 962.4858987383311 + }, + "code_completion": { + "cases": 50, + "passed": 15, + "accuracy": 0.3, + "weighted_points": 31.15, + "possible_weighted_points": 109.89999999999999, + "weighted_accuracy": 0.28343949044585987, + "elo": 927, + "elo_unrounded": 926.9986627252015 + } + }, + "difficulties": { + "easy": { + "cases": 117, + "passed": 43, + "accuracy": 0.36752136752136755, + "weighted_points": 51.2, + "possible_weighted_points": 141.2, + "weighted_accuracy": 0.36260623229461764, + "elo": 769, + "elo_unrounded": 769.4837527375244 + }, + "medium": { + "cases": 117, + "passed": 41, + "accuracy": 0.3504273504273504, + "weighted_points": 74.025, + "possible_weighted_points": 211.875, + "weighted_accuracy": 0.34938053097345134, + "elo": 856, + "elo_unrounded": 856.0389872204846 + }, + "hard": { + "cases": 116, + "passed": 33, + "accuracy": 0.28448275862068967, + "weighted_points": 85.725, + "possible_weighted_points": 315.1125, + "weighted_accuracy": 0.2720456979650125, + "elo": 939, + "elo_unrounded": 938.7728217057725 + } + }, + "context_truncations": 0 + }, + "results": [ + { + "id": "language_completion-001", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Dark clouds gathered above the park, so Mira opened", + "continuations": [ + " a sandwich loudly.", + " the sunlight.", + " her umbrella.", + " the calendar yesterday." + ], + "tags": [ + "coherence" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -92.80339050292969, + "mean_logprob": -11.600423812866211, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -30.03310775756836, + "mean_logprob": -10.011035919189453, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -75.70381927490234, + "mean_logprob": -10.814831324986049, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -87.27207946777344, + "mean_logprob": -10.90900993347168, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.8037954057965955, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-002", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The librarian asked everyone to speak", + "continuations": [ + " brightly.", + " quietly.", + " woodenly.", + " tomorrow." + ], + "tags": [ + "adverb" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -36.769187927246094, + "mean_logprob": -12.256395975748697, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.749488830566406, + "mean_logprob": -12.249829610188803, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -51.38093566894531, + "mean_logprob": -12.845233917236328, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -49.79552459716797, + "mean_logprob": -12.448881149291992, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.006566365559894649, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-003", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "The brass key fit perfectly into the", + "continuations": [ + " pillow.", + " thunder.", + " lock.", + " soup." + ], + "tags": [ + "collocation" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.454429626464844, + "mean_logprob": -10.363607406616211, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -38.76958465576172, + "mean_logprob": -12.923194885253906, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.46735191345215, + "mean_logprob": -10.489117304484049, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.642974853515625, + "mean_logprob": -9.214324951171875, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.149282455444336, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-004", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "After kneading the dough, the baker placed it in the", + "continuations": [ + " envelope.", + " freezer to read.", + " bicycle.", + " oven." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -48.14434051513672, + "mean_logprob": -12.03608512878418, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -53.105613708496094, + "mean_logprob": -10.621122741699219, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.57585144042969, + "mean_logprob": -8.393962860107422, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.702102661132812, + "mean_logprob": -7.9007008870442705, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.49326197306315134, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-005", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Leo was exhausted after the hike and went straight to", + "continuations": [ + " bed.", + " the trumpet.", + " blue.", + " calculate." + ], + "tags": [ + "coherence" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -23.680274963378906, + "mean_logprob": -11.840137481689453, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -62.36741638183594, + "mean_logprob": -12.473483276367187, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -22.811996459960938, + "mean_logprob": -11.405998229980469, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -39.02041244506836, + "mean_logprob": -13.00680414835612, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.4341392517089844, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-006", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The ice cube disappeared because it had", + "continuations": [ + " flown north.", + " melted.", + " written a letter.", + " become louder." + ], + "tags": [ + "verb" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -48.13517761230469, + "mean_logprob": -12.033794403076172, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.716230392456055, + "mean_logprob": -9.572076797485352, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.466365814208984, + "mean_logprob": -10.366591453552246, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -65.03280639648438, + "mean_logprob": -13.006561279296875, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.7945146560668945, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-007", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "One child waited outside; soon, three more", + "continuations": [ + " childs arrived.", + " childes arrived.", + " children arrived.", + " child arriveds." + ], + "tags": [ + "grammar" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -28.992938995361328, + "mean_logprob": -7.248234748840332, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.747535705566406, + "mean_logprob": -8.436883926391602, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.682159423828125, + "mean_logprob": -5.560719807942708, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -35.46637725830078, + "mean_logprob": -8.866594314575195, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.687514940897624, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-008", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Yesterday, Priya walked to the station and", + "continuations": [ + " waits for tomorrow.", + " waiting there now.", + " will walked home.", + " bought a ticket." + ], + "tags": [ + "tense" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -72.78483581542969, + "mean_logprob": -10.397833687918526, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -46.86603927612305, + "mean_logprob": -11.716509819030762, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -56.371280670166016, + "mean_logprob": -11.274256134033203, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -60.66819763183594, + "mean_logprob": -10.111366271972656, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.2864674159458698, + "context_truncated": false, + "elapsed_seconds": 0.028340928874968085 + }, + { + "id": "language_completion-009", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The cup is empty, but the pitcher is", + "continuations": [ + " full.", + " emptier than silence.", + " running yesterday.", + " carefully." + ], + "tags": [ + "contrast" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.035484313964844, + "mean_logprob": -9.517742156982422, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -65.25928497314453, + "mean_logprob": -10.876547495524088, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -64.20753479003906, + "mean_logprob": -12.841506958007812, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -30.716838836669922, + "mean_logprob": -15.358419418334961, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 1.358805338541666, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-010", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Neither the teacher nor the students", + "continuations": [ + " was arrives.", + " were ready.", + " is readiness.", + " be readying." + ], + "tags": [ + "agreement" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -44.49162292480469, + "mean_logprob": -11.122905731201172, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.79950523376465, + "mean_logprob": -10.599835077921549, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.953582763671875, + "mean_logprob": -11.238395690917969, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.62047576904297, + "mean_logprob": -10.655118942260742, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.0552838643391933, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-011", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Nora packed an apple, a bottle of water, and", + "continuations": [ + " because.", + " beneath.", + " a sandwich.", + " although." + ], + "tags": [ + "parallelism" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.670927047729492, + "mean_logprob": -9.835463523864746, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.479061126708984, + "mean_logprob": -10.869765281677246, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -56.35068130493164, + "mean_logprob": -11.270136260986328, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -26.03286361694336, + "mean_logprob": -13.01643180847168, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.0343017578125, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-012", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "A robin is a bird, and a salmon is", + "continuations": [ + " beneath flying.", + " an quickly.", + " yesterday's wing.", + " a fish." + ], + "tags": [ + "syntax" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -65.63457489013672, + "mean_logprob": -13.126914978027344, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.45935821533203, + "mean_logprob": -9.819786071777344, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -102.36324310302734, + "mean_logprob": -14.62332044328962, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -15.825918197631836, + "mean_logprob": -5.275306065877278, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 4.544480005900065, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-013", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "She adopted a puppy from", + "continuations": [ + " an animal shelter.", + " shelter animal an.", + " a sheltering quickly.", + " the adopted because." + ], + "tags": [ + "article" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -59.7615852355957, + "mean_logprob": -11.952317047119141, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -65.02971649169922, + "mean_logprob": -13.005943298339844, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -61.201011657714844, + "mean_logprob": -12.240202331542969, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.739444732666016, + "mean_logprob": -11.934861183166504, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.01745586395263743, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-014", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Daniel thanked Rosa because", + "continuations": [ + " him had helped she.", + " she had helped him.", + " her helped himself.", + " they helps he." + ], + "tags": [ + "pronouns" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.266380310058594, + "mean_logprob": -8.253276062011718, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -50.435813903808594, + "mean_logprob": -10.08716278076172, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -62.187721252441406, + "mean_logprob": -15.546930313110352, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.09117889404297, + "mean_logprob": -10.022794723510742, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 1.7695186614990241, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-015", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "This suitcase is heavy, but that one is even", + "continuations": [ + " most heavy.", + " heavily one.", + " heavier.", + " more heavier." + ], + "tags": [ + "comparative" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.06753158569336, + "mean_logprob": -9.68917719523112, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.48828887939453, + "mean_logprob": -9.829429626464844, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -36.76765060424805, + "mean_logprob": -12.25588353474935, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.546077728271484, + "mean_logprob": -11.886519432067871, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.14025243123372455, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-016", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Monday comes immediately after", + "continuations": [ + " Wednesday.", + " evening.", + " January.", + " Sunday." + ], + "tags": [ + "sequence" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.24907684326172, + "mean_logprob": -10.249815368652344, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.126150131225586, + "mean_logprob": -9.042050043741861, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.358013153076172, + "mean_logprob": -9.679006576538086, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.24964141845703, + "mean_logprob": -9.416547139485678, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.3744970957438163, + "context_truncated": false, + "elapsed_seconds": 0.0011151180005981587 + }, + { + "id": "language_completion-017", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Although the road was narrow, the driver remained", + "continuations": [ + " careful.", + " width.", + " carefully road.", + " narrowness." + ], + "tags": [ + "grammar" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -21.156307220458984, + "mean_logprob": -10.578153610229492, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.85387420654297, + "mean_logprob": -12.284624735514322, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -36.937435150146484, + "mean_logprob": -12.312478383382162, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.75717544555664, + "mean_logprob": -10.919058481852213, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.34090487162272076, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-018", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The forecast promised sunshine. Nevertheless, we packed", + "continuations": [ + " the forecast itself.", + " rain jackets.", + " a darker noon.", + " sunshine into boxes." + ], + "tags": [ + "discourse" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -62.78392791748047, + "mean_logprob": -12.556785583496094, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -72.58822631835938, + "mean_logprob": -14.517645263671875, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -76.14848327636719, + "mean_logprob": -12.691413879394531, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -81.33482360839844, + "mean_logprob": -11.619260515485491, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.9375250680106024, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-019", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Amir revised the report twice before submitting it, thereby", + "continuations": [ + " forgetting what reports are.", + " submitting before revising.", + " reducing the chance of errors.", + " making twice become three." + ], + "tags": [ + "discourse" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -61.50270080566406, + "mean_logprob": -10.250450134277344, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -83.49238586425781, + "mean_logprob": -11.927483694893974, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -52.00019836425781, + "mean_logprob": -8.666699727376303, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.14020538330078, + "mean_logprob": -9.828041076660156, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.1613413492838536, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-020", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The museum closes at six. We arrived at five forty-five, leaving us", + "continuations": [ + " closed before arrival.", + " six museums to visit.", + " yesterday instead.", + " only fifteen minutes." + ], + "tags": [ + "coherence" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -59.41428756713867, + "mean_logprob": -11.882857513427734, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -59.21356201171875, + "mean_logprob": -9.868927001953125, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -56.22765350341797, + "mean_logprob": -11.245530700683593, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.41008377075195, + "mean_logprob": -9.88201675415039, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.01308975219726527, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-021", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The committee postponed the vote until every member had", + "continuations": [ + " reviewed the proposal.", + " been proposal by voting.", + " postponed every committee.", + " review the voted." + ], + "tags": [ + "aspect" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -73.27058410644531, + "mean_logprob": -12.211764017740885, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -82.56571960449219, + "mean_logprob": -11.795102800641741, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.46793365478516, + "mean_logprob": -9.495419093540736, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -57.76830291748047, + "mean_logprob": -11.553660583496093, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 2.058241489955357, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-022", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Had Elena known the bridge was closed, she", + "continuations": [ + " takes that route anyway.", + " would have taken another route.", + " will know it tomorrow yesterday.", + " had closes the bridge." + ], + "tags": [ + "conditional" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -64.84801483154297, + "mean_logprob": -10.808002471923828, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -68.56625366210938, + "mean_logprob": -11.427708943684896, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -92.54377746582031, + "mean_logprob": -9.254377746582032, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -67.59938049316406, + "mean_logprob": -11.266563415527344, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.5536247253417965, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-023", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The novel was praised not only for its plot but also for", + "continuations": [ + " it was long.", + " plotting beyond.", + " its vivid characters.", + " praise was also." + ], + "tags": [ + "parallelism" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.03797149658203, + "mean_logprob": -10.259492874145508, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.7880859375, + "mean_logprob": -10.197021484375, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -53.928924560546875, + "mean_logprob": -10.785784912109374, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.424072265625, + "mean_logprob": -9.884814453125, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.3122070312500007, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-024", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Because the instructions were ambiguous, two technicians interpreted them", + "continuations": [ + " with identical ambiguity only.", + " before they were written.", + " into a screwdriver.", + " differently." + ], + "tags": [ + "semantics" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -94.779052734375, + "mean_logprob": -10.531005859375, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -42.10055160522461, + "mean_logprob": -8.420110321044922, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -109.4315185546875, + "mean_logprob": -13.678939819335938, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.401912689208984, + "mean_logprob": -7.800637563069661, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.6194727579752604, + "context_truncated": false, + "elapsed_seconds": 0.001023645501845749 + }, + { + "id": "language_completion-025", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Sofia lowered her voice so that she would not", + "continuations": [ + " wake the baby.", + " sleeping the quiet.", + " lower the voice's color.", + " have loudly whisper." + ], + "tags": [ + "purpose" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.025108337402344, + "mean_logprob": -10.405021667480469, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -58.78659439086914, + "mean_logprob": -11.757318878173828, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -54.513214111328125, + "mean_logprob": -9.085535685221354, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -99.5632095336914, + "mean_logprob": -12.445401191711426, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.3194859822591152, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-026", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The first experiment failed; the second, by contrast,", + "continuations": [ + " was experiment first.", + " produced reliable results.", + " contrasted before being second.", + " failing reliably." + ], + "tags": [ + "discourse" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -52.93201446533203, + "mean_logprob": -13.233003616333008, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.202857971191406, + "mean_logprob": -8.300714492797852, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -58.84813690185547, + "mean_logprob": -9.808022816975912, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -49.611961364746094, + "mean_logprob": -8.268660227457682, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.03205426534016986, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-027", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Every applicant must attach a resume, regardless of", + "continuations": [ + " attaching is a verb.", + " resumes being paper.", + " prior experience.", + " every regardless." + ], + "tags": [ + "collocation" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -83.44457244873047, + "mean_logprob": -11.920653206961495, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -50.465126037597656, + "mean_logprob": -10.093025207519531, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -24.311176300048828, + "mean_logprob": -8.10372543334961, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.176231384277344, + "mean_logprob": -11.058743794759115, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.9892997741699219, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-028", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The recipe calls for fresh basil; dried basil may be used", + "continuations": [ + " before fresh basil exists.", + " to call the recipe.", + " as a plate.", + " as a substitute." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -74.38973999023438, + "mean_logprob": -12.398289998372396, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -58.704280853271484, + "mean_logprob": -9.78404680887858, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -39.15608215332031, + "mean_logprob": -9.789020538330078, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -44.35456085205078, + "mean_logprob": -8.870912170410156, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.9131346384684242, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-029", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Far from being discouraged by the criticism, Mei became", + "continuations": [ + " more determined.", + " criticism itself.", + " far from becoming.", + " determine yesterday." + ], + "tags": [ + "idiom" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.862960815429688, + "mean_logprob": -7.2876536051432295, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -49.0496711730957, + "mean_logprob": -12.262417793273926, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -48.452301025390625, + "mean_logprob": -12.113075256347656, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -71.11412048339844, + "mean_logprob": -14.222824096679688, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 4.825421651204427, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-030", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The contract is valid provided that both parties", + "continuations": [ + " had validity tomorrow.", + " sign it.", + " providing a party.", + " signs them." + ], + "tags": [ + "conditional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -80.44217681884766, + "mean_logprob": -11.491739545549665, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.984851837158203, + "mean_logprob": -8.661617279052734, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.68141174316406, + "mean_logprob": -10.670352935791016, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.235210418701172, + "mean_logprob": -10.07840347290039, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 1.4167861938476562, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-031", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "No sooner had the bell rung than the students", + "continuations": [ + " have ringing bells.", + " sooner the bell.", + " left the classroom.", + " had leave tomorrow." + ], + "tags": [ + "inversion" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -77.32657623291016, + "mean_logprob": -11.046653747558594, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -65.42047882080078, + "mean_logprob": -10.90341313680013, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.82670211791992, + "mean_logprob": -9.45667552947998, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -63.37792205810547, + "mean_logprob": -10.562987009684244, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.1063114802042637, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-032", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The evidence was incomplete. Consequently, the investigators", + "continuations": [ + " completed evidence by definition.", + " investigated before consequence.", + " were the evidence.", + " withheld judgment." + ], + "tags": [ + "discourse" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -50.21240997314453, + "mean_logprob": -10.042481994628906, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -53.50930404663086, + "mean_logprob": -8.918217341105143, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -19.994230270385742, + "mean_logprob": -4.9985575675964355, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -65.81995391845703, + "mean_logprob": -10.969992319742838, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 3.919659773508707, + "context_truncated": false, + "elapsed_seconds": 0.0009743211230670568 + }, + { + "id": "language_completion-033", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Walking along the shore, I noticed", + "continuations": [ + " a trail of fresh footprints.", + " the shore walked me.", + " noticing had walked.", + " along was a footprint." + ], + "tags": [ + "modifier" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -107.23603820800781, + "mean_logprob": -11.915115356445312, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -75.10311889648438, + "mean_logprob": -10.729016985212054, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -55.68989562988281, + "mean_logprob": -9.281649271647135, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -60.41368103027344, + "mean_logprob": -10.068946838378906, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.7872975667317714, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-034", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The orchestra rehearsed the difficult passage repeatedly until it sounded", + "continuations": [ + " passage by difficulty.", + " effortless.", + " rehearsing the sound.", + " repeatably orchestra." + ], + "tags": [ + "collocation" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -52.19974136352539, + "mean_logprob": -13.049935340881348, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -38.615753173828125, + "mean_logprob": -12.871917724609375, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -92.34956359863281, + "mean_logprob": -13.192794799804688, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -110.09695434570312, + "mean_logprob": -13.76211929321289, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.17801761627197266, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-035", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "What the witness omitted from her statement proved", + "continuations": [ + " that omission witnessed.", + " proving had omitted.", + " more significant than what she included.", + " to include more omittedly." + ], + "tags": [ + "syntax" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -61.69489669799805, + "mean_logprob": -10.282482782999674, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -60.96964645385742, + "mean_logprob": -10.16160774230957, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -55.26149368286133, + "mean_logprob": -7.894499097551618, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -54.67353057861328, + "mean_logprob": -7.8105043683733255, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.08399472917829254, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-036", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Were the assumptions underlying the estimate to change, the conclusion would", + "continuations": [ + " have assumptions beneath it.", + " changing was estimated.", + " underlie tomorrow.", + " need to be reconsidered." + ], + "tags": [ + "conditional" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -96.72835540771484, + "mean_logprob": -12.091044425964355, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.12917709350586, + "mean_logprob": -10.782294273376465, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -95.9933853149414, + "mean_logprob": -13.713340759277344, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -84.6192626953125, + "mean_logprob": -12.088466099330358, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.3061718259538928, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-037", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The policy, while defensible in principle, is difficult to", + "continuations": [ + " implement consistently.", + " principle the defense.", + " difficult implementation was.", + " consistency defending." + ], + "tags": [ + "concession" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.833030700683594, + "mean_logprob": -10.958257675170898, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.16511535644531, + "mean_logprob": -12.791278839111328, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -48.96788787841797, + "mean_logprob": -12.241971969604492, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -58.18153381347656, + "mean_logprob": -11.636306762695312, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.6780490875244141, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-038", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Only after the archive was digitized did researchers", + "continuations": [ + " digitized only before.", + " discover the missing correspondence.", + " archive the discovery did.", + " correspondingly research." + ], + "tags": [ + "inversion" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -63.94670104980469, + "mean_logprob": -10.657783508300781, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -64.166748046875, + "mean_logprob": -10.6944580078125, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -61.84107971191406, + "mean_logprob": -10.306846618652344, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -46.45665740966797, + "mean_logprob": -11.614164352416992, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.3509368896484375, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-039", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The editor objected less to the claim itself than to", + "continuations": [ + " claims having objections.", + " itself being less.", + " the certainty with which it was stated.", + " stating an editor." + ], + "tags": [ + "comparison" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.800628662109375, + "mean_logprob": -9.760125732421875, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.036048889160156, + "mean_logprob": -10.509012222290039, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -73.14825439453125, + "mean_logprob": -8.127583821614584, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -41.967681884765625, + "mean_logprob": -8.393536376953126, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.2659525553385418, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-040", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "So subtle was the change in tone that few listeners", + "continuations": [ + " made tone into change.", + " subtly were few.", + " had listened tomorrow.", + " noticed it." + ], + "tags": [ + "inversion" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -58.066802978515625, + "mean_logprob": -9.677800496419271, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.31622314453125, + "mean_logprob": -10.86324462890625, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.30300903320312, + "mean_logprob": -9.471858433314733, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -41.37181854248047, + "mean_logprob": -10.342954635620117, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.20594206310453878, + "context_truncated": false, + "elapsed_seconds": 0.0010053064979729243 + }, + { + "id": "language_completion-041", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The proposal is neither as costly as its critics allege nor as simple as its supporters", + "continuations": [ + " suggest.", + " allegation was cost.", + " supporting simply.", + " critics and costly." + ], + "tags": [ + "parallelism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.670267105102539, + "mean_logprob": -6.8351335525512695, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -66.17291259765625, + "mean_logprob": -11.028818766276041, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.86675262451172, + "mean_logprob": -10.62225087483724, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -63.912723541259766, + "mean_logprob": -10.652120590209961, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 3.7871173222859706, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-042", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Whatever reservations the panel may have had, they were not sufficient to", + "continuations": [ + " reserve a panel seat.", + " block the appointment.", + " appoint sufficient reservations.", + " having been whatever." + ], + "tags": [ + "concession" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -74.70632934570312, + "mean_logprob": -12.451054890950521, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -48.33087921142578, + "mean_logprob": -9.666175842285156, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -64.36131286621094, + "mean_logprob": -12.872262573242187, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.400177001953125, + "mean_logprob": -10.100044250488281, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.433868408203125, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-043", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Had the data been interpreted in isolation, the apparent trend might have", + "continuations": [ + " isolated its data.", + " been interpreting apparent.", + " seemed more convincing.", + " trended the isolation." + ], + "tags": [ + "counterfactual" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -46.59675598144531, + "mean_logprob": -11.649188995361328, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -48.5769157409668, + "mean_logprob": -9.715383148193359, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -42.85797882080078, + "mean_logprob": -7.142996470133464, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -53.42365264892578, + "mean_logprob": -8.903942108154297, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.760945638020833, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-044", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The mayor's apology did little to allay concerns that the inquiry had been", + "continuations": [ + " mayoral and apologetic.", + " concerned into inquiry.", + " doing little apology.", + " deliberately delayed." + ], + "tags": [ + "collocation" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -103.36712646484375, + "mean_logprob": -14.766732352120536, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -72.52577209472656, + "mean_logprob": -12.087628682454428, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -68.64405822753906, + "mean_logprob": -13.728811645507813, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -67.62759399414062, + "mean_logprob": -11.271265665690104, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.8163630167643241, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-045", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Not until the final chapter does the significance of the opening scene become", + "continuations": [ + " fully apparent.", + " final before opening.", + " chaptered significantly.", + " opening does final." + ], + "tags": [ + "inversion" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -28.48978614807129, + "mean_logprob": -9.49659538269043, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.550933837890625, + "mean_logprob": -9.887733459472656, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -62.891265869140625, + "mean_logprob": -12.578253173828125, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -45.333282470703125, + "mean_logprob": -11.333320617675781, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.39113807678222656, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-046", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The theory's appeal lies not in its novelty but in its ability to", + "continuations": [ + " appeal theoretically.", + " unify previously separate observations.", + " become novel before theory.", + " observe its ability." + ], + "tags": [ + "semantics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -80.18983459472656, + "mean_logprob": -13.364972432454428, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -77.9293212890625, + "mean_logprob": -12.98822021484375, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -76.42617797851562, + "mean_logprob": -15.285235595703124, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -49.95795440673828, + "mean_logprob": -12.48948860168457, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.4987316131591797, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-047", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Much as the negotiators wanted an agreement, neither side was prepared to", + "continuations": [ + " negotiate wanting.", + " agreement much as.", + " concede the central point.", + " prepare neither side." + ], + "tags": [ + "concession" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -61.35455322265625, + "mean_logprob": -12.27091064453125, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -48.00559997558594, + "mean_logprob": -12.001399993896484, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -71.50630187988281, + "mean_logprob": -10.215185982840401, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.478477478027344, + "mean_logprob": -11.869619369506836, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.654433386666435, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-048", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Whether the discrepancy reflects measurement error or a genuine anomaly remains", + "continuations": [ + " measuring a reflection.", + " discrepant in error.", + " anomalously genuine.", + " uncertain." + ], + "tags": [ + "syntax" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -66.70771026611328, + "mean_logprob": -13.341542053222657, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -69.73980712890625, + "mean_logprob": -11.623301188151041, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -99.80760955810547, + "mean_logprob": -12.475951194763184, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -18.677371978759766, + "mean_logprob": -9.338685989379883, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 2.2846151987711583, + "context_truncated": false, + "elapsed_seconds": 0.0010622688751027454 + }, + { + "id": "language_completion-049", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The ruling established a precedent whose consequences would not become clear", + "continuations": [ + " for several years.", + " precedent had ruled.", + " consequences establishing.", + " clearly whose." + ], + "tags": [ + "relative_clause" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -38.95664978027344, + "mean_logprob": -9.73916244506836, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -67.76598358154297, + "mean_logprob": -11.294330596923828, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.18573760986328, + "mean_logprob": -8.04643440246582, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -20.076053619384766, + "mean_logprob": -6.692017873128255, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.3544165293375654, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "language_completion-050", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "To attribute the recovery solely to the new policy would be to", + "continuations": [ + " recover sole attribution.", + " ignore several contributing factors.", + " policy the recovery solely.", + " contribute an attribution." + ], + "tags": [ + "idiom" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -55.3746223449707, + "mean_logprob": -11.07492446899414, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -66.4998779296875, + "mean_logprob": -9.499982561383929, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -64.44564056396484, + "mean_logprob": -10.74094009399414, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -63.54052734375, + "mean_logprob": -12.70810546875, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.2409575326102118, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-001", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A glass slipped from the table onto a tile floor. It most likely", + "continuations": [ + " grew roots.", + " floated upward.", + " shattered.", + " became warmer than fire." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -40.723602294921875, + "mean_logprob": -13.574534098307291, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.73421096801758, + "mean_logprob": -10.946842193603516, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.7600212097168, + "mean_logprob": -11.1900053024292, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -85.60201263427734, + "mean_logprob": -14.26700210571289, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.2431631088256836, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-002", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Jules wanted to mail a letter, so he put a stamp on the", + "continuations": [ + " refrigerator.", + " envelope.", + " sidewalk.", + " spoon." + ], + "tags": [ + "everyday" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -59.94166564941406, + "mean_logprob": -11.988333129882813, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -52.943904876708984, + "mean_logprob": -13.235976219177246, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.79307556152344, + "mean_logprob": -10.69826889038086, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.832401275634766, + "mean_logprob": -10.944133758544922, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.2458648681640625, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-003", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The room was dark until Ana flipped the wall switch and the", + "continuations": [ + " carpet vanished.", + " window froze.", + " lights came on.", + " ceiling became wet." + ], + "tags": [ + "causality" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -63.8540153503418, + "mean_logprob": -12.770803070068359, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -49.6280403137207, + "mean_logprob": -12.407010078430176, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.201934814453125, + "mean_logprob": -8.550483703613281, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -59.68330764770508, + "mean_logprob": -11.936661529541016, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 3.386177825927735, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-004", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Before slicing a loaf of bread, most people reach for", + "continuations": [ + " a paintbrush.", + " a blanket.", + " a shoe.", + " a knife." + ], + "tags": [ + "tool_use" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -72.1986083984375, + "mean_logprob": -14.4397216796875, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.29408264160156, + "mean_logprob": -10.858816528320313, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.43421936035156, + "mean_logprob": -11.60855484008789, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -49.9644660949707, + "mean_logprob": -12.491116523742676, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.7497383117675778, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-005", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a houseplant's soil is very dry, the plant probably needs", + "continuations": [ + " water.", + " a password.", + " louder music.", + " a winter coat." + ], + "tags": [ + "everyday" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -24.758665084838867, + "mean_logprob": -12.379332542419434, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -49.84586715698242, + "mean_logprob": -12.461466789245605, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -67.51216888427734, + "mean_logprob": -13.502433776855469, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -65.7138671875, + "mean_logprob": -13.1427734375, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.08213424682617188, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-006", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Omar put leftovers in the refrigerator to keep them", + "continuations": [ + " invisible.", + " cold.", + " expensive.", + " rectangular." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -50.58351135253906, + "mean_logprob": -12.645877838134766, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.73167610168457, + "mean_logprob": -10.365838050842285, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -23.775691986083984, + "mean_logprob": -11.887845993041992, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -50.856380462646484, + "mean_logprob": -12.714095115661621, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.522007942199707, + "context_truncated": false, + "elapsed_seconds": 0.0010075001264340244 + }, + { + "id": "commonsense-007", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Someone standing outside in heavy rain without shelter will probably get", + "continuations": [ + " taller.", + " younger.", + " wet.", + " magnetic." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.008403778076172, + "mean_logprob": -8.66946792602539, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.364421844482422, + "mean_logprob": -9.182210922241211, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.620254516601562, + "mean_logprob": -8.810127258300781, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -20.213871002197266, + "mean_logprob": -10.106935501098633, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.14065933227539062, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-008", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A cyclist approaching a red traffic light should", + "continuations": [ + " accelerate through every lane.", + " close both eyes.", + " remove the wheels.", + " stop." + ], + "tags": [ + "safety" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -76.72332000732422, + "mean_logprob": -10.960474286760602, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.537193298339844, + "mean_logprob": -9.884298324584961, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -57.454341888427734, + "mean_logprob": -11.490868377685548, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -20.746044158935547, + "mean_logprob": -10.373022079467773, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.4887237548828125, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-009", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "To find out whether someone has a fever, a nurse commonly uses", + "continuations": [ + " a thermometer.", + " a ruler.", + " a compass.", + " a stapler." + ], + "tags": [ + "tool_use" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -69.10940551757812, + "mean_logprob": -11.518234252929688, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -47.94202423095703, + "mean_logprob": -11.985506057739258, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -45.666664123535156, + "mean_logprob": -11.416666030883789, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -59.378814697265625, + "mean_logprob": -9.896469116210938, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.5201969146728516, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-010", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "When the phone battery reaches zero, the phone usually", + "continuations": [ + " doubles in size.", + " turns off.", + " prints a book.", + " becomes waterproof." + ], + "tags": [ + "everyday" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.727535247802734, + "mean_logprob": -10.545507049560547, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -30.384506225585938, + "mean_logprob": -10.128168741861979, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -42.40394592285156, + "mean_logprob": -8.480789184570312, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -56.16632080078125, + "mean_logprob": -11.23326416015625, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.647379557291666, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-011", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A person who has not eaten all day is likely to feel", + "continuations": [ + " polished.", + " transparent.", + " hungry.", + " metallic." + ], + "tags": [ + "human_needs" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -37.186885833740234, + "mean_logprob": -12.395628611246744, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -42.52629089355469, + "mean_logprob": -14.175430297851562, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -60.72548294067383, + "mean_logprob": -15.181370735168457, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -44.21653747558594, + "mean_logprob": -11.054134368896484, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.3414942423502598, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-012", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "After washing wet clothes, a common next step is to", + "continuations": [ + " bury them in concrete.", + " mail them overseas.", + " paint them black.", + " dry them." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -74.89471435546875, + "mean_logprob": -12.482452392578125, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -77.94363403320312, + "mean_logprob": -11.134804861886161, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -51.90760040283203, + "mean_logprob": -12.976900100708008, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -41.840511322021484, + "mean_logprob": -13.946837107340494, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.3476475306919635, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-013", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "If a candle flame is blown out, the room may become", + "continuations": [ + " darker.", + " heavier than a car.", + " filled with snow.", + " twice as wide." + ], + "tags": [ + "causality" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.535093307495117, + "mean_logprob": -8.845031102498373, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -60.8208122253418, + "mean_logprob": -10.136802037556967, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -43.855533599853516, + "mean_logprob": -10.963883399963379, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -25.6857852935791, + "mean_logprob": -6.421446323394775, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 2.4235847791035976, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-014", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "To unlock a typical front door from outside, a resident may use", + "continuations": [ + " soap.", + " a key.", + " a pillow.", + " a forkful of rice." + ], + "tags": [ + "tool_use" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -36.2718505859375, + "mean_logprob": -12.090616861979166, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.43535614013672, + "mean_logprob": -9.811785380045572, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -56.61265563964844, + "mean_logprob": -11.322531127929688, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -71.02947998046875, + "mean_logprob": -10.147068568638392, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.33528318859282, + "context_truncated": false, + "elapsed_seconds": 0.0009957843758456875 + }, + { + "id": "commonsense-015", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Fresh snow on a sidewalk can make the surface", + "continuations": [ + " spicy.", + " musical.", + " slippery.", + " edible." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -37.117515563964844, + "mean_logprob": -12.372505187988281, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.776208877563477, + "mean_logprob": -9.888104438781738, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -51.23200225830078, + "mean_logprob": -12.808000564575195, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.534032821655273, + "mean_logprob": -9.511344273885092, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.37676016489664654, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-016", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a smoke alarm sounds and smoke is visible, people should", + "continuations": [ + " finish a long nap.", + " hide the alarm.", + " open a novel.", + " leave the building safely." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -77.61210632324219, + "mean_logprob": -11.08744376046317, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -73.71466064453125, + "mean_logprob": -12.285776774088541, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.2942008972168, + "mean_logprob": -11.0735502243042, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -62.91383743286133, + "mean_logprob": -10.485639572143555, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.5879106521606445, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-017", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A ceramic mug filled with hot tea may be too hot to", + "continuations": [ + " hold comfortably.", + " remember a name.", + " become ceramic.", + " count backward." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.53901672363281, + "mean_logprob": -9.884754180908203, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -52.583984375, + "mean_logprob": -13.14599609375, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -53.75154495239258, + "mean_logprob": -10.750308990478516, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -64.00326538085938, + "mean_logprob": -16.000816345214844, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.8655548095703125, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-018", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Tara heard thunder only a few seconds after seeing lightning, suggesting the storm was", + "continuations": [ + " beneath the ocean.", + " nearby.", + " made of glass.", + " over last year." + ], + "tags": [ + "causality" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -51.888553619384766, + "mean_logprob": -8.648092269897461, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.036344528198242, + "mean_logprob": -5.018172264099121, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.160125732421875, + "mean_logprob": -8.290031433105469, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.95135498046875, + "mean_logprob": -10.737838745117188, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 3.2718591690063477, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-019", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The freezer door was left open overnight. In the morning, the ice cream was probably", + "continuations": [ + " sharper.", + " newly packaged.", + " soft or melted.", + " louder." + ], + "tags": [ + "causality" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -32.89330291748047, + "mean_logprob": -10.964434305826822, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.016117095947266, + "mean_logprob": -12.754029273986816, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -47.313568115234375, + "mean_logprob": -9.462713623046875, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -45.852210998535156, + "mean_logprob": -11.463052749633789, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.501720682779947, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-020", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Ravi arrived early for an interview but did not want to interrupt the meeting inside, so he", + "continuations": [ + " shouted through the door.", + " canceled every meeting.", + " moved the building.", + " waited quietly." + ], + "tags": [ + "social" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -69.87550354003906, + "mean_logprob": -9.982214791434151, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -77.17655944824219, + "mean_logprob": -12.862759908040365, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.796607971191406, + "mean_logprob": -9.699151992797852, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -72.27806091308594, + "mean_logprob": -14.455612182617188, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.28306279863629946, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-021", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A recipe serves four people, but eight guests are expected. A sensible cook would", + "continuations": [ + " double the ingredients.", + " halve every plate.", + " freeze the oven.", + " omit all food." + ], + "tags": [ + "planning" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.60905075073242, + "mean_logprob": -9.152262687683105, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -82.67689514160156, + "mean_logprob": -13.77948252360026, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -56.72920227050781, + "mean_logprob": -9.454867045084635, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -55.587310791015625, + "mean_logprob": -11.117462158203125, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.30260435740152936, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-022", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Mina's laptop became very hot and its fan ran constantly, so she first checked whether", + "continuations": [ + " the keyboard knew her name.", + " its vents were blocked.", + " the screen was square.", + " noon had arrived." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -91.30882263183594, + "mean_logprob": -13.044117518833705, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -70.70335388183594, + "mean_logprob": -10.100479125976562, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.43141174316406, + "mean_logprob": -9.886282348632813, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -47.6810417175293, + "mean_logprob": -9.536208343505859, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.3500740051269542, + "context_truncated": false, + "elapsed_seconds": 0.0010254098779114429 + }, + { + "id": "commonsense-023", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A friend says, \"I just lost my job.\" The most considerate immediate response is to", + "continuations": [ + " ask for a loan.", + " change the subject to sports.", + " express sympathy and listen.", + " congratulate them." + ], + "tags": [ + "social" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -56.487030029296875, + "mean_logprob": -9.414505004882812, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -68.53366088867188, + "mean_logprob": -11.422276814778646, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -81.60630798339844, + "mean_logprob": -11.658043997628349, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -63.744911193847656, + "mean_logprob": -10.624151865641275, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.209646860758463, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-024", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The label says a medicine may cause drowsiness. After taking it, a person should avoid", + "continuations": [ + " drinking water.", + " reading the label.", + " sitting safely.", + " driving a car." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -30.732694625854492, + "mean_logprob": -10.244231541951498, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -41.53546905517578, + "mean_logprob": -10.383867263793945, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -48.29985046386719, + "mean_logprob": -12.074962615966797, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -45.09211730957031, + "mean_logprob": -11.273029327392578, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.13963572184244732, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-025", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Water is dripping from the ceiling directly below an upstairs bathroom. The leak likely", + "continuations": [ + " originates near that bathroom.", + " comes from sunlight.", + " proves the roof is dry.", + " makes water weightless." + ], + "tags": [ + "diagnosis" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -78.52243041992188, + "mean_logprob": -11.217490059988839, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.321199417114258, + "mean_logprob": -7.8302998542785645, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -79.20344543457031, + "mean_logprob": -11.31477791922433, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -59.86050033569336, + "mean_logprob": -11.972100067138673, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 3.387190205710274, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-026", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The car's fuel gauge is nearly empty and the next town is far away. The prudent action is to", + "continuations": [ + " increase tire size.", + " refuel before leaving.", + " turn off the steering wheel.", + " ignore all signs." + ], + "tags": [ + "planning" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -66.72314453125, + "mean_logprob": -13.34462890625, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -62.02568817138672, + "mean_logprob": -12.405137634277343, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -53.654876708984375, + "mean_logprob": -7.664982386997768, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -58.33391571044922, + "mean_logprob": -11.666783142089844, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 4.001800755092076, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-027", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A package marked \"fragile\" should generally be", + "continuations": [ + " shaken to test it.", + " left in rain.", + " handled carefully.", + " used as a chair." + ], + "tags": [ + "everyday" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -64.69111633300781, + "mean_logprob": -10.781852722167969, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -52.31129455566406, + "mean_logprob": -13.077823638916016, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -58.94094467163086, + "mean_logprob": -14.735236167907715, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -56.72498321533203, + "mean_logprob": -9.454163869222006, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.327688852945963, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-028", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Kai cannot hear the television, but everyone else can. Before replacing it, Kai should check", + "continuations": [ + " whether the walls are blue.", + " how much the sofa weighs.", + " if tomorrow is Tuesday.", + " the volume and audio settings." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -51.22703552246094, + "mean_logprob": -8.53783925374349, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -95.0560073852539, + "mean_logprob": -9.505600738525391, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -115.19352722167969, + "mean_logprob": -12.799280802408854, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -70.07656860351562, + "mean_logprob": -10.010938371930804, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.9677614847819012, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-029", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A wet floor sign in a hallway warns pedestrians that they should", + "continuations": [ + " walk carefully.", + " run faster.", + " turn off every light.", + " remove their shoes immediately." + ], + "tags": [ + "safety" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -35.55739974975586, + "mean_logprob": -11.852466583251953, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.712501525878906, + "mean_logprob": -9.570833841959635, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -66.49526977539062, + "mean_logprob": -13.299053955078126, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -61.575286865234375, + "mean_logprob": -10.262547810872396, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.6917139689127616, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-030", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Nico promised to return a borrowed book on Friday but needs it until Monday. The responsible choice is to", + "continuations": [ + " hide the book.", + " ask the owner for an extension.", + " claim Friday was canceled.", + " lend it to a stranger." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -44.67500305175781, + "mean_logprob": -8.935000610351562, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -78.50614929199219, + "mean_logprob": -9.813268661499023, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -102.62958526611328, + "mean_logprob": -11.403287251790365, + "total_sequence_tokens": 37 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -66.84966278076172, + "mean_logprob": -8.356207847595215, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.578792762756347, + "context_truncated": false, + "elapsed_seconds": 0.0010461728743393905 + }, + { + "id": "commonsense-031", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A refrigerator is running, but food inside is warm. One useful first check is whether", + "continuations": [ + " the food can read.", + " the kitchen is rectangular.", + " the door seals completely.", + " the shelves are named." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -42.72696304321289, + "mean_logprob": -8.545392608642578, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -85.30767822265625, + "mean_logprob": -10.663459777832031, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -70.69599151611328, + "mean_logprob": -11.782665252685547, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -65.2733383178711, + "mean_logprob": -10.878889719645182, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 2.1180671691894535, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-032", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A hiker sees dark clouds approaching and has several hours left on an exposed ridge. The safer choice is to", + "continuations": [ + " climb the tallest tree.", + " hold a metal pole.", + " wait for lightning.", + " descend toward shelter." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -81.91192626953125, + "mean_logprob": -11.701703752790179, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -74.21255493164062, + "mean_logprob": -12.368759155273438, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -51.33500289916992, + "mean_logprob": -10.267000579833985, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -82.29818725585938, + "mean_logprob": -13.716364542643229, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 1.434703172956194, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-033", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "An egg dropped onto a thick cushion is less likely to break because the cushion", + "continuations": [ + " reduces the sudden impact.", + " makes the egg colder.", + " removes gravity.", + " turns shell into rubber." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -46.296531677246094, + "mean_logprob": -9.259306335449219, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -55.959754943847656, + "mean_logprob": -9.32662582397461, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -43.83283615112305, + "mean_logprob": -8.76656723022461, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -80.51770782470703, + "mean_logprob": -13.419617970784506, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.49273910522460973, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-034", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A metal jar lid is stuck. Running warm water over the lid can help because the metal may", + "continuations": [ + " lose all mass.", + " expand slightly.", + " stop being metal.", + " become colder than ice." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -31.753719329833984, + "mean_logprob": -7.938429832458496, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -19.55299949645996, + "mean_logprob": -6.517666498819987, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -47.487396240234375, + "mean_logprob": -11.871849060058594, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -74.0447769165039, + "mean_logprob": -12.34079615275065, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.4207633336385088, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-035", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Lena receives an email demanding immediate payment through gift cards and threatening arrest. The message is most likely", + "continuations": [ + " a routine tax receipt.", + " a weather alert.", + " a scam.", + " a library reminder." + ], + "tags": [ + "digital_safety" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -81.69229888916016, + "mean_logprob": -11.670328412737165, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -67.12223815917969, + "mean_logprob": -13.424447631835937, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -40.71141052246094, + "mean_logprob": -10.177852630615234, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -48.515682220458984, + "mean_logprob": -9.703136444091797, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.4747161865234375, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-036", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A sealed plastic bottle dents inward after hot air inside cools. The best explanation is that", + "continuations": [ + " the bottle gained extra air.", + " cooling erased the plastic.", + " gravity reversed briefly.", + " pressure inside decreased." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -93.06370544433594, + "mean_logprob": -13.294815063476562, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -82.13477325439453, + "mean_logprob": -13.689128875732422, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -87.49702453613281, + "mean_logprob": -10.937128067016602, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.95252227783203, + "mean_logprob": -10.238130569458008, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.6989974975585938, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-037", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A colleague repeatedly misses deadlines because requirements change without notice. The most constructive first step is to", + "continuations": [ + " document changes and clarify priorities.", + " publicly insult the colleague.", + " silently redo all work forever.", + " delete the project." + ], + "tags": [ + "workplace" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -74.88948059082031, + "mean_logprob": -9.361185073852539, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -96.92301940917969, + "mean_logprob": -12.115377426147461, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -124.88504028320312, + "mean_logprob": -13.87611558702257, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -58.881126403808594, + "mean_logprob": -11.776225280761718, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 2.415040206909179, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-038", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A carbon monoxide alarm sounds, but nobody can smell smoke. Occupants should still", + "continuations": [ + " assume silence means safety.", + " leave and contact emergency services.", + " search with a flame.", + " unplug it and sleep." + ], + "tags": [ + "safety" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -77.61759948730469, + "mean_logprob": -12.936266581217447, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -63.90174865722656, + "mean_logprob": -10.650291442871094, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -70.51573181152344, + "mean_logprob": -11.75262196858724, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -77.00824737548828, + "mean_logprob": -11.001178196498326, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.35088675362723265, + "context_truncated": false, + "elapsed_seconds": 0.0010712743751355447 + }, + { + "id": "commonsense-039", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A wooden drawer sticks only during humid weather. The most plausible reason is that the wood", + "continuations": [ + " becomes electrically charged.", + " forgets its shape.", + " swells after absorbing moisture.", + " turns into metal." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -58.28725051879883, + "mean_logprob": -11.657450103759766, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -51.55220413208008, + "mean_logprob": -10.310440826416016, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -81.57727813720703, + "mean_logprob": -10.197159767150879, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -46.83561325073242, + "mean_logprob": -11.708903312683105, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.11328105926513743, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-040", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Sam hears a rumor about a coworker from someone who admits they were not present. Before repeating it, Sam should", + "continuations": [ + " add dramatic details.", + " post it anonymously.", + " treat it as confirmed.", + " verify the information." + ], + "tags": [ + "social" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -44.62236785888672, + "mean_logprob": -11.15559196472168, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -63.20180130004883, + "mean_logprob": -10.533633550008139, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -45.854949951171875, + "mean_logprob": -9.170989990234375, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -59.438209533691406, + "mean_logprob": -11.887641906738281, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 1.3626435597737636, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-041", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Two identical wet towels are hung up, one spread flat and one crumpled. The spread towel usually dries first because it has", + "continuations": [ + " more surface exposed to air.", + " less fabric.", + " no water initially.", + " a different mass forever." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -61.15409851074219, + "mean_logprob": -10.192349751790365, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.400310516357422, + "mean_logprob": -8.46677017211914, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.875732421875, + "mean_logprob": -9.21893310546875, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -74.65986633300781, + "mean_logprob": -12.443311055501303, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.7521629333496094, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-042", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A meeting invite lists 3 p.m. in another time zone. To avoid arriving at the wrong time, a participant should", + "continuations": [ + " ignore the time zone.", + " convert it to local time.", + " attend at every hour.", + " change the calendar date randomly." + ], + "tags": [ + "planning" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -69.88697814941406, + "mean_logprob": -11.64782969156901, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -66.18342590332031, + "mean_logprob": -9.454775129045759, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -32.01374435424805, + "mean_logprob": -6.402748870849609, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -111.71276092529297, + "mean_logprob": -12.412528991699219, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 3.0520262581961495, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-043", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A pan of oil catches fire on a stove. The cook should turn off the heat and", + "continuations": [ + " pour water into the oil.", + " carry the burning pan outside.", + " cover the pan with a lid.", + " fan the flames." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -72.57276916503906, + "mean_logprob": -10.367538452148438, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -68.7281723022461, + "mean_logprob": -11.454695383707682, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -70.72897338867188, + "mean_logprob": -8.841121673583984, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -60.55912399291992, + "mean_logprob": -10.09318733215332, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.252065658569336, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-044", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "After a software update, a printer stops appearing on one computer but works from others. The issue is most likely with", + "continuations": [ + " all paper ever made.", + " the building's plumbing.", + " the printer's physical existence.", + " that computer's configuration." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -53.51640701293945, + "mean_logprob": -10.70328140258789, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -80.12954711914062, + "mean_logprob": -11.447078159877233, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.68590545654297, + "mean_logprob": -9.526557922363281, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -53.005882263183594, + "mean_logprob": -8.8343137105306, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.6922442118326817, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-045", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A person feels dizzy after standing up very quickly. Sitting back down helps primarily by", + "continuations": [ + " reducing the immediate risk of falling.", + " making the floor warmer.", + " stopping all circulation.", + " increasing their height." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -61.14592742919922, + "mean_logprob": -8.735132489885602, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -67.02374267578125, + "mean_logprob": -11.170623779296875, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -69.26512145996094, + "mean_logprob": -11.54418690999349, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.35450744628906, + "mean_logprob": -10.588626861572266, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.8534943716866632, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-046", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A sealed bag of chips puffs up during a drive to high altitude because outside air pressure", + "continuations": [ + " becomes much higher.", + " becomes lower.", + " changes the chips into gas.", + " disappears completely." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -47.28535842895508, + "mean_logprob": -11.82133960723877, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -39.847679138183594, + "mean_logprob": -13.282559712727865, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -97.33935546875, + "mean_logprob": -13.905622209821429, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -51.492218017578125, + "mean_logprob": -12.873054504394531, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.0517148971557617, + "context_truncated": false, + "elapsed_seconds": 0.001046849247359205 + }, + { + "id": "commonsense-047", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A manager asks for feedback but becomes defensive whenever criticism is offered. Employees will likely become", + "continuations": [ + " more candid each time.", + " unable to remember work.", + " reluctant to speak honestly.", + " certain all criticism vanished." + ], + "tags": [ + "social" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -43.18633270263672, + "mean_logprob": -8.637266540527344, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -47.585777282714844, + "mean_logprob": -9.51715545654297, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -112.59152221679688, + "mean_logprob": -11.259152221679688, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -96.17839050292969, + "mean_logprob": -13.739770071847099, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.8798889160156254, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "commonsense-048", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A parked car has one window fogged on the inside while the others are clear. This suggests moisture is concentrated", + "continuations": [ + " inside the engine only.", + " on the road beneath.", + " in the sunlight.", + " near that window." + ], + "tags": [ + "diagnosis" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.48302459716797, + "mean_logprob": -9.696604919433593, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -77.11764526367188, + "mean_logprob": -11.016806466238839, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -35.34257125854492, + "mean_logprob": -8.83564281463623, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.05382537841797, + "mean_logprob": -11.763456344604492, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.8609621047973626, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "commonsense-049", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A child apologizes only after being caught and immediately repeats the behavior. The apology alone is weak evidence of", + "continuations": [ + " genuine change.", + " having been caught.", + " the behavior occurring.", + " spoken language." + ], + "tags": [ + "social" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -59.69689178466797, + "mean_logprob": -11.939378356933593, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.443111419677734, + "mean_logprob": -9.110777854919434, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.12571716308594, + "mean_logprob": -8.031429290771484, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -52.32215881347656, + "mean_logprob": -13.08053970336914, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.0793485641479492, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "commonsense-050", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A rechargeable battery lasts much less time in freezing weather but recovers indoors. The temporary loss is most plausibly due to", + "continuations": [ + " permanent deletion of electricity.", + " slower chemical reactions in the cold.", + " the device becoming heavier.", + " indoor clocks running faster." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -90.413330078125, + "mean_logprob": -12.916190011160714, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -75.50039672851562, + "mean_logprob": -9.437549591064453, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -53.956398010253906, + "mean_logprob": -8.992733001708984, + "total_sequence_tokens": 39 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -82.3921127319336, + "mean_logprob": -11.770301818847656, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.44481658935546875, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "world_knowledge-001", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The capital city of France is", + "continuations": [ + " Vienna.", + " Madrid.", + " Paris.", + " Lisbon." + ], + "tags": [ + "geography" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -47.03569030761719, + "mean_logprob": -11.758922576904297, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -32.64112091064453, + "mean_logprob": -10.88037363688151, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.194250106811523, + "mean_logprob": -9.597125053405762, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.21107482910156, + "mean_logprob": -8.55276870727539, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 1.044356346130371, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "world_knowledge-002", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The planet known for its prominent rings is", + "continuations": [ + " Mercury.", + " Saturn.", + " Earth.", + " Mars." + ], + "tags": [ + "science" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -31.615875244140625, + "mean_logprob": -10.538625081380209, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.236270904541016, + "mean_logprob": -12.078756968180338, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -20.732372283935547, + "mean_logprob": -10.366186141967773, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -18.284561157226562, + "mean_logprob": -9.142280578613281, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 1.2239055633544922, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "world_knowledge-003", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Water freezes at zero degrees on the", + "continuations": [ + " Kelvin scale.", + " Fahrenheit scale.", + " Celsius scale.", + " Beaufort scale." + ], + "tags": [ + "science" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -75.57942199707031, + "mean_logprob": -15.115884399414062, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -103.91032409667969, + "mean_logprob": -14.844332013811384, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -72.78041076660156, + "mean_logprob": -14.556082153320313, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -69.4190673828125, + "mean_logprob": -13.8838134765625, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.6722686767578132, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "world_knowledge-004", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The author of the play Hamlet was", + "continuations": [ + " Charles Dickens.", + " Jane Austen.", + " Mark Twain.", + " William Shakespeare." + ], + "tags": [ + "literature" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.200035095214844, + "mean_logprob": -9.640007019042969, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -58.19147491455078, + "mean_logprob": -11.638294982910157, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.48804473876953, + "mean_logprob": -8.372011184692383, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -81.78927612304688, + "mean_logprob": -13.631546020507812, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.267995834350586, + "context_truncated": false, + "elapsed_seconds": 0.0010076776234200224 + }, + { + "id": "world_knowledge-005", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The largest ocean on Earth is the", + "continuations": [ + " Pacific Ocean.", + " Indian Ocean.", + " Arctic Ocean.", + " Atlantic Ocean." + ], + "tags": [ + "geography" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.774383544921875, + "mean_logprob": -9.924794514973959, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.2781982421875, + "mean_logprob": -9.426066080729166, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -36.976966857910156, + "mean_logprob": -12.325655619303385, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.11210823059082, + "mean_logprob": -10.03736941019694, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.49872843424479285, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-006", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Plants primarily absorb water through their", + "continuations": [ + " flowers.", + " roots.", + " seeds.", + " fruit." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.34701919555664, + "mean_logprob": -9.67350959777832, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -24.186931610107422, + "mean_logprob": -12.093465805053711, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -24.646060943603516, + "mean_logprob": -12.323030471801758, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -25.715824127197266, + "mean_logprob": -12.857912063598633, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 2.4199562072753906, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-007", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The chemical symbol for oxygen is", + "continuations": [ + " Ox.", + " G.", + " O.", + " Og." + ], + "tags": [ + "chemistry" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.348508834838867, + "mean_logprob": -8.674254417419434, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.964183807373047, + "mean_logprob": -9.982091903686523, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.613372802734375, + "mean_logprob": -10.806686401367188, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.193275451660156, + "mean_logprob": -10.064425150553385, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.3078374862670898, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-008", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The Nile River flows through the continent of", + "continuations": [ + " South America.", + " Australia.", + " Europe.", + " Africa." + ], + "tags": [ + "geography" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -36.405792236328125, + "mean_logprob": -12.135264078776041, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -25.159954071044922, + "mean_logprob": -12.579977035522461, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -20.497102737426758, + "mean_logprob": -10.248551368713379, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -25.585678100585938, + "mean_logprob": -12.792839050292969, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.8867127100626622, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-009", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A triangle has", + "continuations": [ + " three sides.", + " four sides.", + " five sides.", + " six sides." + ], + "tags": [ + "mathematics" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -40.79368209838867, + "mean_logprob": -13.597894032796225, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -40.04924011230469, + "mean_logprob": -13.349746704101562, + "total_sequence_tokens": 7 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -39.78889846801758, + "mean_logprob": -13.26296615600586, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -36.817726135253906, + "mean_logprob": -12.272575378417969, + "total_sequence_tokens": 7 + } + ], + "best_vs_second_margin": 0.9903907775878906, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-010", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The organ that pumps blood through the human body is the", + "continuations": [ + " liver.", + " heart.", + " lung.", + " stomach." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -26.166975021362305, + "mean_logprob": -13.083487510681152, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -22.080429077148438, + "mean_logprob": -11.040214538574219, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -27.850160598754883, + "mean_logprob": -13.925080299377441, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -28.11432647705078, + "mean_logprob": -14.05716323852539, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.0432729721069336, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-011", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Japan is an island country in", + "continuations": [ + " South America.", + " western Africa.", + " East Asia.", + " northern Europe." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.817405700683594, + "mean_logprob": -8.272468566894531, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.379985809326172, + "mean_logprob": -10.45999526977539, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.989120483398438, + "mean_logprob": -8.663040161132812, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.653724670410156, + "mean_logprob": -10.884574890136719, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.39057159423828125, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-012", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The first month of the Gregorian calendar is", + "continuations": [ + " March.", + " June.", + " December.", + " January." + ], + "tags": [ + "calendar" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.836162567138672, + "mean_logprob": -8.918081283569336, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.349063873291016, + "mean_logprob": -8.674531936645508, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.083362579345703, + "mean_logprob": -9.041681289672852, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -16.35659408569336, + "mean_logprob": -8.17829704284668, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.4962348937988281, + "context_truncated": false, + "elapsed_seconds": 0.0009528672489977907 + }, + { + "id": "world_knowledge-013", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Bees commonly produce", + "continuations": [ + " honey.", + " salt.", + " wool.", + " paper." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.321331024169922, + "mean_logprob": -9.660665512084961, + "total_sequence_tokens": 6 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -23.258583068847656, + "mean_logprob": -11.629291534423828, + "total_sequence_tokens": 6 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -33.850860595703125, + "mean_logprob": -11.283620198567709, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -22.81767463684082, + "mean_logprob": -11.40883731842041, + "total_sequence_tokens": 6 + } + ], + "best_vs_second_margin": 1.622954686482748, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-014", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The Great Pyramid of Giza is located in", + "continuations": [ + " Peru.", + " Egypt.", + " India.", + " Greece." + ], + "tags": [ + "history" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.98593521118164, + "mean_logprob": -8.661978403727213, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.759544372558594, + "mean_logprob": -9.379772186279297, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.21268653869629, + "mean_logprob": -8.106343269348145, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.542144775390625, + "mean_logprob": -10.847381591796875, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.5556351343790684, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-015", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The gas humans need to breathe for cellular respiration is", + "continuations": [ + " helium.", + " neon.", + " oxygen.", + " argon." + ], + "tags": [ + "biology" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.4013786315918, + "mean_logprob": -11.133792877197266, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.86238479614258, + "mean_logprob": -12.620794932047525, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.798431396484375, + "mean_logprob": -8.399215698242188, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -36.225120544433594, + "mean_logprob": -12.075040181477865, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 2.734577178955078, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-016", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A standard chessboard contains", + "continuations": [ + " forty squares.", + " fifty squares.", + " one hundred squares.", + " sixty-four squares." + ], + "tags": [ + "culture" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -58.06639099121094, + "mean_logprob": -11.613278198242188, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -56.32120132446289, + "mean_logprob": -11.264240264892578, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -53.39397430419922, + "mean_logprob": -10.678794860839844, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -100.57527160644531, + "mean_logprob": -11.175030178493923, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.49623531765407947, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-017", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The treaty that formally ended World War I between Germany and the Allied powers was the Treaty of", + "continuations": [ + " Versailles.", + " Utrecht.", + " Tordesillas.", + " Ghent." + ], + "tags": [ + "history" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -69.08995056152344, + "mean_logprob": -11.514991760253906, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -62.78604507446289, + "mean_logprob": -12.557209014892578, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -73.37225341796875, + "mean_logprob": -12.228708902994791, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -34.14386749267578, + "mean_logprob": -11.38128916422526, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.13370259602864643, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-018", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The process by which plants convert light energy into chemical energy is", + "continuations": [ + " fermentation.", + " photosynthesis.", + " respiration only.", + " erosion." + ], + "tags": [ + "biology" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -51.9575309753418, + "mean_logprob": -12.98938274383545, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -71.40357971191406, + "mean_logprob": -14.280715942382812, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -55.70222473144531, + "mean_logprob": -13.925556182861328, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -34.231685638427734, + "mean_logprob": -11.410561879475912, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.5788208643595372, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-019", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The longest river entirely within Europe is the", + "continuations": [ + " Danube.", + " Rhine.", + " Volga.", + " Seine." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.193052291870117, + "mean_logprob": -7.731017430623372, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.69841957092285, + "mean_logprob": -9.899473190307617, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.31609344482422, + "mean_logprob": -11.079023361206055, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.83823013305664, + "mean_logprob": -7.94607671101888, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.2150592803955078, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-020", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "In computing, RAM is primarily used for", + "continuations": [ + " permanent archival storage.", + " printing documents.", + " cooling the processor.", + " temporary working memory." + ], + "tags": [ + "computing" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.98257064819336, + "mean_logprob": -9.796514129638672, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.452281951904297, + "mean_logprob": -9.4840939839681, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -68.28852844238281, + "mean_logprob": -13.657705688476563, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -48.2769775390625, + "mean_logprob": -12.069244384765625, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.31242014567057197, + "context_truncated": false, + "elapsed_seconds": 0.0009726238749863114 + }, + { + "id": "world_knowledge-021", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The scientist who formulated the three laws of motion was", + "continuations": [ + " Isaac Newton.", + " Louis Pasteur.", + " Gregor Mendel.", + " Niels Bohr." + ], + "tags": [ + "science_history" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -67.20050811767578, + "mean_logprob": -11.200084686279297, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -58.51632308959961, + "mean_logprob": -11.703264617919922, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -97.884765625, + "mean_logprob": -13.983537946428571, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -80.16508483886719, + "mean_logprob": -11.452154976981026, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.25207029070172915, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-022", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The strait separating Spain from Morocco is the Strait of", + "continuations": [ + " Malacca.", + " Gibraltar.", + " Hormuz.", + " Bering." + ], + "tags": [ + "geography" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -50.20819854736328, + "mean_logprob": -12.55204963684082, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -57.04785919189453, + "mean_logprob": -9.507976531982422, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -54.19025421142578, + "mean_logprob": -10.838050842285156, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.814130783081055, + "mean_logprob": -10.604710261027018, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.0967337290445958, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-023", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A word with the opposite meaning of another word is called an", + "continuations": [ + " acronym.", + " homophone.", + " antonym.", + " anagram." + ], + "tags": [ + "language" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.00458526611328, + "mean_logprob": -8.75114631652832, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.250179290771484, + "mean_logprob": -9.812544822692871, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.63125228881836, + "mean_logprob": -8.40781307220459, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -29.956872940063477, + "mean_logprob": -9.985624313354492, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.34333324432373047, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-024", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The part of a cell that contains most of its genetic material is the", + "continuations": [ + " ribosome.", + " cell wall.", + " vacuole.", + " nucleus." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -62.65229797363281, + "mean_logprob": -12.530459594726562, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.74501037597656, + "mean_logprob": -11.581670125325521, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -48.22991180419922, + "mean_logprob": -12.057477951049805, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -41.16331100463867, + "mean_logprob": -13.72110366821289, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.47580782572428326, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-025", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The Renaissance began as a major cultural movement in", + "continuations": [ + " Italy.", + " Iceland.", + " Canada.", + " Mongolia." + ], + "tags": [ + "history" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -24.612625122070312, + "mean_logprob": -12.306312561035156, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.81353759765625, + "mean_logprob": -10.953384399414062, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -25.24436378479004, + "mean_logprob": -12.62218189239502, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -62.832122802734375, + "mean_logprob": -12.566424560546874, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.3529281616210938, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-026", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The SI unit of electrical resistance is the", + "continuations": [ + " watt.", + " ohm.", + " tesla.", + " joule." + ], + "tags": [ + "physics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.034900665283203, + "mean_logprob": -9.0116335550944, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -34.548770904541016, + "mean_logprob": -8.637192726135254, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -45.02874755859375, + "mean_logprob": -11.257186889648438, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.63640594482422, + "mean_logprob": -11.909101486206055, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.37444082895914654, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-027", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The novel Pride and Prejudice was written by", + "continuations": [ + " George Eliot.", + " Virginia Woolf.", + " Jane Austen.", + " Mary Shelley." + ], + "tags": [ + "literature" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.753318786621094, + "mean_logprob": -9.938329696655273, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -50.92729949951172, + "mean_logprob": -10.185459899902344, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -58.29786682128906, + "mean_logprob": -11.659573364257813, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -51.631141662597656, + "mean_logprob": -10.32622833251953, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.24713020324707102, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-028", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The layer of Earth's atmosphere containing most weather is the", + "continuations": [ + " mesosphere.", + " thermosphere.", + " exosphere.", + " troposphere." + ], + "tags": [ + "earth_science" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -63.00170135498047, + "mean_logprob": -12.600340270996094, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.00042724609375, + "mean_logprob": -12.750106811523438, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -57.00255584716797, + "mean_logprob": -14.250638961791992, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -61.13881301879883, + "mean_logprob": -12.227762603759766, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.3725776672363281, + "context_truncated": false, + "elapsed_seconds": 0.0010097131234942935 + }, + { + "id": "world_knowledge-029", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The Magna Carta was sealed in", + "continuations": [ + " England.", + " Brazil.", + " China.", + " Ethiopia." + ], + "tags": [ + "history" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -28.430805206298828, + "mean_logprob": -14.215402603149414, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -24.381694793701172, + "mean_logprob": -12.190847396850586, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -28.042194366455078, + "mean_logprob": -14.021097183227539, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -51.22348403930664, + "mean_logprob": -12.80587100982666, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.6150236129760742, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-030", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "In music, a group of three notes played together is commonly called a", + "continuations": [ + " scale.", + " chord.", + " tempo.", + " clef." + ], + "tags": [ + "music" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -20.276782989501953, + "mean_logprob": -10.138391494750977, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.340669631958008, + "mean_logprob": -8.446889877319336, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -43.268218994140625, + "mean_logprob": -10.817054748535156, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.249229431152344, + "mean_logprob": -9.083076477050781, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.6361865997314453, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-031", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The blood vessels that generally carry blood away from the heart are", + "continuations": [ + " capillaries only.", + " lymph nodes.", + " arteries.", + " alveoli." + ], + "tags": [ + "biology" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -72.70927429199219, + "mean_logprob": -14.541854858398438, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -52.89714813232422, + "mean_logprob": -13.224287033081055, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -49.04841613769531, + "mean_logprob": -16.349472045898438, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -50.266761779785156, + "mean_logprob": -12.566690444946289, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.6575965881347656, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-032", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The decimal number ten is written in binary as", + "continuations": [ + " 1001.", + " 1111.", + " 1100.", + " 1010." + ], + "tags": [ + "computing" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -61.26643753051758, + "mean_logprob": -10.21107292175293, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -54.082000732421875, + "mean_logprob": -9.013666788736979, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -59.8375244140625, + "mean_logprob": -9.972920735677084, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -61.27994918823242, + "mean_logprob": -10.213324864705404, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.9592539469401054, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-033", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The mountain range commonly used as part of the boundary between Europe and Asia is the", + "continuations": [ + " Ural Mountains.", + " Andes.", + " Alps.", + " Atlas Mountains." + ], + "tags": [ + "geography" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -53.562416076660156, + "mean_logprob": -10.712483215332032, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.27874755859375, + "mean_logprob": -12.092915852864584, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -35.723628997802734, + "mean_logprob": -11.907876332600912, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -44.744178771972656, + "mean_logprob": -8.948835754394532, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.7636474609375004, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-034", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The principle that no two electrons in an atom can share all four quantum numbers is the", + "continuations": [ + " Doppler effect.", + " Pauli exclusion principle.", + " uncertainty of measurement.", + " Hubble law." + ], + "tags": [ + "physics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -62.16017150878906, + "mean_logprob": -10.360028584798178, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -69.91123962402344, + "mean_logprob": -11.651873270670572, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -58.709815979003906, + "mean_logprob": -11.741963195800782, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -45.03229904174805, + "mean_logprob": -9.006459808349609, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.3535687764485687, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-035", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The 1648 agreements often associated with the modern state system are known as the Peace of", + "continuations": [ + " Augsburg.", + " Paris.", + " Westphalia.", + " Nijmegen." + ], + "tags": [ + "history" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -44.40636444091797, + "mean_logprob": -11.101591110229492, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.625598907470703, + "mean_logprob": -10.312799453735352, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -68.75965881347656, + "mean_logprob": -13.751931762695312, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -64.37521362304688, + "mean_logprob": -12.875042724609376, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.7887916564941406, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-036", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In linguistics, the smallest unit that carries meaning is a", + "continuations": [ + " phoneme in every case.", + " clause.", + " diphthong.", + " morpheme." + ], + "tags": [ + "linguistics" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -103.50489807128906, + "mean_logprob": -12.938112258911133, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.697126388549805, + "mean_logprob": -8.899042129516602, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -67.25273132324219, + "mean_logprob": -11.208788553873697, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -63.14899444580078, + "mean_logprob": -12.629798889160156, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 2.3097464243570958, + "context_truncated": false, + "elapsed_seconds": 0.0009847552464634646 + }, + { + "id": "world_knowledge-037", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The enzyme that unwinds the DNA double helix during replication is", + "continuations": [ + " helicase.", + " amylase.", + " lipase.", + " pepsin." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -48.804908752441406, + "mean_logprob": -12.201227188110352, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.21010208129883, + "mean_logprob": -10.552525520324707, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -37.57707214355469, + "mean_logprob": -12.525690714518229, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -68.41279602050781, + "mean_logprob": -13.682559204101562, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.6487016677856445, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-038", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The economic concept describing the value of the best forgone alternative is", + "continuations": [ + " sunk cost.", + " opportunity cost.", + " marginal revenue.", + " price elasticity." + ], + "tags": [ + "economics" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -48.049774169921875, + "mean_logprob": -12.012443542480469, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.1827392578125, + "mean_logprob": -12.0609130859375, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -81.47416687011719, + "mean_logprob": -13.579027811686197, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -62.209144592285156, + "mean_logprob": -12.44182891845703, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.04846954345703125, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-039", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The literary device in which an absent or imaginary person is directly addressed is", + "continuations": [ + " metonymy.", + " synecdoche.", + " apostrophe.", + " enjambment." + ], + "tags": [ + "literature" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -51.84832763671875, + "mean_logprob": -12.962081909179688, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -83.56167602539062, + "mean_logprob": -13.926946004231771, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -81.62198638916016, + "mean_logprob": -16.32439727783203, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -63.38931655883789, + "mean_logprob": -12.677863311767577, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.2842185974121101, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-040", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The ocean current that carries warm water across the North Atlantic is commonly called the", + "continuations": [ + " Humboldt Current.", + " Benguela Current.", + " Oyashio Current.", + " Gulf Stream." + ], + "tags": [ + "earth_science" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -73.06683349609375, + "mean_logprob": -12.177805582682291, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -76.078857421875, + "mean_logprob": -12.6798095703125, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -77.61825561523438, + "mean_logprob": -12.936375935872396, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -69.82350158691406, + "mean_logprob": -13.964700317382812, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.5020039876302089, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-041", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alloy traditionally made primarily from copper and tin is", + "continuations": [ + " bronze.", + " brass.", + " steel.", + " pewter." + ], + "tags": [ + "materials" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -45.98841857910156, + "mean_logprob": -11.49710464477539, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -39.316162109375, + "mean_logprob": -13.105387369791666, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.698444366455078, + "mean_logprob": -10.849222183227539, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -46.25318145751953, + "mean_logprob": -11.563295364379883, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.6478824615478516, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-042", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The mathematical constant that is the base of natural logarithms is", + "continuations": [ + " pi.", + " e.", + " phi.", + " tau." + ], + "tags": [ + "mathematics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.555421829223633, + "mean_logprob": -9.185140609741211, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.43297004699707, + "mean_logprob": -10.216485023498535, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -37.085323333740234, + "mean_logprob": -12.361774444580078, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -25.8942928314209, + "mean_logprob": -8.631430943806967, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.5537096659342442, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-043", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In databases, a key that uniquely identifies each row in a table is a", + "continuations": [ + " foreign schema.", + " null index.", + " primary key.", + " stored procedure." + ], + "tags": [ + "computing" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -57.70463943481445, + "mean_logprob": -11.54092788696289, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.427581787109375, + "mean_logprob": -10.106895446777344, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -36.33915328979492, + "mean_logprob": -12.113051096598307, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.812210083007812, + "mean_logprob": -10.270736694335938, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.16384124755859375, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-044", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The branch of philosophy concerned with the nature and limits of knowledge is", + "continuations": [ + " aesthetics.", + " ethics.", + " ontology alone.", + " epistemology." + ], + "tags": [ + "philosophy" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.30574035644531, + "mean_logprob": -10.461148071289063, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.4008674621582, + "mean_logprob": -12.4669558207194, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -67.76773071289062, + "mean_logprob": -13.553546142578124, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -68.76994323730469, + "mean_logprob": -13.753988647460938, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.0058077494303372, + "context_truncated": false, + "elapsed_seconds": 0.001011875250696903 + }, + { + "id": "world_knowledge-045", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The element with atomic number 74 is", + "continuations": [ + " tungsten.", + " titanium.", + " tantalum.", + " tellurium." + ], + "tags": [ + "chemistry" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.73124694824219, + "mean_logprob": -8.346249389648438, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -53.89195251464844, + "mean_logprob": -13.47298812866211, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -48.82057189941406, + "mean_logprob": -9.764114379882812, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -43.28997802734375, + "mean_logprob": -10.822494506835938, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.4178649902343743, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "world_knowledge-046", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The ancient writing system used for the Mycenaean Greek language is", + "continuations": [ + " cuneiform.", + " Linear B.", + " ogham.", + " hieratic." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -45.34986877441406, + "mean_logprob": -11.337467193603516, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -61.55717086791992, + "mean_logprob": -12.311434173583985, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -55.38843536376953, + "mean_logprob": -13.847108840942383, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -46.41596221923828, + "mean_logprob": -11.60399055480957, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.2665233612060547, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "world_knowledge-047", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A star's spectral class in the sequence OBAFGKM is primarily related to its", + "continuations": [ + " distance from Earth.", + " number of planets.", + " surface temperature.", + " age alone." + ], + "tags": [ + "astronomy" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -54.496826171875, + "mean_logprob": -13.62420654296875, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -46.10924530029297, + "mean_logprob": -11.527311325073242, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -42.18549346923828, + "mean_logprob": -14.06183115641276, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -47.40372848510742, + "mean_logprob": -15.80124282836914, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 2.096895217895508, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "world_knowledge-048", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The grammatical case typically marking direct address is the", + "continuations": [ + " accusative.", + " genitive.", + " instrumental.", + " vocative." + ], + "tags": [ + "linguistics" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.627098083496094, + "mean_logprob": -10.406774520874023, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -43.41761016845703, + "mean_logprob": -14.47253672281901, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -35.79242706298828, + "mean_logprob": -11.930809020996094, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -34.928871154785156, + "mean_logprob": -11.642957051595053, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.2361825307210292, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "world_knowledge-049", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The boundary around a black hole beyond which light cannot escape is the", + "continuations": [ + " event horizon.", + " heliopause.", + " Roche limit.", + " photosphere." + ], + "tags": [ + "physics" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -47.00669860839844, + "mean_logprob": -11.75167465209961, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.493675231933594, + "mean_logprob": -12.873418807983398, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -63.458858489990234, + "mean_logprob": -12.691771697998046, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -49.40321350097656, + "mean_logprob": -12.35080337524414, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.5991287231445312, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "world_knowledge-050", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In rhetoric, an argument that attacks the person rather than the claim is called", + "continuations": [ + " reductio ad absurdum.", + " ad hominem.", + " modus ponens.", + " false equivalence only." + ], + "tags": [ + "rhetoric" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -101.65061950683594, + "mean_logprob": -11.294513278537327, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -56.579071044921875, + "mean_logprob": -11.315814208984374, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -80.2905502319336, + "mean_logprob": -13.381758371988932, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -57.957061767578125, + "mean_logprob": -11.591412353515626, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.021300930447047506, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "context_tracking-001", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "On the shelf, the red box contains tea and the blue box contains rice. The tea is in", + "continuations": [ + " the red box.", + " the blue box.", + " both boxes.", + " neither box." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.66572570800781, + "mean_logprob": -8.416431427001953, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.501556396484375, + "mean_logprob": -8.875389099121094, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.74747848510742, + "mean_logprob": -8.436869621276855, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.401460647583008, + "mean_logprob": -8.80048688252767, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.020438194274902344, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "context_tracking-002", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Ivo chose the violin, while Mara chose the flute. Mara's instrument is", + "continuations": [ + " the violin.", + " unspecified.", + " the drum.", + " the flute." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.59260177612305, + "mean_logprob": -10.398150444030762, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.78028106689453, + "mean_logprob": -10.695070266723633, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -45.5455322265625, + "mean_logprob": -11.386383056640625, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -41.50012969970703, + "mean_logprob": -10.375032424926758, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.023118019104003906, + "context_truncated": false, + "elapsed_seconds": 0.0010075071259052493 + }, + { + "id": "context_tracking-003", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The morning bus stops at Oak Street. The evening bus stops at Pine Street. The evening stop is", + "continuations": [ + " Oak Street.", + " Maple Street.", + " Pine Street.", + " Cedar Street." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -42.45604705810547, + "mean_logprob": -10.614011764526367, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -61.46361541748047, + "mean_logprob": -12.292723083496094, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -43.82878494262695, + "mean_logprob": -10.957196235656738, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -48.86858367919922, + "mean_logprob": -9.773716735839844, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.8402950286865227, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-004", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Nia's badge number is 27, and Sol's badge number is 41. Badge 41 belongs to", + "continuations": [ + " Nia.", + " both people.", + " nobody.", + " Sol." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.35945701599121, + "mean_logprob": -9.119819005330404, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.75214385986328, + "mean_logprob": -9.250714619954428, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.874094009399414, + "mean_logprob": -10.624698003133139, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.75359344482422, + "mean_logprob": -8.87679672241211, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.24302228291829486, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-005", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The north garden grows mint. The south garden grows basil. Mint grows in", + "continuations": [ + " the north garden.", + " the south garden.", + " both gardens.", + " the greenhouse." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.492156982421875, + "mean_logprob": -8.623039245605469, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.31883239746094, + "mean_logprob": -8.829708099365234, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.334693908691406, + "mean_logprob": -8.583673477172852, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.2205810546875, + "mean_logprob": -10.4068603515625, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.03936576843261719, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-006", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Room A is painted yellow; Room B is painted green. The green room is", + "continuations": [ + " Room A.", + " Room B.", + " both rooms.", + " Room C." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -52.4699821472168, + "mean_logprob": -13.1174955368042, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -59.23291015625, + "mean_logprob": -14.8082275390625, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -53.48456573486328, + "mean_logprob": -13.37114143371582, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -57.508758544921875, + "mean_logprob": -14.377189636230469, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.2536458969116211, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-007", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "At noon, the cafe serves soup. At night, it serves noodles. The night menu features", + "continuations": [ + " soup.", + " salad.", + " noodles.", + " bread only." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -36.391807556152344, + "mean_logprob": -12.130602518717447, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -40.30162811279297, + "mean_logprob": -13.433876037597656, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -50.876426696777344, + "mean_logprob": -12.719106674194336, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.615158081054688, + "mean_logprob": -10.538386027018229, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 1.5922164916992188, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-008", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Paz owns the bicycle, and Ren owns the scooter. The scooter belongs to", + "continuations": [ + " Paz.", + " both Paz and Ren.", + " neither person.", + " Ren." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.189748764038086, + "mean_logprob": -9.729916254679361, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -65.02399444580078, + "mean_logprob": -9.289142063685826, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.840024948120117, + "mean_logprob": -9.280008316040039, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.56781768798828, + "mean_logprob": -10.855939229329428, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.009133747645787338, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-009", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The silver key opens the attic. The copper key opens the cellar. The attic key is", + "continuations": [ + " silver.", + " copper.", + " wooden.", + " black." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.387798309326172, + "mean_logprob": -8.693899154663086, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.194150924682617, + "mean_logprob": -8.097075462341309, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -37.477439880371094, + "mean_logprob": -12.492479960123697, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -16.31523895263672, + "mean_logprob": -8.15761947631836, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.06054401397705078, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-010", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Toma ordered pears, while Eda ordered plums. Eda ordered", + "continuations": [ + " pears.", + " plums.", + " peaches.", + " both fruits." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -35.15100860595703, + "mean_logprob": -11.717002868652344, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.55870819091797, + "mean_logprob": -10.519569396972656, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -37.9765739440918, + "mean_logprob": -12.658857981363932, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -35.00164794921875, + "mean_logprob": -11.667215983072916, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.1476465861002598, + "context_truncated": false, + "elapsed_seconds": 0.000978529376880033 + }, + { + "id": "context_tracking-011", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The first train goes to Linz. The second goes to Graz. The second destination is", + "continuations": [ + " Linz.", + " Vienna.", + " Graz.", + " Salzburg." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -31.083538055419922, + "mean_logprob": -7.7708845138549805, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -37.27201461791992, + "mean_logprob": -9.31800365447998, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.673728942871094, + "mean_logprob": -8.557909647623697, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -44.74026107788086, + "mean_logprob": -11.185065269470215, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.7870251337687169, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-012", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "File K is dated March, and File M is dated June. The June file is", + "continuations": [ + " File K.", + " both files.", + " undated.", + " File M." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.72386932373047, + "mean_logprob": -8.930967330932617, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.56492233276367, + "mean_logprob": -11.188307444254557, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -28.686620712280273, + "mean_logprob": -9.562206904093424, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.89557647705078, + "mean_logprob": -9.723894119262695, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.6312395731608067, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-013", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Luma wears a square pin. Oren wears a round pin. Luma's pin is", + "continuations": [ + " square.", + " round.", + " triangular.", + " missing." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.013163566589355, + "mean_logprob": -6.006581783294678, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.04383087158203, + "mean_logprob": -8.521915435791016, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -40.7132568359375, + "mean_logprob": -10.178314208984375, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -15.031862258911133, + "mean_logprob": -7.515931129455566, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.5093493461608887, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-014", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The code for apple is 6. The code for orange is 9. Code 9 means", + "continuations": [ + " apple.", + " orange.", + " pear.", + " fruit in general." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -32.11772155761719, + "mean_logprob": -10.705907185872396, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.369647979736328, + "mean_logprob": -9.456549326578775, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -42.94758605957031, + "mean_logprob": -14.315862019856771, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -43.80382537841797, + "mean_logprob": -10.950956344604492, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 1.249357859293621, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-015", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Kira arrived before Bo, and Bo arrived before Luis. The last to arrive was", + "continuations": [ + " Kira.", + " Bo.", + " Luis.", + " nobody." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.62009620666504, + "mean_logprob": -9.87336540222168, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -15.91169548034668, + "mean_logprob": -7.95584774017334, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.03851890563965, + "mean_logprob": -7.509629726409912, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.392553329467773, + "mean_logprob": -8.797517776489258, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.44621801376342773, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-016", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The small parcel weighs two kilos. The large parcel weighs five. The heavier parcel is", + "continuations": [ + " the small parcel.", + " both equally.", + " unknown.", + " the large parcel." + ], + "tags": [ + "comparison" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -45.05766296386719, + "mean_logprob": -9.011532592773438, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -20.91739845275879, + "mean_logprob": -6.972466150919597, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -23.174354553222656, + "mean_logprob": -11.587177276611328, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -39.66143798828125, + "mean_logprob": -7.93228759765625, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.9598214467366537, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-017", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Tuesday's password is comet. Wednesday's password is lantern. On Tuesday, the password is", + "continuations": [ + " comet.", + " lantern.", + " Tuesday.", + " Wednesday." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.926393508911133, + "mean_logprob": -8.975464502970377, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.721290588378906, + "mean_logprob": -10.930322647094727, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -48.00009536743164, + "mean_logprob": -12.00002384185791, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -57.146087646484375, + "mean_logprob": -11.429217529296874, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 1.9548581441243496, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-018", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Three couriers carry different folders: Aya has violet, Ben has white, and Cy has black. The white folder is carried by", + "continuations": [ + " Aya.", + " Ben.", + " Cy.", + " no courier." + ], + "tags": [ + "multi_binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.50574493408203, + "mean_logprob": -8.16858164469401, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.814693450927734, + "mean_logprob": -8.907346725463867, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.36609649658203, + "mean_logprob": -8.183048248291016, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -60.428794860839844, + "mean_logprob": -15.107198715209961, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.0144666035970058, + "context_truncated": false, + "elapsed_seconds": 0.0009741832509462256 + }, + { + "id": "context_tracking-019", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The gallery opens after the cafe but before the theater. The theater opens after the library. Among the gallery, cafe, and theater, the earliest is", + "continuations": [ + " the gallery.", + " the theater.", + " the cafe.", + " impossible to know." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.949676513671875, + "mean_logprob": -8.487419128417969, + "total_sequence_tokens": 45 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.07248306274414, + "mean_logprob": -9.018120765686035, + "total_sequence_tokens": 45 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -39.76935958862305, + "mean_logprob": -7.9538719177246096, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -48.30253219604492, + "mean_logprob": -12.07563304901123, + "total_sequence_tokens": 45 + } + ], + "best_vs_second_margin": 0.5335472106933592, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-020", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "A token moves from Box A to Box C, then from Box C to Box B. Its final location is", + "continuations": [ + " Box A.", + " Box C.", + " every box.", + " Box B." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.49848175048828, + "mean_logprob": -9.12462043762207, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.20964813232422, + "mean_logprob": -9.802412033081055, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -30.335063934326172, + "mean_logprob": -10.111687978108725, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.1957893371582, + "mean_logprob": -9.54894733428955, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.42432689666748047, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-021", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Rin lends her map to Dev. Dev then gives the map back to Rin. At the end, the map is with", + "continuations": [ + " Rin.", + " Dev.", + " both of them.", + " nobody." + ], + "tags": [ + "state_update" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.99566650390625, + "mean_logprob": -8.66522216796875, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.097444534301758, + "mean_logprob": -8.548722267150879, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.085906982421875, + "mean_logprob": -8.271476745605469, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.317691802978516, + "mean_logprob": -10.772563934326172, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.27724552154541016, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-022", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The amber team scored below the teal team but above the gray team. The highest-scoring of these teams was", + "continuations": [ + " amber.", + " teal.", + " gray.", + " tied." + ], + "tags": [ + "order" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -30.124832153320312, + "mean_logprob": -10.041610717773438, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -46.4231071472168, + "mean_logprob": -15.474369049072266, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -42.781898498535156, + "mean_logprob": -14.260632832845053, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.81403350830078, + "mean_logprob": -10.271344502766928, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.22973378499349018, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-023", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Ava's shift is Monday, Bram's is Thursday, and Cato's is Tuesday. The person working between Ava and Bram in the week is", + "continuations": [ + " Ava.", + " Bram.", + " Cato.", + " nobody." + ], + "tags": [ + "calendar" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.0821533203125, + "mean_logprob": -11.027384440104166, + "total_sequence_tokens": 42 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.313934326171875, + "mean_logprob": -9.771311442057291, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -24.020843505859375, + "mean_logprob": -8.006947835286459, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -45.141605377197266, + "mean_logprob": -15.047201792399088, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 1.7643636067708321, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-024", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The note began under the lamp. Jo moved it to the desk, and Uma moved it from the desk into a drawer. The note ended", + "continuations": [ + " under the lamp.", + " on the desk.", + " with Jo.", + " in the drawer." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.1395263671875, + "mean_logprob": -10.4279052734375, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -39.15525436401367, + "mean_logprob": -7.831050872802734, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -30.13128089904785, + "mean_logprob": -10.043760299682617, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -38.87596130371094, + "mean_logprob": -7.7751922607421875, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.055858612060546875, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-025", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The labels are reassigned: north now means left, south means right, east means up, and west means down. Under this rule, north means", + "continuations": [ + " left.", + " right.", + " up.", + " down." + ], + "tags": [ + "remapping" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.55514144897461, + "mean_logprob": -7.277570724487305, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.883102416992188, + "mean_logprob": -6.941551208496094, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.51163673400879, + "mean_logprob": -9.755818367004395, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -21.054794311523438, + "mean_logprob": -10.527397155761719, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.33601951599121094, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-026", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The bronze cup is taller than the glass cup. The glass cup is taller than the clay cup. The shortest is", + "continuations": [ + " bronze.", + " clay.", + " glass.", + " not specified." + ], + "tags": [ + "transitive" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -50.303409576416016, + "mean_logprob": -12.575852394104004, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.61474609375, + "mean_logprob": -11.87158203125, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.85732650756836, + "mean_logprob": -9.42866325378418, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -25.813201904296875, + "mean_logprob": -6.453300476074219, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 2.975362777709961, + "context_truncated": false, + "elapsed_seconds": 0.0010170338755415287 + }, + { + "id": "context_tracking-027", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Nell assigns fox to 14, owl to 22, and hare to 31. The animal assigned 31 is", + "continuations": [ + " fox.", + " owl.", + " hare.", + " none." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -41.660865783691406, + "mean_logprob": -13.886955261230469, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.042930603027344, + "mean_logprob": -10.010732650756836, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -43.25337219238281, + "mean_logprob": -14.417790730794271, + "total_sequence_tokens": 37 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -25.81450653076172, + "mean_logprob": -12.90725326538086, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 2.8965206146240234, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-028", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "A meeting moves from Room 3 to Room 8, then is moved back to its original room. It will occur in", + "continuations": [ + " Room 8.", + " both rooms.", + " no room.", + " Room 3." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -45.6904296875, + "mean_logprob": -9.1380859375, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -53.17036056518555, + "mean_logprob": -13.292590141296387, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -34.396888732910156, + "mean_logprob": -11.465629577636719, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -41.36860275268555, + "mean_logprob": -8.27372055053711, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.8643653869628896, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-029", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "In a made-up calendar, Dax follows Rul, and Vem follows Dax. The day immediately after Rul is", + "continuations": [ + " Dax.", + " Vem.", + " Rul.", + " unknown." + ], + "tags": [ + "local_rules" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.843585968017578, + "mean_logprob": -7.281195322672526, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -38.65679168701172, + "mean_logprob": -12.885597229003906, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.080869674682617, + "mean_logprob": -8.693623224894205, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -16.76009750366211, + "mean_logprob": -8.380048751831055, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.0988534291585283, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-030", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Faye is older than Gus but younger than Hana. The oldest of the three is", + "continuations": [ + " Faye.", + " Hana.", + " Gus.", + " tied." + ], + "tags": [ + "transitive" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.414276123046875, + "mean_logprob": -9.853569030761719, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -19.523046493530273, + "mean_logprob": -6.507682164510091, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -24.298892974853516, + "mean_logprob": -8.099630991617838, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -36.71498107910156, + "mean_logprob": -12.238327026367188, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.591948827107747, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-031", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The red route reaches Delta before Echo. The blue route reaches Echo before Delta. On the blue route, the first of those stops is", + "continuations": [ + " Delta.", + " neither stop.", + " Echo.", + " both together." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.27510070800781, + "mean_logprob": -10.818775177001953, + "total_sequence_tokens": 42 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -39.736812591552734, + "mean_logprob": -13.245604197184244, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.42176818847656, + "mean_logprob": -10.60544204711914, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.014312744140625, + "mean_logprob": -8.671437581380209, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 1.9340044657389317, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-032", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Iris gives the ring to Juno, Juno gives it to Kato, and Kato keeps it. The ring is finally with", + "continuations": [ + " Iris.", + " Juno.", + " everyone.", + " Kato." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.404891967773438, + "mean_logprob": -8.134963989257812, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -37.66615676879883, + "mean_logprob": -9.416539192199707, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.294584274291992, + "mean_logprob": -9.147292137145996, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -25.412918090820312, + "mean_logprob": -8.470972696940104, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.3360087076822911, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-033", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "For this report, green means urgent and red means routine. A green item is therefore", + "continuations": [ + " urgent.", + " routine.", + " canceled.", + " unclassified." + ], + "tags": [ + "remapping" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.43700408935547, + "mean_logprob": -8.145668029785156, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.155841827392578, + "mean_logprob": -9.577920913696289, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.413848876953125, + "mean_logprob": -11.603462219238281, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.71080780029297, + "mean_logprob": -9.177701950073242, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.032033920288086, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-034", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The oak is east of the pond. The bench is east of the oak. Relative to the pond, the bench is", + "continuations": [ + " west.", + " east.", + " north.", + " at the same place." + ], + "tags": [ + "spatial" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.386913299560547, + "mean_logprob": -6.193456649780273, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.7109375, + "mean_logprob": -5.85546875, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.587013244628906, + "mean_logprob": -7.793506622314453, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -41.032413482666016, + "mean_logprob": -8.206482696533204, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.33798789978027344, + "context_truncated": false, + "elapsed_seconds": 0.001048722624545917 + }, + { + "id": "context_tracking-035", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Four files are ordered J, K, L, M. K is moved after M, and J is then moved between L and M. The resulting order is", + "continuations": [ + " J, L, M, K.", + " L, M, J, K.", + " L, J, M, K.", + " M, L, J, K." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -71.21925354003906, + "mean_logprob": -8.902406692504883, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -72.3400650024414, + "mean_logprob": -9.042508125305176, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -77.79480743408203, + "mean_logprob": -9.724350929260254, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -67.77804565429688, + "mean_logprob": -8.47225570678711, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.43015098571777344, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-036", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "In a codebook, tulip means north, north means copper, and copper means seven. Following the chain from tulip leads to", + "continuations": [ + " north only.", + " tulip.", + " copper only.", + " seven." + ], + "tags": [ + "multi_hop" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -34.97745895385742, + "mean_logprob": -11.65915298461914, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -60.29480743408203, + "mean_logprob": -15.073701858520508, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -41.69165802001953, + "mean_logprob": -13.89721934000651, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -12.678321838378906, + "mean_logprob": -6.339160919189453, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 5.3199920654296875, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-037", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A cube starts in Bin R. It moves to S unless the flag is blue. The flag is blue, so it moves to T instead. Its final bin is", + "continuations": [ + " Bin T.", + " Bin S.", + " Bin R.", + " unknown." + ], + "tags": [ + "conditional_update" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -30.072601318359375, + "mean_logprob": -7.518150329589844, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.83147430419922, + "mean_logprob": -8.457868576049805, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -28.865478515625, + "mean_logprob": -7.21636962890625, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.88202667236328, + "mean_logprob": -8.94101333618164, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.30178070068359375, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-038", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Mira stands left of Noor. Olin stands right of Noor but left of Peta. From left to right, the second person is", + "continuations": [ + " Mira.", + " Noor.", + " Olin.", + " Peta." + ], + "tags": [ + "spatial" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -28.112262725830078, + "mean_logprob": -9.37075424194336, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.47229766845703, + "mean_logprob": -10.490765889485678, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.273412704467773, + "mean_logprob": -9.757804234822592, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.30517578125, + "mean_logprob": -10.768391927083334, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.38704999287923236, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-039", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The archive uses rotating labels. On day one A means red; on day two A means blue; on day three A means gold. Today is day three, so A means", + "continuations": [ + " red.", + " blue.", + " gold.", + " every color." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.496295928955078, + "mean_logprob": -6.748147964477539, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -15.127614974975586, + "mean_logprob": -7.563807487487793, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.792293548583984, + "mean_logprob": -6.896146774291992, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.55938720703125, + "mean_logprob": -10.853129069010416, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.14799880981445312, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-040", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A coin begins with Lio, passes to Mina, returns to Lio, then passes to Niko. Niko gives it to the person who held it immediately before him. The coin ends with", + "continuations": [ + " Mina.", + " Niko.", + " nobody.", + " Lio." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.771827697753906, + "mean_logprob": -8.923942565917969, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -37.75263214111328, + "mean_logprob": -9.43815803527832, + "total_sequence_tokens": 52 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -36.36043930053711, + "mean_logprob": -12.12014643351237, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.673444747924805, + "mean_logprob": -9.557814915974935, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.5142154693603516, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-041", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "In a race, Uma finished before Vic. Wes finished after Xena but before Uma. The earliest among Uma, Vic, Wes, and Xena was", + "continuations": [ + " Xena.", + " Wes.", + " Uma.", + " Vic." + ], + "tags": [ + "transitive" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -22.73081398010254, + "mean_logprob": -7.576937993367513, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.194042205810547, + "mean_logprob": -7.398014068603516, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.197547912597656, + "mean_logprob": -8.732515970865885, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.024688720703125, + "mean_logprob": -7.674896240234375, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.1789239247639971, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-042", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "A ledger maps 4 to pine, pine to west, and west to 19. Looking up 4 repeatedly until reaching a number returns", + "continuations": [ + " west.", + " 19.", + " pine.", + " 4." + ], + "tags": [ + "multi_hop" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -22.10761260986328, + "mean_logprob": -11.05380630493164, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -28.98568344116211, + "mean_logprob": -7.246420860290527, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -35.674842834472656, + "mean_logprob": -11.891614278157553, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.508155822753906, + "mean_logprob": -7.836051940917969, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.5896310806274414, + "context_truncated": false, + "elapsed_seconds": 0.0010654958750819787 + }, + { + "id": "context_tracking-043", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "The black card outranks the white card. The striped card ranks below white but above gray. The second-highest card is", + "continuations": [ + " black.", + " striped.", + " white.", + " gray." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -16.806875228881836, + "mean_logprob": -8.403437614440918, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.227855682373047, + "mean_logprob": -9.07595189412435, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.38714599609375, + "mean_logprob": -7.693572998046875, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.21112060546875, + "mean_logprob": -11.07037353515625, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.709864616394043, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-044", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A marker is in Drawer A. Rule one moves it to B; rule two swaps the contents of B and C; rule three moves whatever is in C to D. The marker finishes in", + "continuations": [ + " Drawer A.", + " Drawer B.", + " Drawer C.", + " Drawer D." + ], + "tags": [ + "state_update" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -42.967079162597656, + "mean_logprob": -8.593415832519531, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.51902770996094, + "mean_logprob": -8.703805541992187, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.43254852294922, + "mean_logprob": -8.086509704589844, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -42.92791748046875, + "mean_logprob": -8.58558349609375, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.49907379150390696, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-045", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Q is north of R. S is east of Q. T is south of S and level with R. Relative to R, T is", + "continuations": [ + " east.", + " west.", + " north.", + " at the same position." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -28.223785400390625, + "mean_logprob": -14.111892700195312, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -29.06546401977539, + "mean_logprob": -14.532732009887695, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -25.037765502929688, + "mean_logprob": -12.518882751464844, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -44.264122009277344, + "mean_logprob": -8.852824401855468, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 3.6660583496093757, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-046", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "There are three switches. Flipping X toggles Y; flipping Y toggles Z. Initially all are off. X is flipped once, then Y is flipped once. At the end, Z is", + "continuations": [ + " unchanged and off.", + " on.", + " removed.", + " both on and off." + ], + "tags": [ + "state_update" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -60.54683303833008, + "mean_logprob": -10.09113883972168, + "total_sequence_tokens": 55 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.386526107788086, + "mean_logprob": -8.693263053894043, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.63401985168457, + "mean_logprob": -7.817009925842285, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -43.978797912597656, + "mean_logprob": -8.795759582519532, + "total_sequence_tokens": 54 + } + ], + "best_vs_second_margin": 0.8762531280517578, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-047", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A roster initially reads Arin, Bela, Ciro, Dena. Bela and Dena swap places; then the first and third names swap. The first name becomes", + "continuations": [ + " Arin.", + " Dena.", + " Ciro.", + " Bela." + ], + "tags": [ + "permutation" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.489601135253906, + "mean_logprob": -5.829867045084636, + "total_sequence_tokens": 44 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -24.22303009033203, + "mean_logprob": -8.07434336344401, + "total_sequence_tokens": 44 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.44243621826172, + "mean_logprob": -9.61060905456543, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.311939239501953, + "mean_logprob": -9.10397974650065, + "total_sequence_tokens": 44 + } + ], + "best_vs_second_margin": 2.244476318359374, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-048", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "On odd dates, Gate P leads to Hall 1; on even dates it leads to Hall 2. Gate Q does the opposite. The date is the 18th, and a visitor enters Gate Q, reaching", + "continuations": [ + " Hall 2.", + " both halls.", + " no hall.", + " Hall 1." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.756568908691406, + "mean_logprob": -8.689142227172852, + "total_sequence_tokens": 55 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -34.52032470703125, + "mean_logprob": -8.630081176757812, + "total_sequence_tokens": 55 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.90447998046875, + "mean_logprob": -11.726119995117188, + "total_sequence_tokens": 55 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -35.82444763183594, + "mean_logprob": -8.956111907958984, + "total_sequence_tokens": 55 + } + ], + "best_vs_second_margin": 0.05906105041503906, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-049", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Nora is assigned to Vega unless Ishan takes Vega; if he does, Nora takes Lyra. Ishan takes Vega. Nora is therefore assigned to", + "continuations": [ + " Lyra.", + " Vega.", + " both.", + " neither." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.30630874633789, + "mean_logprob": -11.102102915445963, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.40412139892578, + "mean_logprob": -11.13470713297526, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.77387237548828, + "mean_logprob": -9.38693618774414, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -15.310232162475586, + "mean_logprob": -7.655116081237793, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 1.7318201065063477, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "context_tracking-050", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The sequence begins amber, cobalt, jade, silver. Remove cobalt, place silver first, then insert rose immediately after amber. The final sequence is", + "continuations": [ + " silver, rose, amber, jade.", + " silver, amber, rose, jade.", + " amber, rose, silver, jade.", + " silver, amber, jade, rose." + ], + "tags": [ + "permutation" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -105.95479583740234, + "mean_logprob": -10.595479583740234, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -107.36271667480469, + "mean_logprob": -10.73627166748047, + "total_sequence_tokens": 47 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -107.36823272705078, + "mean_logprob": -10.736823272705077, + "total_sequence_tokens": 47 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -104.2591552734375, + "mean_logprob": -10.42591552734375, + "total_sequence_tokens": 47 + } + ], + "best_vs_second_margin": 0.1695640563964833, + "context_truncated": false, + "elapsed_seconds": 0.0010603390001051594 + }, + { + "id": "quantitative-001", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Seven plus five equals", + "continuations": [ + " twelve.", + " eleven.", + " thirteen.", + " fourteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -58.53070831298828, + "mean_logprob": -14.63267707824707, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.91692352294922, + "mean_logprob": -9.972307840983072, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.953800201416016, + "mean_logprob": -11.738450050354004, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -24.48255157470703, + "mean_logprob": -8.160850524902344, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.8114573160807286, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-002", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A box holds nine pencils. After three are removed, it holds", + "continuations": [ + " five pencils.", + " twelve pencils.", + " seven pencils.", + " six pencils." + ], + "tags": [ + "subtraction" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -61.87773132324219, + "mean_logprob": -12.375546264648438, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -93.45236206054688, + "mean_logprob": -13.350337437220983, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -62.72536087036133, + "mean_logprob": -12.545072174072265, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -64.21088409423828, + "mean_logprob": -12.842176818847657, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.16952590942382706, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-003", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Four groups of three objects contain", + "continuations": [ + " seven objects.", + " ten objects.", + " twelve objects.", + " sixteen objects." + ], + "tags": [ + "multiplication" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.874332427978516, + "mean_logprob": -8.291444142659506, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.986637115478516, + "mean_logprob": -8.995545705159506, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -53.33905792236328, + "mean_logprob": -10.667811584472656, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.0954647064209, + "mean_logprob": -7.273866176605225, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 1.0175779660542812, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-004", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Twenty divided equally among five people gives each person", + "continuations": [ + " two.", + " three.", + " five.", + " four." + ], + "tags": [ + "division" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -18.346099853515625, + "mean_logprob": -9.173049926757812, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.317350387573242, + "mean_logprob": -9.158675193786621, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.89371681213379, + "mean_logprob": -8.946858406066895, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.25568389892578, + "mean_logprob": -8.62784194946289, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.3190164566040039, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-005", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The number immediately after 199 is", + "continuations": [ + " 200.", + " 198.", + " 209.", + " 299." + ], + "tags": [ + "number_sense" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -37.09755325317383, + "mean_logprob": -7.4195106506347654, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -28.096216201782227, + "mean_logprob": -5.619243240356445, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -32.88772201538086, + "mean_logprob": -6.577544403076172, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -28.480411529541016, + "mean_logprob": -5.696082305908203, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.07683906555175835, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-006", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Half of eighteen is", + "continuations": [ + " eight.", + " nine.", + " ten.", + " thirty-six." + ], + "tags": [ + "fractions" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.127948760986328, + "mean_logprob": -9.563974380493164, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.04364776611328, + "mean_logprob": -9.52182388305664, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.08776092529297, + "mean_logprob": -9.543880462646484, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -55.3458251953125, + "mean_logprob": -9.22430419921875, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.2975196838378906, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-007", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A rectangle with length five and width two has area", + "continuations": [ + " seven.", + " twelve.", + " ten.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -23.453693389892578, + "mean_logprob": -11.726846694946289, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.629615783691406, + "mean_logprob": -12.907403945922852, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.453468322753906, + "mean_logprob": -9.726734161376953, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.665557861328125, + "mean_logprob": -9.555185953776041, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.17154820760091205, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-008", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Three quarters of an hour is", + "continuations": [ + " thirty minutes.", + " forty minutes.", + " sixty minutes.", + " forty-five minutes." + ], + "tags": [ + "time" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -30.33171844482422, + "mean_logprob": -7.582929611206055, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.50766372680664, + "mean_logprob": -10.12691593170166, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -57.1715202331543, + "mean_logprob": -11.43430404663086, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -75.24260711669922, + "mean_logprob": -10.748943873814174, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 2.5439863204956055, + "context_truncated": false, + "elapsed_seconds": 0.0009536318757454865 + }, + { + "id": "quantitative-009", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "A ten-euro item discounted by two euros costs", + "continuations": [ + " eight euros.", + " twelve euros.", + " five euros.", + " twenty euros." + ], + "tags": [ + "money" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -61.91088104248047, + "mean_logprob": -12.382176208496094, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -110.3941650390625, + "mean_logprob": -15.770595005580358, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -65.8458251953125, + "mean_logprob": -13.1691650390625, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -62.55614471435547, + "mean_logprob": -12.511228942871094, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.12905273437499964, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-010", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The perimeter of a square with side length three is", + "continuations": [ + " nine.", + " twelve.", + " six.", + " fifteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.103952407836914, + "mean_logprob": -7.551976203918457, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.84244918823242, + "mean_logprob": -10.960612297058105, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.863807678222656, + "mean_logprob": -6.931903839111328, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.389785766601562, + "mean_logprob": -6.1299285888671875, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.8019752502441406, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-011", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Thirty is five more than", + "continuations": [ + " twenty.", + " thirty-five.", + " twenty-five.", + " fifteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.409133911132812, + "mean_logprob": -8.704566955566406, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -67.17459869384766, + "mean_logprob": -11.19576644897461, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -53.131996154785156, + "mean_logprob": -10.62639923095703, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.147247314453125, + "mean_logprob": -10.715749104817709, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.9218322753906243, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-012", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "If one notebook costs four euros, two notebooks cost", + "continuations": [ + " four euros.", + " six euros.", + " twelve euros.", + " eight euros." + ], + "tags": [ + "money" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -49.572818756103516, + "mean_logprob": -9.914563751220703, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -48.9205322265625, + "mean_logprob": -9.7841064453125, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -84.6905517578125, + "mean_logprob": -12.098650251116071, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -51.088356018066406, + "mean_logprob": -10.217671203613282, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.13045730590820348, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-013", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "The difference between fifteen and nine is", + "continuations": [ + " six.", + " five.", + " seven.", + " twenty-four." + ], + "tags": [ + "subtraction" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.702835083007812, + "mean_logprob": -7.851417541503906, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.943904876708984, + "mean_logprob": -8.471952438354492, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.169294357299805, + "mean_logprob": -8.084647178649902, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -45.11499786376953, + "mean_logprob": -9.022999572753907, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.2332296371459961, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-014", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Counting by tens, the number after forty is", + "continuations": [ + " forty-one.", + " fifty.", + " sixty.", + " thirty." + ], + "tags": [ + "number_sense" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.94867706298828, + "mean_logprob": -10.589735412597657, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.18026351928711, + "mean_logprob": -9.393421173095703, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.829933166503906, + "mean_logprob": -10.707483291625977, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -34.63414764404297, + "mean_logprob": -11.544715881347656, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.1963142395019535, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-015", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Two meters equal", + "continuations": [ + " twenty centimeters.", + " two hundred millimeters.", + " two hundred centimeters.", + " two thousand centimeters." + ], + "tags": [ + "units" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -47.23136901855469, + "mean_logprob": -9.446273803710938, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -67.80763244628906, + "mean_logprob": -11.30127207438151, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -60.33980941772461, + "mean_logprob": -10.056634902954102, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -65.03059387207031, + "mean_logprob": -10.838432312011719, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.6103610992431641, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-016", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The average of four and eight is", + "continuations": [ + " four.", + " five.", + " eight.", + " six." + ], + "tags": [ + "average" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.09528923034668, + "mean_logprob": -6.04764461517334, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.555801391601562, + "mean_logprob": -5.777900695800781, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.629536628723145, + "mean_logprob": -6.314768314361572, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -12.92236328125, + "mean_logprob": -6.461181640625, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.2697439193725586, + "context_truncated": false, + "elapsed_seconds": 0.0009379023722431157 + }, + { + "id": "quantitative-017", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "A dozen eggs contains", + "continuations": [ + " twelve eggs.", + " ten eggs.", + " six eggs.", + " twenty eggs." + ], + "tags": [ + "number_sense" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -68.08395385742188, + "mean_logprob": -13.616790771484375, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -45.29124450683594, + "mean_logprob": -15.097081502278646, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -46.9666748046875, + "mean_logprob": -15.655558268229166, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -44.82463836669922, + "mean_logprob": -14.941546122233072, + "total_sequence_tokens": 8 + } + ], + "best_vs_second_margin": 1.3247553507486973, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-018", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A train travels 180 kilometers in three hours at a constant rate. Its speed is", + "continuations": [ + " forty kilometers per hour.", + " sixty kilometers per hour.", + " ninety kilometers per hour.", + " one hundred eighty kilometers per hour." + ], + "tags": [ + "rate" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -78.2773208618164, + "mean_logprob": -11.182474408830915, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -92.80613708496094, + "mean_logprob": -11.600767135620117, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -75.18689727783203, + "mean_logprob": -9.398362159729004, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -87.92876434326172, + "mean_logprob": -9.769862704806858, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.3715005450778541, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-019", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A jacket priced at 80 euros is reduced by 25 percent. The sale price is", + "continuations": [ + " fifty euros.", + " fifty-five euros.", + " sixty euros.", + " sixty-five euros." + ], + "tags": [ + "percent" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -77.30064392089844, + "mean_logprob": -12.883440653483072, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -109.08311462402344, + "mean_logprob": -12.120346069335938, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -83.2864990234375, + "mean_logprob": -11.8980712890625, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -108.69227600097656, + "mean_logprob": -10.869227600097656, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 1.0288436889648445, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-020", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A tank contains 36 liters. One third is drained, leaving", + "continuations": [ + " twelve liters.", + " eighteen liters.", + " thirty liters.", + " twenty-four liters." + ], + "tags": [ + "fractions" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -80.74087524414062, + "mean_logprob": -13.456812540690104, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -49.326419830322266, + "mean_logprob": -9.865283966064453, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -64.85773468017578, + "mean_logprob": -12.971546936035157, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -97.47381591796875, + "mean_logprob": -13.924830845424108, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 3.106262969970704, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-021", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "Five consecutive integers begin with 12. Their sum is", + "continuations": [ + " seventy.", + " sixty.", + " seventy-five.", + " eighty." + ], + "tags": [ + "sequence" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -37.524757385253906, + "mean_logprob": -9.381189346313477, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -45.3996696472168, + "mean_logprob": -11.3499174118042, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.4299087524414, + "mean_logprob": -9.489986964634486, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.484935760498047, + "mean_logprob": -9.494978586832682, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.10879761832100954, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-022", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A right triangle has legs of length 6 and 8. Its hypotenuse has length", + "continuations": [ + " nine.", + " ten.", + " twelve.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.587217330932617, + "mean_logprob": -7.793608665466309, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.04330825805664, + "mean_logprob": -8.02165412902832, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.23641586303711, + "mean_logprob": -11.559103965759277, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.611534118652344, + "mean_logprob": -6.537178039550781, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.2564306259155273, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-023", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "If x plus 7 equals 19, then x equals", + "continuations": [ + " ten.", + " eleven.", + " twelve.", + " twenty-six." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -18.59610939025879, + "mean_logprob": -9.298054695129395, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.25155258178711, + "mean_logprob": -12.417184193929037, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -55.3399658203125, + "mean_logprob": -13.834991455078125, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.9217414855957, + "mean_logprob": -10.784348297119141, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.4862936019897468, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-024", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A worker earns 18 euros per hour for seven hours. Total earnings are", + "continuations": [ + " 108 euros.", + " 116 euros.", + " 136 euros.", + " 126 euros." + ], + "tags": [ + "multiplication" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -76.87730407714844, + "mean_logprob": -9.609663009643555, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -76.26702880859375, + "mean_logprob": -9.533378601074219, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -78.73773193359375, + "mean_logprob": -9.842216491699219, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -76.2882080078125, + "mean_logprob": -9.536026000976562, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.00264739990234375, + "context_truncated": false, + "elapsed_seconds": 0.000991786873782985 + }, + { + "id": "quantitative-025", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The ratio of red to blue marbles is 3 to 5. If there are 15 red marbles, there are", + "continuations": [ + " twenty-five blue marbles.", + " nine blue marbles.", + " eighteen blue marbles.", + " seventy-five blue marbles." + ], + "tags": [ + "ratio" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -89.21296691894531, + "mean_logprob": -11.151620864868164, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -55.45628356933594, + "mean_logprob": -11.091256713867187, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -66.2847900390625, + "mean_logprob": -11.047465006510416, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -112.06869506835938, + "mean_logprob": -11.206869506835938, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 0.04379170735677107, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-026", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A circle has radius 4. Using pi as 3.14, its circumference is approximately", + "continuations": [ + " 12.56.", + " 25.12.", + " 50.24.", + " 16.00." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -40.60220718383789, + "mean_logprob": -5.800315311976841, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -40.47172927856445, + "mean_logprob": -5.781675611223493, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -39.6677131652832, + "mean_logprob": -5.666816166469029, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -39.493404388427734, + "mean_logprob": -5.641914912632534, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.024901253836494774, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-027", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "An account balance rises from 240 to 300. The percentage increase is", + "continuations": [ + " fifteen percent.", + " twenty percent.", + " twenty-five percent.", + " sixty percent." + ], + "tags": [ + "percent" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -26.231616973876953, + "mean_logprob": -6.557904243469238, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.10055923461914, + "mean_logprob": -7.36685307820638, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -60.854217529296875, + "mean_logprob": -10.142369588216146, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.433773040771484, + "mean_logprob": -9.886754608154297, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.8089488347371416, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-028", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A bus leaves at 09:35 and arrives at 11:05. The journey lasts", + "continuations": [ + " seventy minutes.", + " eighty minutes.", + " one hundred minutes.", + " ninety minutes." + ], + "tags": [ + "time" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -55.372955322265625, + "mean_logprob": -11.074591064453125, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.903228759765625, + "mean_logprob": -12.975807189941406, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.89409637451172, + "mean_logprob": -9.47352409362793, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -60.38469314575195, + "mean_logprob": -12.07693862915039, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.6010669708251957, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-029", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The median of 3, 7, 8, 12, and 20 is", + "continuations": [ + " eight.", + " seven.", + " ten.", + " twelve." + ], + "tags": [ + "statistics" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.828088760375977, + "mean_logprob": -6.914044380187988, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.263343811035156, + "mean_logprob": -6.631671905517578, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.159292221069336, + "mean_logprob": -6.579646110534668, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.65314483642578, + "mean_logprob": -10.663286209106445, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.052025794982910156, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-030", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A 48-page booklet is printed with four pages per sheet. It requires", + "continuations": [ + " ten sheets.", + " twelve sheets.", + " sixteen sheets.", + " forty-four sheets." + ], + "tags": [ + "division" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.60862350463867, + "mean_logprob": -10.152155876159668, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -83.53076171875, + "mean_logprob": -13.921793619791666, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -47.74553298950195, + "mean_logprob": -9.54910659790039, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -92.6368179321289, + "mean_logprob": -11.579602241516113, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.6030492782592773, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-031", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Three fifths written as a decimal is", + "continuations": [ + " 0.3.", + " 0.5.", + " 0.6.", + " 0.8." + ], + "tags": [ + "fractions" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -34.27191162109375, + "mean_logprob": -6.85438232421875, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -36.058528900146484, + "mean_logprob": -7.2117057800292965, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -34.88420867919922, + "mean_logprob": -6.976841735839844, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.89845657348633, + "mean_logprob": -7.179691314697266, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.12245941162109375, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-032", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A shop buys an item for 30 euros and sells it for 42 euros. The profit is", + "continuations": [ + " 10 euros.", + " 11 euros.", + " 72 euros.", + " 12 euros." + ], + "tags": [ + "money" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -67.04220581054688, + "mean_logprob": -9.577457972935267, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -65.39778137207031, + "mean_logprob": -9.342540196010045, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -68.4997329711914, + "mean_logprob": -9.78567613874163, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -65.834716796875, + "mean_logprob": -9.404959542410714, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.062419346400668374, + "context_truncated": false, + "elapsed_seconds": 0.0010216222508461215 + }, + { + "id": "quantitative-033", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "If 2x minus 4 equals 16, then x equals", + "continuations": [ + " ten.", + " six.", + " eight.", + " twelve." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.601879119873047, + "mean_logprob": -9.800939559936523, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -22.20266342163086, + "mean_logprob": -11.10133171081543, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.828617095947266, + "mean_logprob": -10.914308547973633, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -59.95399856567383, + "mean_logprob": -14.988499641418457, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.1133689880371094, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-034", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A price increases by 20 percent and is then reduced by 20 percent. Starting from 100 euros, the final price is", + "continuations": [ + " 100 euros.", + " 96 euros.", + " 80 euros.", + " 104 euros." + ], + "tags": [ + "percent" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -83.61085510253906, + "mean_logprob": -10.451356887817383, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -75.07186889648438, + "mean_logprob": -10.724552699497767, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -77.82572174072266, + "mean_logprob": -11.117960248674665, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -82.75381469726562, + "mean_logprob": -10.344226837158203, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.10713005065917969, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-035", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A mixture contains milk and water in a 5 to 2 ratio. In 21 liters of mixture, the amount of water is", + "continuations": [ + " five liters.", + " seven liters.", + " six liters.", + " fifteen liters." + ], + "tags": [ + "ratio" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.307552337646484, + "mean_logprob": -9.826888084411621, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.34499740600586, + "mean_logprob": -10.086249351501465, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.07255554199219, + "mean_logprob": -9.518138885498047, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -52.323883056640625, + "mean_logprob": -10.464776611328125, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.3087491989135742, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-036", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "Two numbers have sum 46 and difference 8. The larger number is", + "continuations": [ + " nineteen.", + " twenty-three.", + " thirty-five.", + " twenty-seven." + ], + "tags": [ + "algebra" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -37.63522720336914, + "mean_logprob": -9.408806800842285, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -47.387569427490234, + "mean_logprob": -9.477513885498047, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -55.38041687011719, + "mean_logprob": -9.230069478352865, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -52.97425079345703, + "mean_logprob": -10.594850158691406, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.17873732248941998, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-037", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A car travels the first 120 kilometers at 60 km/h and the next 120 at 40 km/h. Its average speed for the full trip is", + "continuations": [ + " 48 km/h.", + " 50 km/h.", + " 45 km/h.", + " 52 km/h." + ], + "tags": [ + "rate" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -77.892822265625, + "mean_logprob": -11.127546037946429, + "total_sequence_tokens": 49 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -77.77400207519531, + "mean_logprob": -11.110571725027901, + "total_sequence_tokens": 49 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -76.33123779296875, + "mean_logprob": -10.904462541852679, + "total_sequence_tokens": 49 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -76.09575653076172, + "mean_logprob": -10.870822361537389, + "total_sequence_tokens": 49 + } + ], + "best_vs_second_margin": 0.033640180315289925, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-038", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A rectangle's length is three more than its width. If its perimeter is 30, its width is", + "continuations": [ + " five.", + " six.", + " seven.", + " nine." + ], + "tags": [ + "algebra" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.57919692993164, + "mean_logprob": -7.28959846496582, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -14.409192085266113, + "mean_logprob": -7.204596042633057, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.637920379638672, + "mean_logprob": -7.818960189819336, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.59064769744873, + "mean_logprob": -7.295323848724365, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.08500242233276367, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-039", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A bag has four red, three blue, and five green balls. The probability of drawing a blue ball is", + "continuations": [ + " one third.", + " three fifths.", + " one quarter.", + " three quarters." + ], + "tags": [ + "probability" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -22.60073471069336, + "mean_logprob": -7.533578236897786, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -30.65195083618164, + "mean_logprob": -6.130390167236328, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -19.383203506469727, + "mean_logprob": -6.461067835489909, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.025596618652344, + "mean_logprob": -9.506399154663086, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.330677668253581, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-040", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "An investment of 2,000 euros earns simple interest at 4 percent per year for three years. The interest earned is", + "continuations": [ + " 80 euros.", + " 160 euros.", + " 2,240 euros.", + " 240 euros." + ], + "tags": [ + "finance" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -59.82445526123047, + "mean_logprob": -8.546350751604352, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -67.59230041503906, + "mean_logprob": -8.449037551879883, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -82.61723327636719, + "mean_logprob": -8.26172332763672, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -68.16059112548828, + "mean_logprob": -8.520073890686035, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 0.18731422424316335, + "context_truncated": false, + "elapsed_seconds": 0.001069421625288669 + }, + { + "id": "quantitative-041", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "The sequence 2, 6, 12, 20, 30 continues with", + "continuations": [ + " 42.", + " 36.", + " 40.", + " 44." + ], + "tags": [ + "sequence" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.96985626220703, + "mean_logprob": -9.242464065551758, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.80557632446289, + "mean_logprob": -9.201394081115723, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.146583557128906, + "mean_logprob": -9.286645889282227, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.04112243652344, + "mean_logprob": -9.01028060913086, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.19111347198486328, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-042", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A machine completes three jobs in 18 minutes at a constant rate. At that rate, eight jobs take", + "continuations": [ + " 36 minutes.", + " 48 minutes.", + " 54 minutes.", + " 144 minutes." + ], + "tags": [ + "rate" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.8034553527832, + "mean_logprob": -10.36069107055664, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -53.098228454589844, + "mean_logprob": -10.619645690917968, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -51.5724983215332, + "mean_logprob": -10.314499664306641, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -59.01431655883789, + "mean_logprob": -9.835719426472982, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.478780237833659, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-043", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "The mean of six numbers is 14. Five of them sum to 61. The sixth number is", + "continuations": [ + " fourteen.", + " seventeen.", + " twenty-three.", + " eighty-four." + ], + "tags": [ + "statistics" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.33763313293457, + "mean_logprob": -3.7792110443115234, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.17460632324219, + "mean_logprob": -8.793651580810547, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -29.028841018676758, + "mean_logprob": -5.805768203735352, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -49.011024475097656, + "mean_logprob": -8.168504079182943, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 2.0265571594238283, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-044", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A map scale is 1 to 50,000. A distance of 6 centimeters on the map represents", + "continuations": [ + " 300 meters.", + " 30 kilometers.", + " 300 kilometers.", + " 3 kilometers." + ], + "tags": [ + "scale" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -49.293914794921875, + "mean_logprob": -8.215652465820312, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -52.14571762084961, + "mean_logprob": -8.690952936808268, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -58.73967742919922, + "mean_logprob": -8.391382489885602, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -45.33264923095703, + "mean_logprob": -9.066529846191406, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.17573002406528992, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-045", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "If three quarters of a number is 27, the number is", + "continuations": [ + " thirty-six.", + " twenty-one.", + " thirty.", + " forty." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -50.46824264526367, + "mean_logprob": -8.411373774210611, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -34.846473693847656, + "mean_logprob": -8.711618423461914, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -24.473007202148438, + "mean_logprob": -8.157669067382812, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.94614028930664, + "mean_logprob": -6.64871342976888, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.5089556376139326, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-046", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A 15-liter solution is 40 percent salt. Adding 5 liters of water makes the new salt concentration", + "continuations": [ + " twenty percent.", + " thirty percent.", + " thirty-five percent.", + " forty-five percent." + ], + "tags": [ + "mixture" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -18.041231155395508, + "mean_logprob": -6.013743718465169, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -26.89980125427246, + "mean_logprob": -6.724950313568115, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.95402526855469, + "mean_logprob": -9.56486075265067, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -68.28321075439453, + "mean_logprob": -9.754744393484932, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.7112065951029463, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-047", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A square's diagonal has length 10. Its area is", + "continuations": [ + " twenty-five.", + " forty.", + " fifty.", + " one hundred." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.90211486816406, + "mean_logprob": -8.380422973632813, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.2315731048584, + "mean_logprob": -8.743857701619467, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -39.572296142578125, + "mean_logprob": -13.190765380859375, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -22.539701461791992, + "mean_logprob": -7.513233820597331, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.8671891530354818, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-048", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A clock gains two minutes every three hours. After 24 real hours, it will be ahead by", + "continuations": [ + " eight minutes.", + " twelve minutes.", + " eighteen minutes.", + " sixteen minutes." + ], + "tags": [ + "rate" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.210365295410156, + "mean_logprob": -9.070121765136719, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -76.61856079101562, + "mean_logprob": -15.323712158203126, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.920597076416016, + "mean_logprob": -8.480149269104004, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.31632614135742, + "mean_logprob": -8.079081535339355, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.40106773376464844, + "context_truncated": false, + "elapsed_seconds": 0.0010098259990627412 + }, + { + "id": "quantitative-049", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A number is increased by 8, doubled, and then reduced by 6 to give 30. The original number is", + "continuations": [ + " ten.", + " eight.", + " twelve.", + " fourteen." + ], + "tags": [ + "algebra" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -22.240764617919922, + "mean_logprob": -11.120382308959961, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.731048583984375, + "mean_logprob": -9.365524291992188, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -43.75515365600586, + "mean_logprob": -10.938788414001465, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -29.068981170654297, + "mean_logprob": -9.6896603902181, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.32413609822591205, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "quantitative-050", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A store marks an item up 25 percent from a cost of 64 euros. The marked price is", + "continuations": [ + " 72 euros.", + " 80 euros.", + " 84 euros.", + " 89 euros." + ], + "tags": [ + "percent" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -82.27996826171875, + "mean_logprob": -11.754281180245536, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -82.95674133300781, + "mean_logprob": -11.850963047572545, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -80.60213470458984, + "mean_logprob": -11.514590672084264, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -82.8143310546875, + "mean_logprob": -11.830618722098214, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.23969050816127258, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-001", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "All robins are birds. Pip is a robin. Therefore, Pip is", + "continuations": [ + " a bird.", + " not a bird.", + " a fish.", + " necessarily blue." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.23310089111328, + "mean_logprob": -7.744366963704427, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -38.55419921875, + "mean_logprob": -9.6385498046875, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -20.171154022216797, + "mean_logprob": -6.723718007405599, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -41.48497009277344, + "mean_logprob": -13.828323364257812, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.0206489562988281, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-002", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No squares are circles. Shape K is a square. Shape K is therefore", + "continuations": [ + " a circle.", + " not a square.", + " both shapes.", + " not a circle." + ], + "tags": [ + "deduction" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.8961181640625, + "mean_logprob": -6.632039388020833, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -27.221660614013672, + "mean_logprob": -6.805415153503418, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -32.98468780517578, + "mean_logprob": -10.994895935058594, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.93111228942871, + "mean_logprob": -7.482778072357178, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.17337576548258493, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-003", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "If the switch is on, the lamp glows. The switch is on, so the lamp", + "continuations": [ + " must be broken.", + " is off.", + " glows.", + " has no switch." + ], + "tags": [ + "modus_ponens" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -31.12826919555664, + "mean_logprob": -7.78206729888916, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.593992233276367, + "mean_logprob": -8.531330744425455, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.783809661865234, + "mean_logprob": -8.445952415466309, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -43.53190994262695, + "mean_logprob": -10.882977485656738, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.6638851165771484, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-004", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "A is taller than B, and B is taller than C. It follows that", + "continuations": [ + " C is taller than A.", + " A and C are equal.", + " B is shortest.", + " A is taller than C." + ], + "tags": [ + "transitive" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -55.297306060791016, + "mean_logprob": -7.899615151541574, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -47.28986358642578, + "mean_logprob": -7.881643931070964, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.08476638793945, + "mean_logprob": -9.81695327758789, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -56.58064270019531, + "mean_logprob": -8.082948957170759, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.01797122047060995, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-005", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Either the coin is in the cup or in the box. It is not in the cup. Thus it is", + "continuations": [ + " in the box.", + " in both places.", + " nowhere.", + " still in the cup." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -28.708221435546875, + "mean_logprob": -7.177055358886719, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -38.501197814941406, + "mean_logprob": -9.625299453735352, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.83718490600586, + "mean_logprob": -9.945728302001953, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -41.031166076660156, + "mean_logprob": -8.20623321533203, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 1.0291778564453118, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-006", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Every metal key conducts electricity. This object is a metal key, so it", + "continuations": [ + " cannot be metal.", + " conducts electricity.", + " must be wooden.", + " proves all objects conduct." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.162315368652344, + "mean_logprob": -8.790578842163086, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.14353561401367, + "mean_logprob": -8.785883903503418, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -51.482635498046875, + "mean_logprob": -10.296527099609374, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -62.58480453491211, + "mean_logprob": -10.430800755818685, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.004694938659667969, + "context_truncated": false, + "elapsed_seconds": 0.0009597013740858529 + }, + { + "id": "logical_reasoning-007", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If today is Monday, tomorrow is Tuesday. Today is Monday. Tomorrow is", + "continuations": [ + " Sunday.", + " Wednesday.", + " Tuesday.", + " Monday." + ], + "tags": [ + "conditional" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -18.397624969482422, + "mean_logprob": -6.132541656494141, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -50.41315460205078, + "mean_logprob": -10.082630920410157, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -23.818340301513672, + "mean_logprob": -5.954585075378418, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -33.721649169921875, + "mean_logprob": -8.430412292480469, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.17795658111572266, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-008", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No red token is large. Token P is red. Token P is", + "continuations": [ + " definitely large.", + " both red and large.", + " not red.", + " not large." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -42.21651840209961, + "mean_logprob": -10.554129600524902, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -52.73724365234375, + "mean_logprob": -10.54744873046875, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.603893280029297, + "mean_logprob": -9.201297760009766, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.03830909729004, + "mean_logprob": -10.346103032430014, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.144805272420248, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-009", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "Only one of Door A and Door B is open. Door A is closed, so", + "continuations": [ + " Door B is open.", + " both are closed.", + " both are open.", + " Door B must be closed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -60.214691162109375, + "mean_logprob": -10.035781860351562, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.3933219909668, + "mean_logprob": -9.0983304977417, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.948970794677734, + "mean_logprob": -8.737242698669434, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -69.2431869506836, + "mean_logprob": -9.891883850097656, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.3610877990722656, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-010", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "All cedar trees are plants. No plant is a machine. Therefore, no cedar tree is", + "continuations": [ + " a plant.", + " a machine.", + " a tree.", + " alive." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.66943359375, + "mean_logprob": -7.22314453125, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.1416015625, + "mean_logprob": -8.7138671875, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.497848510742188, + "mean_logprob": -7.4992828369140625, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -16.748842239379883, + "mean_logprob": -8.374421119689941, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.2761383056640625, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-011", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "If it rains, the path gets wet. The path is not wet. Therefore, it", + "continuations": [ + " certainly snowed.", + " must be night.", + " did not rain.", + " is still raining." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.763160705566406, + "mean_logprob": -10.190790176391602, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.64503479003906, + "mean_logprob": -10.911258697509766, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -50.561973571777344, + "mean_logprob": -12.640493392944336, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -51.65583419799805, + "mean_logprob": -10.331166839599609, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.14037666320800746, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-012", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "L is before M, and M is before N. In this ordering, N is", + "continuations": [ + " first.", + " before L.", + " between L and M.", + " after L." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -25.664012908935547, + "mean_logprob": -12.832006454467773, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.63257598876953, + "mean_logprob": -10.54419199625651, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -43.05575180053711, + "mean_logprob": -8.611150360107422, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -29.036195755004883, + "mean_logprob": -9.678731918334961, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.0675815582275394, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-013", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Some poets are teachers. Every teacher reads. Therefore, some poets", + "continuations": [ + " read.", + " never read.", + " are not teachers.", + " are all teachers." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.998289108276367, + "mean_logprob": -7.999144554138184, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.36034393310547, + "mean_logprob": -8.786781311035156, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.150787353515625, + "mean_logprob": -8.287696838378906, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.68013000488281, + "mean_logprob": -8.670032501220703, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.28855228424072266, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-014", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "Exactly one card is marked. Card X is not marked and Card Y is marked. Card Z is therefore", + "continuations": [ + " also marked.", + " not marked.", + " the same as Y.", + " impossible to classify." + ], + "tags": [ + "constraint" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.754146575927734, + "mean_logprob": -8.584715525309244, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.3232421875, + "mean_logprob": -8.7744140625, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -33.07429504394531, + "mean_logprob": -6.614859008789063, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -39.34959030151367, + "mean_logprob": -7.8699180603027346, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.2550590515136717, + "context_truncated": false, + "elapsed_seconds": 0.000974042501184158 + }, + { + "id": "logical_reasoning-015", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If a number is divisible by four, it is even. Twenty is divisible by four, so twenty is", + "continuations": [ + " odd.", + " prime.", + " even.", + " negative." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.385637283325195, + "mean_logprob": -8.795212427775065, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.24374008178711, + "mean_logprob": -9.41458002726237, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.601098537445068, + "mean_logprob": -3.800549268722534, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.169453620910645, + "mean_logprob": -5.584726810455322, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.784177541732788, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-016", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "No blue item is fragile. Vase V is fragile. Therefore, Vase V is", + "continuations": [ + " definitely blue.", + " both blue and fragile.", + " not fragile.", + " not blue." + ], + "tags": [ + "contraposition" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.48705291748047, + "mean_logprob": -9.871763229370117, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -64.77378845214844, + "mean_logprob": -10.795631408691406, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.39675521850586, + "mean_logprob": -10.599188804626465, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.218502044677734, + "mean_logprob": -8.739500681559244, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.132262547810873, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-017", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "All glims are nords. Some nords are pelts. From these facts alone, it follows that", + "continuations": [ + " no conclusion links glims to pelts.", + " all glims are pelts.", + " some glims are pelts.", + " no nords are glims." + ], + "tags": [ + "quantifiers" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -145.32606506347656, + "mean_logprob": -13.21146046031605, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -103.00093078613281, + "mean_logprob": -11.444547865125868, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -114.34152221679688, + "mean_logprob": -12.704613579644096, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -89.08880615234375, + "mean_logprob": -11.136100769042969, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.3084470960828991, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-018", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "If R then S. If S then T. R is true. Therefore,", + "continuations": [ + " T is false.", + " T is true.", + " S is false.", + " R is false." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.808006286621094, + "mean_logprob": -9.202001571655273, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -29.35900115966797, + "mean_logprob": -7.339750289916992, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.576637268066406, + "mean_logprob": -10.394159317016602, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.96452331542969, + "mean_logprob": -10.741130828857422, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.8622512817382812, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-019", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Exactly two of A, B, and C are true. A is true and B is false. Therefore, C is", + "continuations": [ + " false.", + " unknown.", + " true.", + " both true and false." + ], + "tags": [ + "constraint" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.426522254943848, + "mean_logprob": -7.713261127471924, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -12.785540580749512, + "mean_logprob": -6.392770290374756, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.469666481018066, + "mean_logprob": -5.234833240509033, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -38.55765151977539, + "mean_logprob": -7.711530303955078, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.1579370498657227, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-020", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Kai sits somewhere left of Lio. Mina sits right of Lio. Which statement must be true?", + "continuations": [ + " Mina sits left of Kai.", + " Kai sits beside Mina.", + " Lio sits left of Kai.", + " Kai sits left of Mina." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -94.76506042480469, + "mean_logprob": -10.529451158311632, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -110.12782287597656, + "mean_logprob": -11.012782287597656, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -94.45169830322266, + "mean_logprob": -10.494633144802517, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -93.47952270507812, + "mean_logprob": -10.38661363389757, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.10801951090494732, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-021", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Every zor is a vek, and no vek is a lum. Therefore,", + "continuations": [ + " no zor is a lum.", + " every lum is a zor.", + " some zors are lums.", + " no zor is a vek." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -72.24020385742188, + "mean_logprob": -9.030025482177734, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -86.24103546142578, + "mean_logprob": -10.780129432678223, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -86.23545837402344, + "mean_logprob": -12.319351196289062, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -79.2609634399414, + "mean_logprob": -9.907620429992676, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.8775949478149414, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-022", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alarm rings if and only if the sensor is active. The alarm is silent. Thus the sensor is", + "continuations": [ + " active.", + " inactive.", + " both states.", + " unrelated to the alarm." + ], + "tags": [ + "biconditional" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.83162784576416, + "mean_logprob": -7.91581392288208, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -24.528423309326172, + "mean_logprob": -8.176141103108725, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -43.07128143310547, + "mean_logprob": -14.357093811035156, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -86.49199676513672, + "mean_logprob": -12.355999537876674, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.2603271802266445, + "context_truncated": false, + "elapsed_seconds": 0.0009802713721001055 + }, + { + "id": "logical_reasoning-023", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "At least one of P or Q is false. P is true. It follows that Q is", + "continuations": [ + " true.", + " undetermined.", + " false.", + " equal to P." + ], + "tags": [ + "propositional" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.790517807006836, + "mean_logprob": -6.395258903503418, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -56.42156219482422, + "mean_logprob": -11.284312438964843, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.321063995361328, + "mean_logprob": -9.660531997680664, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -26.10745620727539, + "mean_logprob": -6.526864051818848, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.1316051483154297, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-024", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Three runners finish with no ties. Ada is not first. Bo finishes after Cy. The only possible first-place runner is", + "continuations": [ + " Ada.", + " Bo.", + " either Ada or Bo.", + " Cy." + ], + "tags": [ + "constraint" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.992191314697266, + "mean_logprob": -11.330730438232422, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -21.700164794921875, + "mean_logprob": -10.850082397460938, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -64.4281005859375, + "mean_logprob": -10.738016764322916, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -30.689435958862305, + "mean_logprob": -15.344717979431152, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.11206563313802143, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-025", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "If the report is approved, funding continues. Funding did not continue. Therefore, the report was", + "continuations": [ + " not approved.", + " approved late.", + " necessarily reviewed.", + " unrelated to funding." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.918304443359375, + "mean_logprob": -7.972768147786458, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.20856285095215, + "mean_logprob": -9.736187616984049, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.74882507324219, + "mean_logprob": -10.437206268310547, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -56.24339294433594, + "mean_logprob": -11.248678588867188, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.7634194691975908, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-026", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Some artists are engineers, and no engineer is careless. Therefore, some artists are", + "continuations": [ + " careless.", + " not careless.", + " not engineers.", + " all engineers." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.968116760253906, + "mean_logprob": -7.989372253417969, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.22990417480469, + "mean_logprob": -10.557476043701172, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -30.666587829589844, + "mean_logprob": -10.222195943196615, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.083206176757812, + "mean_logprob": -9.027735392252604, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.0383631388346348, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-027", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "A statement and its negation cannot both be true. The statement is true, so its negation is", + "continuations": [ + " true.", + " unknown.", + " false.", + " equivalent." + ], + "tags": [ + "negation" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.13509750366211, + "mean_logprob": -6.567548751831055, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.628036499023438, + "mean_logprob": -10.314018249511719, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.505672454833984, + "mean_logprob": -5.252836227416992, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -18.212894439697266, + "mean_logprob": -9.106447219848633, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.3147125244140625, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-028", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "If X is selected, Y must be selected. If Y is selected, Z cannot be selected. X is selected, so Z", + "continuations": [ + " must be selected.", + " may equal X.", + " makes Y unselected.", + " cannot be selected." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -29.461515426635742, + "mean_logprob": -7.3653788566589355, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.742698669433594, + "mean_logprob": -8.935674667358398, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -87.30157470703125, + "mean_logprob": -12.471653529575892, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.7110481262207, + "mean_logprob": -9.677762031555176, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.570295810699463, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-029", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "No historians are timeless beings. Some writers are historians. Therefore,", + "continuations": [ + " some writers are not timeless beings.", + " all writers are timeless.", + " no writers are historians.", + " all historians are writers." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -106.55276489257812, + "mean_logprob": -13.319095611572266, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -73.56819152832031, + "mean_logprob": -12.261365254720053, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -65.05790710449219, + "mean_logprob": -10.842984517415365, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -65.29291534423828, + "mean_logprob": -10.882152557373047, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.0391680399576817, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-030", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Either J or K, but not both, is open. If J is open, L is closed. J is open. Hence", + "continuations": [ + " K and L are open.", + " K and L are closed.", + " K is open and L closed.", + " K is closed and L open." + ], + "tags": [ + "propositional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -50.52782440185547, + "mean_logprob": -8.421304066975912, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -45.10450744628906, + "mean_logprob": -7.517417907714844, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -64.72964477539062, + "mean_logprob": -9.247092110770089, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -56.424224853515625, + "mean_logprob": -8.060603550502233, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.5431856427873889, + "context_truncated": false, + "elapsed_seconds": 0.0009956388748832978 + }, + { + "id": "logical_reasoning-031", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Rhea is before Sami. Taro is after Sami. Ula is before Rhea. The second person in order is", + "continuations": [ + " Ula.", + " Sami.", + " Rhea.", + " Taro." + ], + "tags": [ + "order" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.909732818603516, + "mean_logprob": -9.969910939534506, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.422407150268555, + "mean_logprob": -8.474135716756185, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -38.00476837158203, + "mean_logprob": -12.668256123860678, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.72105026245117, + "mean_logprob": -9.430262565612793, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.9561268488566075, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-032", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "All valid tickets have a stamp. This ticket has no stamp. It is therefore", + "continuations": [ + " valid.", + " stamped invisibly.", + " both valid and invalid.", + " not valid." + ], + "tags": [ + "contraposition" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.51222038269043, + "mean_logprob": -7.256110191345215, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -73.79500579833984, + "mean_logprob": -10.54214368547712, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -60.171443939208984, + "mean_logprob": -8.59592056274414, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -21.963024139404297, + "mean_logprob": -7.321008046468099, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.06489785512288382, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-033", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Exactly one of the claims \"A is true\" and \"B is false\" is true. A is false. Therefore, B is", + "continuations": [ + " false.", + " true.", + " both.", + " undetermined." + ], + "tags": [ + "propositional" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.391382217407227, + "mean_logprob": -6.195691108703613, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.81505012512207, + "mean_logprob": -5.407525062561035, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.489299774169922, + "mean_logprob": -7.744649887084961, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -60.607337951660156, + "mean_logprob": -12.121467590332031, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.7881660461425781, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-034", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Every quill is a renn. No renn is a soth. Some varks are quills. It follows that some varks are", + "continuations": [ + " soths.", + " not soths.", + " not quills.", + " all renns." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.65414810180664, + "mean_logprob": -10.91353702545166, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -55.57721710205078, + "mean_logprob": -11.115443420410156, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -40.150177001953125, + "mean_logprob": -10.037544250488281, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -54.663028717041016, + "mean_logprob": -10.932605743408203, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 0.8759927749633789, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-035", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Exactly two of P, Q, R, and S are true. P and Q have opposite truth values. R is true and S is false. Therefore,", + "continuations": [ + " P and Q are both true.", + " P and Q are both false.", + " exactly one of P and Q is true.", + " neither P nor Q matters." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -39.059993743896484, + "mean_logprob": -5.579999106270926, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -42.693763732910156, + "mean_logprob": -6.0991091047014505, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -61.75218200683594, + "mean_logprob": -6.8613535563151045, + "total_sequence_tokens": 43 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -50.020606994628906, + "mean_logprob": -8.33676783243815, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.5191099984305243, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-036", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "If A then B. If not C then not B. A is true. Which conclusion must follow?", + "continuations": [ + " C is false.", + " B is false.", + " A is false.", + " C is true." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -31.845439910888672, + "mean_logprob": -7.961359977722168, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -37.11009216308594, + "mean_logprob": -9.277523040771484, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -31.720535278320312, + "mean_logprob": -7.930133819580078, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.822303771972656, + "mean_logprob": -7.705575942993164, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.22455787658691406, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-037", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "Five seats are numbered 1 through 5. Iva sits immediately left of Jae. Kian sits in seat 5. Iva sits in seat 2. Jae sits in", + "continuations": [ + " seat 3.", + " seat 1.", + " seat 4.", + " seat 5." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -23.291847229003906, + "mean_logprob": -5.822961807250977, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -23.31136703491211, + "mean_logprob": -5.827841758728027, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -24.795713424682617, + "mean_logprob": -6.198928356170654, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -24.54903793334961, + "mean_logprob": -6.137259483337402, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.004879951477050781, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-038", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No fal is a mer. Every tov is a fal. Some riks are mers. Which statement must be true?", + "continuations": [ + " Some riks are tovs.", + " No tov is a mer.", + " Every rik is a fal.", + " Some mers are fals." + ], + "tags": [ + "quantifiers" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -80.48735809326172, + "mean_logprob": -8.94303978814019, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -70.23312377929688, + "mean_logprob": -10.033303397042411, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -103.08189392089844, + "mean_logprob": -12.885236740112305, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -81.7691421508789, + "mean_logprob": -11.68130602155413, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 1.0902636089022213, + "context_truncated": false, + "elapsed_seconds": 0.0010560832488408778 + }, + { + "id": "logical_reasoning-039", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A safe opens exactly when both keys are turned. The safe did not open, and the first key was turned. Therefore, the second key was", + "continuations": [ + " turned.", + " broken necessarily.", + " not turned.", + " unrelated." + ], + "tags": [ + "conjunction" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.305379867553711, + "mean_logprob": -5.6526899337768555, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -32.84999465942383, + "mean_logprob": -10.949998219807943, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.626249313354492, + "mean_logprob": -5.542083104451497, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.766883850097656, + "mean_logprob": -11.255627950032553, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.11060682932535837, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-040", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A committee selects exactly three of J, K, L, and M. If J is selected, K is not. L and M are selected. Therefore, the third member must be", + "continuations": [ + " J.", + " L again.", + " M again.", + " K." + ], + "tags": [ + "constraint" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -20.214515686035156, + "mean_logprob": -10.107257843017578, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -32.824039459228516, + "mean_logprob": -10.941346486409506, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.404020309448242, + "mean_logprob": -9.801340103149414, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.65143585205078, + "mean_logprob": -8.82571792602539, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.9756221771240234, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-041", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Some nels are paks. All paks are wims. No wim is a dor. Therefore,", + "continuations": [ + " some nels are not dors.", + " all nels are wims.", + " some dors are paks.", + " no nels are paks." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -72.26261901855469, + "mean_logprob": -9.032827377319336, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -69.47428894042969, + "mean_logprob": -8.684286117553711, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -50.84495162963867, + "mean_logprob": -7.26356451851981, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -67.08775329589844, + "mean_logprob": -9.58396475655692, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 1.420721599033901, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-042", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Four tasks W, X, Y, Z are scheduled once each. W precedes X. Y immediately precedes Z. X is first. These conditions are", + "continuations": [ + " satisfied by X, W, Y, Z.", + " impossible to satisfy.", + " satisfied by W, X, Y, Z.", + " satisfied by Y, Z, W, X." + ], + "tags": [ + "consistency" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -108.66563415527344, + "mean_logprob": -9.878694014115768, + "total_sequence_tokens": 46 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -56.0281867980957, + "mean_logprob": -11.205637359619141, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -104.86241149902344, + "mean_logprob": -9.53294649991122, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -112.93009948730469, + "mean_logprob": -10.266372680664062, + "total_sequence_tokens": 46 + } + ], + "best_vs_second_margin": 0.34574751420454675, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-043", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If either M or N is true, P is true. P is false. Therefore,", + "continuations": [ + " M is true.", + " N is true.", + " both M and N are false.", + " exactly one is false." + ], + "tags": [ + "contraposition" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -27.520423889160156, + "mean_logprob": -6.880105972290039, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -30.140037536621094, + "mean_logprob": -7.535009384155273, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -53.795677185058594, + "mean_logprob": -7.685096740722656, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -42.091068267822266, + "mean_logprob": -8.418213653564454, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.6549034118652344, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-044", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "All A-things are B-things. Some B-things are C-things. No C-thing is a D-thing. Which claim is guaranteed?", + "continuations": [ + " Some A-things are C-things.", + " No A-thing is a D-thing.", + " All B-things are A-things.", + " Some B-things are not D-things." + ], + "tags": [ + "quantifiers" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -97.88928985595703, + "mean_logprob": -8.899026350541549, + "total_sequence_tokens": 50 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -78.7789306640625, + "mean_logprob": -7.87789306640625, + "total_sequence_tokens": 49 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -105.96389770507812, + "mean_logprob": -9.633081609552557, + "total_sequence_tokens": 50 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -113.51749420166016, + "mean_logprob": -9.45979118347168, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 1.0211332841352991, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-045", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "A code is valid if exactly one of conditions X and Y holds. X holds and the code is valid. Therefore, Y", + "continuations": [ + " does not hold.", + " also holds.", + " is equivalent to X.", + " cannot be discussed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.56473159790039, + "mean_logprob": -10.391182899475098, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.053348541259766, + "mean_logprob": -11.684449513753256, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.781681060791016, + "mean_logprob": -8.156336212158203, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -43.4193115234375, + "mean_logprob": -10.854827880859375, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 2.234846687316894, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-046", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "R is earlier than S. T is later than U. S is earlier than U. Which ordering must hold?", + "continuations": [ + " T is earlier than R.", + " R is earlier than T.", + " U is earlier than S.", + " S is later than T." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -53.565399169921875, + "mean_logprob": -8.927566528320312, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -59.12684631347656, + "mean_logprob": -9.854474385579428, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -68.385986328125, + "mean_logprob": -11.397664388020834, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -61.4848518371582, + "mean_logprob": -10.247475306193033, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.9269078572591152, + "context_truncated": false, + "elapsed_seconds": 0.0010565329976088833 + }, + { + "id": "logical_reasoning-047", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If the engine starts, either fuel or battery power is available. The engine starts, and no fuel is available. Thus", + "continuations": [ + " no power is available.", + " the engine did not start.", + " battery power is available.", + " fuel is available." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -36.70459747314453, + "mean_logprob": -7.340919494628906, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -62.772552490234375, + "mean_logprob": -10.462092081705729, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -50.17485046386719, + "mean_logprob": -10.034970092773438, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -39.68201446533203, + "mean_logprob": -9.920503616333008, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 2.5795841217041016, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "logical_reasoning-048", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Exactly one person among A, B, and C lies. A says B tells the truth. B says C tells the truth. C says A lies. The liar is", + "continuations": [ + " A.", + " B.", + " impossible to determine.", + " C." + ], + "tags": [ + "truth_tellers" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.138799667358398, + "mean_logprob": -6.069399833679199, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.505229949951172, + "mean_logprob": -8.252614974975586, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -29.860645294189453, + "mean_logprob": -7.465161323547363, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.113693237304688, + "mean_logprob": -8.556846618652344, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 1.395761489868164, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "logical_reasoning-049", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No glens are haps. Some jors are glens. Every jor is a mip. It follows that some mips are", + "continuations": [ + " not haps.", + " haps.", + " not jors.", + " all glens." + ], + "tags": [ + "syllogism" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -50.9048957824707, + "mean_logprob": -12.726223945617676, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.01067352294922, + "mean_logprob": -12.003557840983072, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.0836181640625, + "mean_logprob": -11.520904541015625, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.50160598754883, + "mean_logprob": -9.375401496887207, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 2.145503044128418, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "logical_reasoning-050", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "A proposition is true only if Q is true, and Q is true only if R is false. The proposition is true. Therefore, R is", + "continuations": [ + " true.", + " false.", + " both true and false.", + " independent of Q." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -9.399139404296875, + "mean_logprob": -4.6995697021484375, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.533833503723145, + "mean_logprob": -6.766916751861572, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -32.11164855957031, + "mean_logprob": -6.422329711914062, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.37384796142578, + "mean_logprob": -7.593461990356445, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 1.7227600097656248, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "code_completion-001", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def square(x):\n", + "continuations": [ + " return x * x\n", + " return x + x\n", + " print(x)\n", + " return 2\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -97.57011413574219, + "mean_logprob": -12.196264266967773, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -92.40805053710938, + "mean_logprob": -11.551006317138672, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -78.99382781982422, + "mean_logprob": -9.874228477478027, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -75.73626708984375, + "mean_logprob": -10.819466727120536, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.9452382496425091, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "code_completion-002", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def is_empty(items):\n", + "continuations": [ + " return items[0]\n", + " return len(items) == 0\n", + " return len(items) + 1\n", + " items.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -79.7905502319336, + "mean_logprob": -8.865616692437065, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -138.89581298828125, + "mean_logprob": -9.259720865885416, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -135.52818298339844, + "mean_logprob": -9.680584498814174, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -88.14127349853516, + "mean_logprob": -8.814127349853516, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.0514893425835492, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "code_completion-003", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def greet(name):\n", + "continuations": [ + " return name - \"Hello\"\n", + " print = name\n", + " return \"Hello, \" + name\n", + " return len(\"Hello\")\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -142.83755493164062, + "mean_logprob": -11.903129577636719, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -77.143798828125, + "mean_logprob": -11.020542689732142, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -132.3349151611328, + "mean_logprob": -10.179608858548677, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -174.28909301757812, + "mean_logprob": -12.449220929827009, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.8409338311834649, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "code_completion-004", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def first(items):\n", + "continuations": [ + " return items[-1]\n", + " return len(items)\n", + " return items\n", + " return items[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -78.84619903564453, + "mean_logprob": -7.884619903564453, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -97.02578735351562, + "mean_logprob": -8.820526123046875, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -44.2044677734375, + "mean_logprob": -7.367411295572917, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -74.59817504882812, + "mean_logprob": -8.288686116536459, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.5172086079915363, + "context_truncated": false, + "elapsed_seconds": 0.000989087999187177 + }, + { + "id": "code_completion-005", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def add(a, b):\n", + "continuations": [ + " return a + b\n", + " return a - b\n", + " return a == b\n", + " return [a, b]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -84.39249420166016, + "mean_logprob": -10.54906177520752, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -87.61111450195312, + "mean_logprob": -10.95138931274414, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -86.24517822265625, + "mean_logprob": -9.58279758029514, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -107.43618774414062, + "mean_logprob": -10.743618774414063, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.9662641949123802, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-006", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def last(items):\n", + "continuations": [ + " return items[0]\n", + " return items[-1]\n", + " return items[1:]\n", + " return -len(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -86.77592468261719, + "mean_logprob": -9.641769409179688, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -94.96633911132812, + "mean_logprob": -9.496633911132813, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -90.82090759277344, + "mean_logprob": -9.082090759277344, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -126.55792236328125, + "mean_logprob": -10.546493530273438, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.41454315185546875, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-007", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def is_even(n):\n", + "continuations": [ + " return n / 2\n", + " return n % 2 == 1\n", + " return n % 2 == 0\n", + " return n * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -96.92322540283203, + "mean_logprob": -10.769247266981337, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -146.2318572998047, + "mean_logprob": -10.445132664271764, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -146.2080535888672, + "mean_logprob": -10.443432399204799, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -96.49711608886719, + "mean_logprob": -10.721901787651909, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.0017002650669653008, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-008", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def absolute_value(x):\n", + "continuations": [ + " return -abs(x)\n", + " return x * 0\n", + " return bool(x)\n", + " return abs(x)\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -127.18485260009766, + "mean_logprob": -11.562259327281605, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -102.0481185913086, + "mean_logprob": -11.338679843478733, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -101.27601623535156, + "mean_logprob": -10.127601623535156, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -100.4654541015625, + "mean_logprob": -11.162828233506945, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.0352266099717884, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-009", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def double_all(numbers):\n", + "continuations": [ + " return [n * 2 for n in numbers]\n", + " return numbers * 0\n", + " return [2 for n in numbers]\n", + " return len(numbers) * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -147.64785766601562, + "mean_logprob": -9.843190511067709, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -99.27908325195312, + "mean_logprob": -11.031009250217014, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -117.9864501953125, + "mean_logprob": -9.832204182942709, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -164.3397216796875, + "mean_logprob": -10.9559814453125, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.010986328125, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-010", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def contains(items, value):\n", + "continuations": [ + " return items == value\n", + " return value in items\n", + " return items + value\n", + " return items.index\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -74.20751190185547, + "mean_logprob": -8.245279100206163, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -84.86701965332031, + "mean_logprob": -10.608377456665039, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -75.13380432128906, + "mean_logprob": -9.391725540161133, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -82.65824890136719, + "mean_logprob": -9.184249877929688, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.9389707777235241, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-011", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def maximum(a, b):\n", + "continuations": [ + " return a + b\n", + " return a < b\n", + " return a if a > b else b\n", + " return None\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -82.65176391601562, + "mean_logprob": -10.331470489501953, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -91.59890747070312, + "mean_logprob": -11.44986343383789, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -111.78024291992188, + "mean_logprob": -9.315020243326822, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -67.12684631347656, + "mean_logprob": -9.589549473353795, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.2745292300269728, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-012", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def repeat(text, count):\n", + "continuations": [ + " return count / text\n", + " return len(text)\n", + " return text + count\n", + " return text * count\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -72.44398498535156, + "mean_logprob": -9.055498123168945, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -104.80321502685547, + "mean_logprob": -9.527565002441406, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -74.30772399902344, + "mean_logprob": -9.28846549987793, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -72.01290893554688, + "mean_logprob": -9.00161361694336, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.05388450622558594, + "context_truncated": false, + "elapsed_seconds": 0.0009684612523415126 + }, + { + "id": "code_completion-013", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def keys_list(mapping):\n", + "continuations": [ + " return list(mapping.keys())\n", + " return mapping.values\n", + " return [mapping]\n", + " return mapping.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -153.36859130859375, + "mean_logprob": -10.22457275390625, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -99.77587890625, + "mean_logprob": -9.977587890625, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -85.28521728515625, + "mean_logprob": -9.47613525390625, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -117.36581420898438, + "mean_logprob": -9.780484517415365, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.3043492635091152, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-014", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def starts_with_a(text):\n", + "continuations": [ + " return text[-1] == \"a\"\n", + " return text.startswith(\"a\")\n", + " return \"a\" + text\n", + " return text.count\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -124.22039794921875, + "mean_logprob": -8.28135986328125, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 16, + "sum_logprob": -168.06932067871094, + "mean_logprob": -10.504332542419434, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -93.84590911865234, + "mean_logprob": -9.384590911865235, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -86.09814453125, + "mean_logprob": -9.566460503472221, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.1032310485839858, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-015", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def total(numbers):\n", + "continuations": [ + " return len(numbers)\n", + " return max(numbers)\n", + " return sum(numbers)\n", + " return numbers[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -136.12796020507812, + "mean_logprob": -11.34399668375651, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -106.15505981445312, + "mean_logprob": -9.650459983132102, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -114.7218246459961, + "mean_logprob": -10.429256785999645, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -86.51661682128906, + "mean_logprob": -9.612957424587673, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.03750255854442841, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-016", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def clamp_zero(x):\n", + "continuations": [ + " return min(0, x)\n", + " return x == 0\n", + " return -x\n", + " return max(0, x)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -104.54830932617188, + "mean_logprob": -9.504391756924717, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -95.88739013671875, + "mean_logprob": -9.588739013671875, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -81.86072540283203, + "mean_logprob": -11.694389343261719, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -104.11245727539062, + "mean_logprob": -9.46476884321733, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.03962291370738669, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-017", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def reversed_copy(items):\n", + "continuations": [ + " return items[::-1]\n", + " return items[::1]\n", + " return items.reverse\n", + " return sorted(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -100.96229553222656, + "mean_logprob": -8.413524627685547, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -90.78009033203125, + "mean_logprob": -8.252735484730113, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -92.41345977783203, + "mean_logprob": -10.268162197536892, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -120.94824981689453, + "mean_logprob": -10.995295437899502, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.16078914295543356, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-018", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def count_positive(numbers):\n", + "continuations": [ + " return sum(numbers)\n", + " return sum(1 for n in numbers if n > 0)\n", + " return [n > 0 for n in numbers]\n", + " return len(numbers) > 0\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -113.18867492675781, + "mean_logprob": -10.289879538796164, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -159.12124633789062, + "mean_logprob": -8.840069240993923, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 15, + "sum_logprob": -147.34698486328125, + "mean_logprob": -9.82313232421875, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -163.1123046875, + "mean_logprob": -10.874153645833333, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.9830630832248275, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-019", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def unique_in_order(items):\n seen = set()\n", + "continuations": [ + " return set(items)\n", + " return sorted(items)\n", + " return [x for x in items if not (x in seen or seen.add(x))]\n", + " return list(reversed(items))\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -101.99751281738281, + "mean_logprob": -10.199751281738282, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -113.73632049560547, + "mean_logprob": -10.339665499600498, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 26, + "sum_logprob": -207.07992553710938, + "mean_logprob": -7.9646125206580525, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -162.52268981933594, + "mean_logprob": -10.83484598795573, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 2.235138761080229, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-020", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def safe_divide(a, b):\n if b == 0:\n", + "continuations": [ + " return a\n return b / a\n", + " b = 1\n return b\n", + " raise StopIteration\n return a + b\n", + " return None\n return a / b\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 18, + "sum_logprob": -172.92190551757812, + "mean_logprob": -9.606772528754341, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -173.09210205078125, + "mean_logprob": -9.61622789171007, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -221.42010498046875, + "mean_logprob": -10.064550226384943, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -178.76327514648438, + "mean_logprob": -9.408593428762336, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 0.19817909999200545, + "context_truncated": false, + "elapsed_seconds": 0.00102095562760951 + }, + { + "id": "code_completion-021", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def word_lengths(words):\n", + "continuations": [ + " return {word: len(word) for word in words}\n", + " return {len(words): word for word in words}\n", + " return [words, len(words)]\n", + " return words.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 19, + "sum_logprob": -204.7855224609375, + "mean_logprob": -10.778185392680921, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 19, + "sum_logprob": -232.39859008789062, + "mean_logprob": -12.231504741467928, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 16, + "sum_logprob": -183.40101623535156, + "mean_logprob": -11.462563514709473, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -126.35425567626953, + "mean_logprob": -11.486750516024502, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.6843781220285514, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-022", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def flatten_once(groups):\n", + "continuations": [ + " return list(groups)\n", + " return [item for group in groups for item in group]\n", + " return [groups[0]]\n", + " return sum(len(g) for g in groups)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -114.79316711425781, + "mean_logprob": -10.43574246493253, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -176.40701293945312, + "mean_logprob": -10.376883114085478, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -139.39584350585938, + "mean_logprob": -10.722757192758413, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 17, + "sum_logprob": -163.3931121826172, + "mean_logprob": -9.611359540153952, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.7655235739315263, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-023", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def factorial(n):\n result = 1\n for value in range(2, n + 1):\n", + "continuations": [ + " result += value\n return n\n", + " result = value\n return value\n", + " result *= value\n return result\n", + " n -= value\n return n\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 18, + "sum_logprob": -116.01634216308594, + "mean_logprob": -6.445352342393663, + "total_sequence_tokens": 49 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -126.11636352539062, + "mean_logprob": -7.418609619140625, + "total_sequence_tokens": 48 + }, + { + "index": 2, + "tokens": 18, + "sum_logprob": -129.35305786132812, + "mean_logprob": -7.186280992296007, + "total_sequence_tokens": 49 + }, + { + "index": 3, + "tokens": 18, + "sum_logprob": -125.0396499633789, + "mean_logprob": -6.946647220187717, + "total_sequence_tokens": 49 + } + ], + "best_vs_second_margin": 0.501294877794054, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-024", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def first_index(items, target):\n for index, item in enumerate(items):\n if item == target:\n", + "continuations": [ + " continue\n return 0\n", + " return item\n return target\n", + " break\n return index\n", + " return index\n return -1\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 20, + "sum_logprob": -135.6446533203125, + "mean_logprob": -6.782232666015625, + "total_sequence_tokens": 62 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -146.94973754882812, + "mean_logprob": -7.347486877441407, + "total_sequence_tokens": 62 + }, + { + "index": 2, + "tokens": 19, + "sum_logprob": -124.68810272216797, + "mean_logprob": -6.562531722219367, + "total_sequence_tokens": 61 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -156.81280517578125, + "mean_logprob": -7.467276436941964, + "total_sequence_tokens": 63 + } + ], + "best_vs_second_margin": 0.21970094379625849, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-025", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def merge_counts(left, right):\n result = dict(left)\n for key, value in right.items():\n", + "continuations": [ + " result[key] = result.get(key, 0) + value\n return result\n", + " result[value] = key\n return left\n", + " result.clear()\n return right\n", + " key += value\n return result.values()\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 30, + "sum_logprob": -244.63058471679688, + "mean_logprob": -8.154352823893229, + "total_sequence_tokens": 71 + }, + { + "index": 1, + "tokens": 21, + "sum_logprob": -177.833740234375, + "mean_logprob": -8.468273344494047, + "total_sequence_tokens": 62 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -131.110595703125, + "mean_logprob": -6.55552978515625, + "total_sequence_tokens": 61 + }, + { + "index": 3, + "tokens": 23, + "sum_logprob": -169.78488159179688, + "mean_logprob": -7.381951373556386, + "total_sequence_tokens": 64 + } + ], + "best_vs_second_margin": 0.8264215884001356, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-026", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def chunks(items, size):\n", + "continuations": [ + " return items[:size]\n", + " return [items[i:i + size] for i in range(0, len(items), size)]\n", + " return [size] * len(items)\n", + " return items[::size][0]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -89.96931457519531, + "mean_logprob": -8.17902859774503, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 31, + "sum_logprob": -312.9232177734375, + "mean_logprob": -10.094297347530242, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 16, + "sum_logprob": -166.2152557373047, + "mean_logprob": -10.388453483581543, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -121.03994750976562, + "mean_logprob": -8.069329833984375, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.10969876376065457, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-027", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def balanced_parentheses(text):\n depth = 0\n for char in text:\n if char == \"(\":\n depth += 1\n elif char == \")\":\n depth -= 1\n if depth < 0:\n return False\n", + "continuations": [ + " return True\n", + " return depth > 0\n", + " return depth == 0\n", + " return len(text) == depth\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -71.82276916503906, + "mean_logprob": -8.977846145629883, + "total_sequence_tokens": 134 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -79.83505249023438, + "mean_logprob": -8.87056138780382, + "total_sequence_tokens": 135 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -71.50790405273438, + "mean_logprob": -7.150790405273438, + "total_sequence_tokens": 136 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -148.07493591308594, + "mean_logprob": -10.576781136648995, + "total_sequence_tokens": 140 + } + ], + "best_vs_second_margin": 1.719770982530382, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-028", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def rotate_left(items, amount):\n if not items:\n return []\n", + "continuations": [ + " return items[::-1]\n", + " return items[:amount]\n", + " return items + amount\n", + " amount %= len(items)\n return items[amount:] + items[:amount]\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -95.48272705078125, + "mean_logprob": -7.9568939208984375, + "total_sequence_tokens": 44 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -94.53218078613281, + "mean_logprob": -8.593834616921164, + "total_sequence_tokens": 43 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -68.34432220458984, + "mean_logprob": -8.54304027557373, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -306.9648132324219, + "mean_logprob": -9.592650413513184, + "total_sequence_tokens": 64 + } + ], + "best_vs_second_margin": 0.586146354675293, + "context_truncated": false, + "elapsed_seconds": 0.0012281032504688483 + }, + { + "id": "code_completion-029", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def group_by_first_letter(words):\n groups = {}\n for word in words:\n", + "continuations": [ + " groups.setdefault(word[0], []).append(word)\n return groups\n", + " groups[word] = word[0]\n return words\n", + " groups.append(word[0])\n return groups\n", + " word[0] = groups\n return groups.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 32, + "sum_logprob": -254.250244140625, + "mean_logprob": -7.945320129394531, + "total_sequence_tokens": 65 + }, + { + "index": 1, + "tokens": 23, + "sum_logprob": -179.99609375, + "mean_logprob": -7.825917119565218, + "total_sequence_tokens": 56 + }, + { + "index": 2, + "tokens": 24, + "sum_logprob": -178.24038696289062, + "mean_logprob": -7.426682790120442, + "total_sequence_tokens": 57 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -235.32566833496094, + "mean_logprob": -9.413026733398437, + "total_sequence_tokens": 58 + } + ], + "best_vs_second_margin": 0.3992343294447753, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-030", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def moving_pairs(items):\n", + "continuations": [ + " return zip(items, items)\n", + " return list(zip(items, items[1:]))\n", + " return items[::2]\n", + " return [(items[0], items[-1])]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -148.17298889160156, + "mean_logprob": -11.39792222243089, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -184.58615112304688, + "mean_logprob": -9.229307556152344, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -104.77357482910156, + "mean_logprob": -9.524870439009232, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -206.5869140625, + "mean_logprob": -10.872995476973685, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.29556288285688836, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-031", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def invert_mapping(mapping):\n", + "continuations": [ + " return dict(mapping)\n", + " return list(mapping.items())\n", + " return {value: key for key, value in mapping.items()}\n", + " return mapping[::-1]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -95.92056274414062, + "mean_logprob": -8.720051158558238, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -134.69100952148438, + "mean_logprob": -8.979400634765625, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 24, + "sum_logprob": -226.30459594726562, + "mean_logprob": -9.4293581644694, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -107.95085144042969, + "mean_logprob": -8.303911649263823, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.4161395092944158, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-032", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def all_equal(items):\n iterator = iter(items)\n try:\n first = next(iterator)\n except StopIteration:\n", + "continuations": [ + " return False\n return first == iterator\n", + " first = None\n return items == first\n", + " return []\n return len(items)\n", + " return True\n return all(item == first for item in iterator)\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -181.52481079101562, + "mean_logprob": -8.251127763227982, + "total_sequence_tokens": 78 + }, + { + "index": 1, + "tokens": 21, + "sum_logprob": -173.49615478515625, + "mean_logprob": -8.261721656436013, + "total_sequence_tokens": 77 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -177.7333526611328, + "mean_logprob": -8.078788757324219, + "total_sequence_tokens": 78 + }, + { + "index": 3, + "tokens": 31, + "sum_logprob": -255.40794372558594, + "mean_logprob": -8.238965926631804, + "total_sequence_tokens": 87 + } + ], + "best_vs_second_margin": 0.16017716930758574, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-033", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def transpose(rows):\n", + "continuations": [ + " return [list(column) for column in zip(*rows)]\n", + " return rows[::-1]\n", + " return list(zip(rows))\n", + " return [sum(row) for row in rows]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -219.0318603515625, + "mean_logprob": -9.95599365234375, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -119.20109558105469, + "mean_logprob": -9.169315044696514, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -137.8314971923828, + "mean_logprob": -10.602422860952524, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 17, + "sum_logprob": -171.3200225830078, + "mean_logprob": -10.077648387235755, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.7866786076472359, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-034", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def running_totals(numbers):\n totals = []\n current = 0\n for number in numbers:\n", + "continuations": [ + " totals.append(number)\n return current\n", + " current += number\n totals.append(current)\n return totals\n", + " current = number\n return [current]\n", + " totals += current\n return numbers\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 23, + "sum_logprob": -191.52711486816406, + "mean_logprob": -8.32726586383322, + "total_sequence_tokens": 61 + }, + { + "index": 1, + "tokens": 36, + "sum_logprob": -277.66473388671875, + "mean_logprob": -7.712909274631077, + "total_sequence_tokens": 74 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -158.94686889648438, + "mean_logprob": -7.947343444824218, + "total_sequence_tokens": 58 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -156.03346252441406, + "mean_logprob": -8.212287501284951, + "total_sequence_tokens": 57 + } + ], + "best_vs_second_margin": 0.2344341701931416, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-035", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def binary_search(items, target):\n low, high = 0, len(items) - 1\n while low <= high:\n mid = (low + high) // 2\n if items[mid] == target:\n return mid\n if items[mid] < target:\n low = mid + 1\n else:\n", + "continuations": [ + " low = mid - 1\n return low\n", + " high = mid + 1\n return high\n", + " high = mid - 1\n return -1\n", + " return mid\n return None\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 24, + "sum_logprob": -173.46302795410156, + "mean_logprob": -7.2276261647542315, + "total_sequence_tokens": 163 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -165.760986328125, + "mean_logprob": -6.906707763671875, + "total_sequence_tokens": 163 + }, + { + "index": 2, + "tokens": 25, + "sum_logprob": -194.31053161621094, + "mean_logprob": -7.772421264648438, + "total_sequence_tokens": 164 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -164.673583984375, + "mean_logprob": -7.841599237351191, + "total_sequence_tokens": 160 + } + ], + "best_vs_second_margin": 0.3209184010823565, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-036", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def merge_sorted(left, right):\n result = []\n i = j = 0\n while i < len(left) and j < len(right):\n if left[i] <= right[j]:\n result.append(left[i])\n i += 1\n else:\n result.append(right[j])\n j += 1\n", + "continuations": [ + " return result\n", + " result.extend(left)\n return result\n", + " return sorted(left)\n", + " result.extend(left[i:])\n result.extend(right[j:])\n return result\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -52.18792724609375, + "mean_logprob": -8.697987874348959, + "total_sequence_tokens": 167 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -153.0741424560547, + "mean_logprob": -8.504119025336372, + "total_sequence_tokens": 179 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -120.54737854003906, + "mean_logprob": -10.958852594549006, + "total_sequence_tokens": 172 + }, + { + "index": 3, + "tokens": 37, + "sum_logprob": -304.4341125488281, + "mean_logprob": -8.227948987806165, + "total_sequence_tokens": 198 + } + ], + "best_vs_second_margin": 0.27617003753020697, + "context_truncated": false, + "elapsed_seconds": 0.0013785808769171126 + }, + { + "id": "code_completion-037", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def topological_visit(node, graph, visiting, visited, order):\n if node in visiting:\n raise ValueError(\"cycle\")\n if node in visited:\n return\n visiting.add(node)\n for neighbor in graph.get(node, []):\n topological_visit(neighbor, graph, visiting, visited, order)\n", + "continuations": [ + " visiting.remove(node)\n visited.add(node)\n order.append(node)\n", + " order.append(node)\n visiting.add(node)\n", + " visited.remove(node)\n order.clear()\n", + " return graph[node]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 36, + "sum_logprob": -305.34515380859375, + "mean_logprob": -8.481809828016493, + "total_sequence_tokens": 162 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -188.66275024414062, + "mean_logprob": -7.860947926839192, + "total_sequence_tokens": 150 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -188.90399169921875, + "mean_logprob": -8.586545077237217, + "total_sequence_tokens": 148 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -84.91897583007812, + "mean_logprob": -8.491897583007812, + "total_sequence_tokens": 136 + } + ], + "best_vs_second_margin": 0.6208619011773004, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-038", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def longest_common_prefix(words):\n if not words:\n return \"\"\n prefix = words[0]\n for word in words[1:]:\n while not word.startswith(prefix):\n", + "continuations": [ + " prefix += word\n", + " prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n", + " word = word[:-1]\n return word\n", + " return words[0]\n return prefix + word\n" + ], + "tags": [ + "python", + "string" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 17, + "sum_logprob": -107.31185913085938, + "mean_logprob": -6.312462301815257, + "total_sequence_tokens": 98 + }, + { + "index": 1, + "tokens": 65, + "sum_logprob": -379.3370056152344, + "mean_logprob": -5.8359539325420675, + "total_sequence_tokens": 146 + }, + { + "index": 2, + "tokens": 26, + "sum_logprob": -156.85940551757812, + "mean_logprob": -6.033054058368389, + "total_sequence_tokens": 107 + }, + { + "index": 3, + "tokens": 26, + "sum_logprob": -177.33370971679688, + "mean_logprob": -6.82052729679988, + "total_sequence_tokens": 107 + } + ], + "best_vs_second_margin": 0.19710012582632164, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-039", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def memoized_fibonacci(n, cache=None):\n if cache is None:\n cache = {0: 0, 1: 1}\n if n not in cache:\n", + "continuations": [ + " cache[n] = n\n", + " return cache\n", + " cache[n] = memoized_fibonacci(n - 1, cache) + memoized_fibonacci(n - 2, cache)\n return cache[n]\n", + " cache.clear()\n return n\n" + ], + "tags": [ + "python", + "recursion" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -99.8953628540039, + "mean_logprob": -6.659690856933594, + "total_sequence_tokens": 81 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -90.60519409179688, + "mean_logprob": -8.236835826526988, + "total_sequence_tokens": 77 + }, + { + "index": 2, + "tokens": 61, + "sum_logprob": -657.0621337890625, + "mean_logprob": -10.77151038998463, + "total_sequence_tokens": 127 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -162.08450317382812, + "mean_logprob": -7.718309674944196, + "total_sequence_tokens": 87 + } + ], + "best_vs_second_margin": 1.0586188180106024, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-040", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def intervals_overlap(a_start, a_end, b_start, b_end):\n", + "continuations": [ + " return a_end < b_start\n", + " return a_start == b_end\n", + " return a_start + b_start\n", + " return max(a_start, b_start) <= min(a_end, b_end)\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -147.03858947753906, + "mean_logprob": -11.310660729041466, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -146.76197814941406, + "mean_logprob": -10.482998439243861, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -166.15640258789062, + "mean_logprob": -11.868314470563616, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 29, + "sum_logprob": -311.6648864746094, + "mean_logprob": -10.7470650508486, + "total_sequence_tokens": 57 + } + ], + "best_vs_second_margin": 0.2640666116047381, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-041", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def breadth_first_distances(graph, start):\n from collections import deque\n distances = {start: 0}\n queue = deque([start])\n while queue:\n node = queue.popleft()\n for neighbor in graph.get(node, []):\n if neighbor not in distances:\n", + "continuations": [ + " distances[neighbor] = distances[node] + 1\n queue.append(neighbor)\n return distances\n", + " distances[node] += 1\n return queue\n", + " queue.append(node)\n return graph\n", + " return neighbor\n return distances[start]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 67, + "sum_logprob": -505.9909362792969, + "mean_logprob": -7.55210352655667, + "total_sequence_tokens": 194 + }, + { + "index": 1, + "tokens": 34, + "sum_logprob": -209.7467803955078, + "mean_logprob": -6.169022952809053, + "total_sequence_tokens": 161 + }, + { + "index": 2, + "tokens": 32, + "sum_logprob": -188.80076599121094, + "mean_logprob": -5.900023937225342, + "total_sequence_tokens": 159 + }, + { + "index": 3, + "tokens": 29, + "sum_logprob": -170.319091796875, + "mean_logprob": -5.873072130926724, + "total_sequence_tokens": 156 + } + ], + "best_vs_second_margin": 0.026951806298617598, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-042", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def deep_get(mapping, path, default=None):\n current = mapping\n for key in path:\n if not isinstance(current, dict) or key not in current:\n return default\n", + "continuations": [ + " current = mapping[key]\n return mapping\n", + " current = current[key]\n return current\n", + " path = current\n return key\n", + " return current.get(path)\n return default\n" + ], + "tags": [ + "python", + "nested_data" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -150.05130004882812, + "mean_logprob": -6.820513638583097, + "total_sequence_tokens": 98 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -166.23956298828125, + "mean_logprob": -8.311978149414063, + "total_sequence_tokens": 96 + }, + { + "index": 2, + "tokens": 17, + "sum_logprob": -126.64860534667969, + "mean_logprob": -7.449917961569393, + "total_sequence_tokens": 93 + }, + { + "index": 3, + "tokens": 23, + "sum_logprob": -190.35177612304688, + "mean_logprob": -8.276164179262908, + "total_sequence_tokens": 99 + } + ], + "best_vs_second_margin": 0.6294043229862964, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-043", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def stable_partition(items, predicate):\n matching = []\n other = []\n for item in items:\n if predicate(item):\n matching.append(item)\n else:\n other.append(item)\n", + "continuations": [ + " return sorted(items)\n", + " return matching or other\n", + " return matching + other\n", + " return zip(matching, other)\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -106.69406127929688, + "mean_logprob": -9.699460116299717, + "total_sequence_tokens": 116 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -89.26392364501953, + "mean_logprob": -9.918213738335503, + "total_sequence_tokens": 114 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -87.8135986328125, + "mean_logprob": -9.757066514756945, + "total_sequence_tokens": 114 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -126.45347595214844, + "mean_logprob": -9.032391139439174, + "total_sequence_tokens": 119 + } + ], + "best_vs_second_margin": 0.6670689768605431, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-044", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def parse_key_value(lines):\n result = {}\n for line in lines:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n key, separator, value = line.partition(\"=\")\n if not separator:\n raise ValueError(\"missing equals sign\")\n", + "continuations": [ + " result[value] = separator\n return lines\n", + " result[key] = key\n return value\n", + " result.clear()\n return key\n", + " result[key.strip()] = value.strip()\n return result\n" + ], + "tags": [ + "python", + "parsing" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -174.09463500976562, + "mean_logprob": -7.913392500443892, + "total_sequence_tokens": 170 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -142.224853515625, + "mean_logprob": -7.11124267578125, + "total_sequence_tokens": 168 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -122.56908416748047, + "mean_logprob": -6.128454208374023, + "total_sequence_tokens": 168 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -252.6643524169922, + "mean_logprob": -7.895761013031006, + "total_sequence_tokens": 180 + } + ], + "best_vs_second_margin": 0.9827884674072269, + "context_truncated": false, + "elapsed_seconds": 0.0013656384981004521 + }, + { + "id": "code_completion-045", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def sliding_window_sums(numbers, width):\n if width <= 0 or width > len(numbers):\n return []\n current = sum(numbers[:width])\n result = [current]\n for index in range(width, len(numbers)):\n", + "continuations": [ + " current += numbers[index] - numbers[index - width]\n result.append(current)\n return result\n", + " current = numbers[index]\n return [current]\n", + " result.append(sum(numbers))\n return numbers\n", + " width += index\n return result[:width]\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 47, + "sum_logprob": -406.92645263671875, + "mean_logprob": -8.658009630568484, + "total_sequence_tokens": 148 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -188.46385192871094, + "mean_logprob": -7.852660497029622, + "total_sequence_tokens": 125 + }, + { + "index": 2, + "tokens": 27, + "sum_logprob": -210.1231231689453, + "mean_logprob": -7.782337895146123, + "total_sequence_tokens": 128 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -214.68893432617188, + "mean_logprob": -8.587557373046875, + "total_sequence_tokens": 126 + } + ], + "best_vs_second_margin": 0.07032260188349948, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + }, + { + "id": "code_completion-046", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def retry(operation, attempts):\n last_error = None\n for _ in range(attempts):\n try:\n return operation()\n except Exception as error:\n last_error = error\n", + "continuations": [ + " return last_error\n", + " if last_error is not None:\n raise last_error\n raise ValueError(\"attempts must be positive\")\n", + " operation()\n return None\n", + " except last_error:\n return attempts\n" + ], + "tags": [ + "python", + "exceptions" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -93.41097259521484, + "mean_logprob": -10.378996955023872, + "total_sequence_tokens": 101 + }, + { + "index": 1, + "tokens": 47, + "sum_logprob": -452.1944580078125, + "mean_logprob": -9.621158681017286, + "total_sequence_tokens": 139 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -131.586669921875, + "mean_logprob": -9.3990478515625, + "total_sequence_tokens": 106 + }, + { + "index": 3, + "tokens": 20, + "sum_logprob": -166.6189422607422, + "mean_logprob": -8.330947113037109, + "total_sequence_tokens": 112 + } + ], + "best_vs_second_margin": 1.0681007385253913, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + }, + { + "id": "code_completion-047", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def normalize_rows(matrix):\n result = []\n for row in matrix:\n total = sum(row)\n if total == 0:\n result.append([0.0 for _ in row])\n else:\n", + "continuations": [ + " result.append(row)\n", + " result.append([total for _ in row])\n", + " result.append([value / total for value in row])\n return result\n", + " result = row / total\n return matrix\n" + ], + "tags": [ + "python", + "numeric" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 19, + "sum_logprob": -129.2610321044922, + "mean_logprob": -6.803212216025905, + "total_sequence_tokens": 116 + }, + { + "index": 1, + "tokens": 27, + "sum_logprob": -172.866943359375, + "mean_logprob": -6.402479383680555, + "total_sequence_tokens": 124 + }, + { + "index": 2, + "tokens": 34, + "sum_logprob": -260.188720703125, + "mean_logprob": -7.652609432444853, + "total_sequence_tokens": 131 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -199.45738220214844, + "mean_logprob": -7.9782952880859375, + "total_sequence_tokens": 122 + } + ], + "best_vs_second_margin": 0.4007328323453496, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + }, + { + "id": "code_completion-048", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def merge_ranges(ranges):\n if not ranges:\n return []\n ordered = sorted(ranges)\n merged = [list(ordered[0])]\n for start, end in ordered[1:]:\n previous = merged[-1]\n if start <= previous[1]:\n previous[1] = max(previous[1], end)\n else:\n", + "continuations": [ + " previous[0] = start\n return ranges\n", + " merged = [[start, end]]\n return ordered\n", + " return previous\n return merged\n", + " merged.append([start, end])\n return [tuple(pair) for pair in merged]\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 25, + "sum_logprob": -197.65135192871094, + "mean_logprob": -7.906054077148437, + "total_sequence_tokens": 171 + }, + { + "index": 1, + "tokens": 29, + "sum_logprob": -261.44549560546875, + "mean_logprob": -9.015361917429956, + "total_sequence_tokens": 175 + }, + { + "index": 2, + "tokens": 21, + "sum_logprob": -179.13096618652344, + "mean_logprob": -8.530046008882069, + "total_sequence_tokens": 167 + }, + { + "index": 3, + "tokens": 44, + "sum_logprob": -393.7121887207031, + "mean_logprob": -8.948004289106889, + "total_sequence_tokens": 190 + } + ], + "best_vs_second_margin": 0.6239919317336318, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + }, + { + "id": "code_completion-049", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def count_paths(grid, row=0, col=0, memo=None):\n if memo is None:\n memo = {}\n if row >= len(grid) or col >= len(grid[0]) or grid[row][col] == 1:\n return 0\n if (row, col) == (len(grid) - 1, len(grid[0]) - 1):\n return 1\n if (row, col) not in memo:\n", + "continuations": [ + " memo[(row, col)] = count_paths(grid, row + 1, col, memo) + count_paths(grid, row, col + 1, memo)\n return memo[(row, col)]\n", + " memo.clear()\n return 0\n", + " return count_paths(grid, row - 1, col - 1, memo)\n return memo\n", + " memo[(row, col)] = 1\n return len(memo)\n" + ], + "tags": [ + "python", + "dynamic_programming" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 71, + "sum_logprob": -657.2684326171875, + "mean_logprob": -9.257301867847712, + "total_sequence_tokens": 234 + }, + { + "index": 1, + "tokens": 22, + "sum_logprob": -161.1197967529297, + "mean_logprob": -7.323627125133168, + "total_sequence_tokens": 185 + }, + { + "index": 2, + "tokens": 38, + "sum_logprob": -347.64495849609375, + "mean_logprob": -9.148551539370889, + "total_sequence_tokens": 201 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -272.0635070800781, + "mean_logprob": -8.501984596252441, + "total_sequence_tokens": 195 + } + ], + "best_vs_second_margin": 1.1783574711192735, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + }, + { + "id": "code_completion-050", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def compose(*functions):\n def composed(value):\n result = value\n for function in reversed(functions):\n", + "continuations": [ + " result = functions(result)\n return value\n", + " result = function(result)\n return result\n return composed\n", + " return function\n return functions\n return result\n", + " result.append(function)\n return result\n return functions\n" + ], + "tags": [ + "python", + "higher_order" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 29, + "sum_logprob": -197.3267822265625, + "mean_logprob": -6.804371800915948, + "total_sequence_tokens": 79 + }, + { + "index": 1, + "tokens": 35, + "sum_logprob": -226.88583374023438, + "mean_logprob": -6.482452392578125, + "total_sequence_tokens": 85 + }, + { + "index": 2, + "tokens": 30, + "sum_logprob": -209.857666015625, + "mean_logprob": -6.995255533854166, + "total_sequence_tokens": 80 + }, + { + "index": 3, + "tokens": 36, + "sum_logprob": -264.1563720703125, + "mean_logprob": -7.337677001953125, + "total_sequence_tokens": 86 + } + ], + "best_vs_second_margin": 0.3219194083378234, + "context_truncated": false, + "elapsed_seconds": 0.0016606184993482505 + } + ] +}