diff --git "a/eval_results/base_bench_1.1/c_only/report.json" "b/eval_results/base_bench_1.1/c_only/report.json" new file mode 100644--- /dev/null +++ "b/eval_results/base_bench_1.1/c_only/report.json" @@ -0,0 +1,19763 @@ +{ + "created_at": "2026-08-19T11:29:02.890733+00:00", + "benchmark": "BananaMind Base Bench 1.1", + "benchmark_version": "1.1", + "dataset_id": "BananaMind/BananaMind-Base-Bench-1.1", + "dataset_revision": "main", + "dataset_file": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-Base-Bench-1.1/data/test.jsonl", + "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21", + "model": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-2.1-Unified", + "model_revision": "main", + "tokenizer": "/home/banaxi/Desktop/BananaMind/BananaMind-2/BananaMind-2.1-Unified", + "device": "cuda:0", + "dtype": "bfloat16", + "model_context_length": 4096, + "scoring": { + "task": "four-choice base-text continuation", + "choice_score": "mean conditional token log-probability", + "chat_template": false, + "generation": false, + "add_special_tokens": false, + "add_bos": false + }, + "elo": { + "scale": 400.0, + "prior_rating": 1000.0, + "prior_weight": 4.0, + "method": "weighted fixed-item logistic maximum-likelihood rating" + }, + "summary": { + "official_complete_run": true, + "cases": 350, + "passed": 82, + "accuracy": 0.2342857142857143, + "weighted_points": 147.2375, + "possible_weighted_points": 668.1875, + "weighted_accuracy": 0.22035356842203724, + "overall_elo": 774, + "overall_elo_unrounded": 773.6592594258416, + "categories": { + "language_completion": { + "cases": 50, + "passed": 10, + "accuracy": 0.2, + "weighted_points": 13.25, + "possible_weighted_points": 78.5, + "weighted_accuracy": 0.16878980891719744, + "elo": 626, + "elo_unrounded": 625.8836702912365 + }, + "commonsense": { + "cases": 50, + "passed": 16, + "accuracy": 0.32, + "weighted_points": 25.3, + "possible_weighted_points": 87.17500000000001, + "weighted_accuracy": 0.2902208201892744, + "elo": 794, + "elo_unrounded": 794.3895637573371 + }, + "world_knowledge": { + "cases": 50, + "passed": 12, + "accuracy": 0.24, + "weighted_points": 22.275000000000002, + "possible_weighted_points": 87.72500000000001, + "weighted_accuracy": 0.25391849529780564, + "elo": 764, + "elo_unrounded": 763.65670714409 + }, + "context_tracking": { + "cases": 50, + "passed": 10, + "accuracy": 0.2, + "weighted_points": 15.899999999999999, + "possible_weighted_points": 94.19999999999999, + "weighted_accuracy": 0.16878980891719747, + "elo": 719, + "elo_unrounded": 718.677085203929 + }, + "quantitative": { + "cases": 50, + "passed": 11, + "accuracy": 0.22, + "weighted_points": 22.1, + "possible_weighted_points": 103.025, + "weighted_accuracy": 0.21451104100946372, + "elo": 819, + "elo_unrounded": 818.6219205710788 + }, + "logical_reasoning": { + "cases": 50, + "passed": 17, + "accuracy": 0.34, + "weighted_points": 34.7625, + "possible_weighted_points": 107.66250000000001, + "weighted_accuracy": 0.322884012539185, + "elo": 965, + "elo_unrounded": 965.0494041149989 + }, + "code_completion": { + "cases": 50, + "passed": 6, + "accuracy": 0.12, + "weighted_points": 13.649999999999999, + "possible_weighted_points": 109.89999999999999, + "weighted_accuracy": 0.12420382165605096, + "elo": 753, + "elo_unrounded": 753.4945744675929 + } + }, + "difficulties": { + "easy": { + "cases": 117, + "passed": 36, + "accuracy": 0.3076923076923077, + "weighted_points": 43.25, + "possible_weighted_points": 141.2, + "weighted_accuracy": 0.3063031161473088, + "elo": 725, + "elo_unrounded": 725.2040095593443 + }, + "medium": { + "cases": 117, + "passed": 23, + "accuracy": 0.19658119658119658, + "weighted_points": 40.425000000000004, + "possible_weighted_points": 211.875, + "weighted_accuracy": 0.19079646017699117, + "elo": 713, + "elo_unrounded": 712.5612708956528 + }, + "hard": { + "cases": 116, + "passed": 23, + "accuracy": 0.19827586206896552, + "weighted_points": 63.5625, + "possible_weighted_points": 315.1125, + "weighted_accuracy": 0.20171367368796858, + "elo": 870, + "elo_unrounded": 870.1613026093421 + } + }, + "context_truncations": 0 + }, + "results": [ + { + "id": "language_completion-001", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Dark clouds gathered above the park, so Mira opened", + "continuations": [ + " a sandwich loudly.", + " the sunlight.", + " her umbrella.", + " the calendar yesterday." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -75.09344482421875, + "mean_logprob": -9.386680603027344, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.005577087402344, + "mean_logprob": -11.001859029134115, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -74.81809997558594, + "mean_logprob": -10.688299996512276, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -74.84278106689453, + "mean_logprob": -9.355347633361816, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.031332969665527344, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-002", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "The librarian asked everyone to speak", + "continuations": [ + " brightly.", + " quietly.", + " woodenly.", + " tomorrow." + ], + "tags": [ + "adverb" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.389522552490234, + "mean_logprob": -6.463174184163411, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.48577308654785, + "mean_logprob": -8.495257695515951, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.191974639892578, + "mean_logprob": -7.5479936599731445, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.382034301757812, + "mean_logprob": -7.595508575439453, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.0848194758097334, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-003", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "The brass key fit perfectly into the", + "continuations": [ + " pillow.", + " thunder.", + " lock.", + " soup." + ], + "tags": [ + "collocation" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.00227355957031, + "mean_logprob": -8.250568389892578, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -30.534887313842773, + "mean_logprob": -10.178295771280924, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.383766174316406, + "mean_logprob": -8.461255391438803, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -29.29128074645996, + "mean_logprob": -9.763760248819986, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.21068700154622455, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-004", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "After kneading the dough, the baker placed it in the", + "continuations": [ + " envelope.", + " freezer to read.", + " bicycle.", + " oven." + ], + "tags": [ + "coherence" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.45635223388672, + "mean_logprob": -8.36408805847168, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -60.79744338989258, + "mean_logprob": -12.159488677978516, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -53.4483642578125, + "mean_logprob": -13.362091064453125, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -36.88429641723633, + "mean_logprob": -12.29476547241211, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 3.795400619506836, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-005", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Leo was exhausted after the hike and went straight to", + "continuations": [ + " bed.", + " the trumpet.", + " blue.", + " calculate." + ], + "tags": [ + "coherence" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -16.689783096313477, + "mean_logprob": -8.344891548156738, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.797420501708984, + "mean_logprob": -8.759484100341798, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -22.8536376953125, + "mean_logprob": -11.42681884765625, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.967803955078125, + "mean_logprob": -11.322601318359375, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.4145925521850593, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-006", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "The ice cube disappeared because it had", + "continuations": [ + " flown north.", + " melted.", + " written a letter.", + " become louder." + ], + "tags": [ + "verb" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.83747863769531, + "mean_logprob": -10.209369659423828, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -43.31185531616211, + "mean_logprob": -14.43728510538737, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.72602081298828, + "mean_logprob": -9.18150520324707, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.09503173828125, + "mean_logprob": -10.01900634765625, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.8375011444091793, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-007", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "One child waited outside; soon, three more", + "continuations": [ + " childs arrived.", + " childes arrived.", + " children arrived.", + " child arriveds." + ], + "tags": [ + "grammar" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -42.61653137207031, + "mean_logprob": -10.654132843017578, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -51.82566833496094, + "mean_logprob": -12.956417083740234, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.407758712768555, + "mean_logprob": -10.469252904256185, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -47.05091094970703, + "mean_logprob": -11.762727737426758, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.18487993876139264, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-008", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Yesterday, Priya walked to the station and", + "continuations": [ + " waits for tomorrow.", + " waiting there now.", + " will walked home.", + " bought a ticket." + ], + "tags": [ + "tense" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -62.267967224121094, + "mean_logprob": -8.895423889160156, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.706077575683594, + "mean_logprob": -9.176519393920898, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -42.279502868652344, + "mean_logprob": -8.455900573730469, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -53.898963928222656, + "mean_logprob": -8.983160654703775, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.43952331542968714, + "context_truncated": false, + "elapsed_seconds": 0.0276520721236011 + }, + { + "id": "language_completion-009", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The cup is empty, but the pitcher is", + "continuations": [ + " full.", + " emptier than silence.", + " running yesterday.", + " carefully." + ], + "tags": [ + "contrast" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.668416023254395, + "mean_logprob": -7.834208011627197, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -53.67939758300781, + "mean_logprob": -8.946566263834635, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.28616714477539, + "mean_logprob": -9.857233428955078, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -19.868234634399414, + "mean_logprob": -9.934117317199707, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 1.1123582522074376, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-010", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Neither the teacher nor the students", + "continuations": [ + " was arrives.", + " were ready.", + " is readiness.", + " be readying." + ], + "tags": [ + "agreement" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -55.78727722167969, + "mean_logprob": -13.946819305419922, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.39411926269531, + "mean_logprob": -12.464706420898438, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -52.26451110839844, + "mean_logprob": -13.06612777709961, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -58.030635833740234, + "mean_logprob": -14.507658958435059, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.6014213562011719, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-011", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Nora packed an apple, a bottle of water, and", + "continuations": [ + " because.", + " beneath.", + " a sandwich.", + " although." + ], + "tags": [ + "parallelism" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.133544921875, + "mean_logprob": -8.5667724609375, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.110225677490234, + "mean_logprob": -9.777556419372559, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -48.251548767089844, + "mean_logprob": -9.650309753417968, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -23.05919647216797, + "mean_logprob": -11.529598236083984, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.083537292480468, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-012", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "A robin is a bird, and a salmon is", + "continuations": [ + " beneath flying.", + " an quickly.", + " yesterday's wing.", + " a fish." + ], + "tags": [ + "syntax" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -50.73292541503906, + "mean_logprob": -10.146585083007812, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.667434692382812, + "mean_logprob": -8.889144897460938, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -61.895389556884766, + "mean_logprob": -8.842198508126396, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.74273681640625, + "mean_logprob": -10.580912272135416, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.04694638933454165, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-013", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "She adopted a puppy from", + "continuations": [ + " an animal shelter.", + " shelter animal an.", + " a sheltering quickly.", + " the adopted because." + ], + "tags": [ + "article" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -44.76959991455078, + "mean_logprob": -8.953919982910156, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -45.21521759033203, + "mean_logprob": -9.043043518066407, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -50.30296325683594, + "mean_logprob": -10.060592651367188, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -45.28905487060547, + "mean_logprob": -11.322263717651367, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.08912353515625071, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-014", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Daniel thanked Rosa because", + "continuations": [ + " him had helped she.", + " she had helped him.", + " her helped himself.", + " they helps he." + ], + "tags": [ + "pronouns" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -46.08085632324219, + "mean_logprob": -9.216171264648438, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -50.34266662597656, + "mean_logprob": -10.068533325195313, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -45.75080871582031, + "mean_logprob": -11.437702178955078, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.93122100830078, + "mean_logprob": -9.232805252075195, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.016633987426757812, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-015", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "This suitcase is heavy, but that one is even", + "continuations": [ + " most heavy.", + " heavily one.", + " heavier.", + " more heavier." + ], + "tags": [ + "comparative" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -26.086139678955078, + "mean_logprob": -8.695379892985025, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.587026596069336, + "mean_logprob": -8.86234219868978, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -30.297840118408203, + "mean_logprob": -10.0992800394694, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.557373046875, + "mean_logprob": -10.13934326171875, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.1669623057047538, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-016", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Monday comes immediately after", + "continuations": [ + " Wednesday.", + " evening.", + " January.", + " Sunday." + ], + "tags": [ + "sequence" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -54.22332763671875, + "mean_logprob": -10.84466552734375, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.126399993896484, + "mean_logprob": -10.375466664632162, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.553394317626953, + "mean_logprob": -8.776697158813477, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.807945251464844, + "mean_logprob": -10.269315083821615, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.4926179250081386, + "context_truncated": false, + "elapsed_seconds": 0.0006647242516919505 + }, + { + "id": "language_completion-017", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "Although the road was narrow, the driver remained", + "continuations": [ + " careful.", + " width.", + " carefully road.", + " narrowness." + ], + "tags": [ + "grammar" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.629852294921875, + "mean_logprob": -9.814926147460938, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.954055786132812, + "mean_logprob": -7.3180185953776045, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -33.476844787597656, + "mean_logprob": -11.158948262532553, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.66669464111328, + "mean_logprob": -8.888898213704428, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.5708796183268232, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-018", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The forecast promised sunshine. Nevertheless, we packed", + "continuations": [ + " the forecast itself.", + " rain jackets.", + " a darker noon.", + " sunshine into boxes." + ], + "tags": [ + "discourse" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.06196594238281, + "mean_logprob": -8.212393188476563, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -46.920936584472656, + "mean_logprob": -9.384187316894531, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -45.308799743652344, + "mean_logprob": -7.551466623942058, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -59.53303527832031, + "mean_logprob": -8.50471932547433, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.6609265645345053, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-019", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Amir revised the report twice before submitting it, thereby", + "continuations": [ + " forgetting what reports are.", + " submitting before revising.", + " reducing the chance of errors.", + " making twice become three." + ], + "tags": [ + "discourse" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -50.82826232910156, + "mean_logprob": -8.47137705485026, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -65.62808990478516, + "mean_logprob": -9.375441414969307, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -46.210296630859375, + "mean_logprob": -7.7017161051432295, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -48.51474380493164, + "mean_logprob": -9.702948760986327, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.7696609497070304, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-020", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The museum closes at six. We arrived at five forty-five, leaving us", + "continuations": [ + " closed before arrival.", + " six museums to visit.", + " yesterday instead.", + " only fifteen minutes." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -50.33039093017578, + "mean_logprob": -10.066078186035156, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -52.92252731323242, + "mean_logprob": -8.82042121887207, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -54.83954620361328, + "mean_logprob": -10.967909240722657, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.06440353393555, + "mean_logprob": -7.01288070678711, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.8075405120849606, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-021", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The committee postponed the vote until every member had", + "continuations": [ + " reviewed the proposal.", + " been proposal by voting.", + " postponed every committee.", + " review the voted." + ], + "tags": [ + "aspect" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -65.63237762451172, + "mean_logprob": -10.938729604085287, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -75.02275848388672, + "mean_logprob": -10.717536926269531, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -67.61900329589844, + "mean_logprob": -9.659857613699776, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -52.80656433105469, + "mean_logprob": -10.561312866210937, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.9014552525111608, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-022", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Had Elena known the bridge was closed, she", + "continuations": [ + " takes that route anyway.", + " would have taken another route.", + " will know it tomorrow yesterday.", + " had closes the bridge." + ], + "tags": [ + "conditional" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -62.978729248046875, + "mean_logprob": -10.496454874674479, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -54.1201057434082, + "mean_logprob": -9.020017623901367, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -81.77124786376953, + "mean_logprob": -8.177124786376954, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -44.001930236816406, + "mean_logprob": -7.333655039469401, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.8434697469075525, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-023", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The novel was praised not only for its plot but also for", + "continuations": [ + " it was long.", + " plotting beyond.", + " its vivid characters.", + " praise was also." + ], + "tags": [ + "parallelism" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -38.83384323120117, + "mean_logprob": -9.708460807800293, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.696311950683594, + "mean_logprob": -10.174077987670898, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -50.63893127441406, + "mean_logprob": -10.127786254882812, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -42.67822265625, + "mean_logprob": -8.53564453125, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.172816276550293, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-024", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Because the instructions were ambiguous, two technicians interpreted them", + "continuations": [ + " with identical ambiguity only.", + " before they were written.", + " into a screwdriver.", + " differently." + ], + "tags": [ + "semantics" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -87.57284545898438, + "mean_logprob": -9.730316162109375, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -45.40427017211914, + "mean_logprob": -9.080854034423828, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -76.99366760253906, + "mean_logprob": -9.624208450317383, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -25.359052658081055, + "mean_logprob": -8.453017552693685, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.6278364817301423, + "context_truncated": false, + "elapsed_seconds": 0.0006502225005533546 + }, + { + "id": "language_completion-025", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Sofia lowered her voice so that she would not", + "continuations": [ + " wake the baby.", + " sleeping the quiet.", + " lower the voice's color.", + " have loudly whisper." + ], + "tags": [ + "purpose" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -54.539833068847656, + "mean_logprob": -10.907966613769531, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.692527770996094, + "mean_logprob": -10.938505554199219, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -67.81717681884766, + "mean_logprob": -11.302862803141275, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -79.29618835449219, + "mean_logprob": -9.912023544311523, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.9959430694580078, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-026", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The first experiment failed; the second, by contrast,", + "continuations": [ + " was experiment first.", + " produced reliable results.", + " contrasted before being second.", + " failing reliably." + ], + "tags": [ + "discourse" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -37.19580078125, + "mean_logprob": -9.2989501953125, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -41.411441802978516, + "mean_logprob": -10.352860450744629, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -65.63911437988281, + "mean_logprob": -10.939852396647135, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -66.90092468261719, + "mean_logprob": -11.150154113769531, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.053910255432129, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-027", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Every applicant must attach a resume, regardless of", + "continuations": [ + " attaching is a verb.", + " resumes being paper.", + " prior experience.", + " every regardless." + ], + "tags": [ + "collocation" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -50.260047912597656, + "mean_logprob": -7.180006844656808, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -52.60321044921875, + "mean_logprob": -10.52064208984375, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.35788345336914, + "mean_logprob": -10.452627817789713, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.805084228515625, + "mean_logprob": -10.601694742838541, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 3.272620973132905, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-028", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The recipe calls for fresh basil; dried basil may be used", + "continuations": [ + " before fresh basil exists.", + " to call the recipe.", + " as a plate.", + " as a substitute." + ], + "tags": [ + "coherence" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -64.7353744506836, + "mean_logprob": -10.789229075113932, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -53.53578186035156, + "mean_logprob": -8.922630310058594, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -35.373722076416016, + "mean_logprob": -8.843430519104004, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -48.321815490722656, + "mean_logprob": -9.66436309814453, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.07919979095458984, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-029", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Far from being discouraged by the criticism, Mei became", + "continuations": [ + " more determined.", + " criticism itself.", + " far from becoming.", + " determine yesterday." + ], + "tags": [ + "idiom" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.04005432128906, + "mean_logprob": -11.013351440429688, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -30.09972381591797, + "mean_logprob": -7.524930953979492, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.38550567626953, + "mean_logprob": -8.346376419067383, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -52.619781494140625, + "mean_logprob": -10.523956298828125, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.8214454650878906, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-030", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The contract is valid provided that both parties", + "continuations": [ + " had validity tomorrow.", + " sign it.", + " providing a party.", + " signs them." + ], + "tags": [ + "conditional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -66.72197723388672, + "mean_logprob": -9.531711033412389, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.726505279541016, + "mean_logprob": -7.242168426513672, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -29.143911361694336, + "mean_logprob": -7.285977840423584, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.248451232910156, + "mean_logprob": -10.749483744303385, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.04380941390991211, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-031", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "No sooner had the bell rung than the students", + "continuations": [ + " have ringing bells.", + " sooner the bell.", + " left the classroom.", + " had leave tomorrow." + ], + "tags": [ + "inversion" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -63.907142639160156, + "mean_logprob": -9.129591805594307, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -49.10709762573242, + "mean_logprob": -8.184516270955404, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.18304443359375, + "mean_logprob": -9.545761108398438, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -47.85120391845703, + "mean_logprob": -7.975200653076172, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.20931561787923236, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-032", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The evidence was incomplete. Consequently, the investigators", + "continuations": [ + " completed evidence by definition.", + " investigated before consequence.", + " were the evidence.", + " withheld judgment." + ], + "tags": [ + "discourse" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -45.82288360595703, + "mean_logprob": -9.164576721191406, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -55.77093505859375, + "mean_logprob": -9.295155843098959, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.967529296875, + "mean_logprob": -9.49188232421875, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -67.962158203125, + "mean_logprob": -11.3270263671875, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.13057912190755339, + "context_truncated": false, + "elapsed_seconds": 0.0006125451218395028 + }, + { + "id": "language_completion-033", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "Walking along the shore, I noticed", + "continuations": [ + " a trail of fresh footprints.", + " the shore walked me.", + " noticing had walked.", + " along was a footprint." + ], + "tags": [ + "modifier" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -102.52287292480469, + "mean_logprob": -11.391430324978298, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -58.065120697021484, + "mean_logprob": -8.29501724243164, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -71.87545776367188, + "mean_logprob": -11.979242960611979, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -52.431541442871094, + "mean_logprob": -8.738590240478516, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.443572998046875, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-034", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The orchestra rehearsed the difficult passage repeatedly until it sounded", + "continuations": [ + " passage by difficulty.", + " effortless.", + " rehearsing the sound.", + " repeatably orchestra." + ], + "tags": [ + "collocation" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.30592727661133, + "mean_logprob": -10.076481819152832, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.203590393066406, + "mean_logprob": -11.734530131022135, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -56.60176086425781, + "mean_logprob": -8.085965837751116, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -75.12060546875, + "mean_logprob": -9.39007568359375, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.3041098458426337, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-035", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "What the witness omitted from her statement proved", + "continuations": [ + " that omission witnessed.", + " proving had omitted.", + " more significant than what she included.", + " to include more omittedly." + ], + "tags": [ + "syntax" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -61.48310089111328, + "mean_logprob": -10.247183481852213, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -56.33897018432617, + "mean_logprob": -9.389828364054361, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -68.1934814453125, + "mean_logprob": -9.741925920758929, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -69.9925537109375, + "mean_logprob": -9.998936244419642, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.35209755670456744, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-036", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Were the assumptions underlying the estimate to change, the conclusion would", + "continuations": [ + " have assumptions beneath it.", + " changing was estimated.", + " underlie tomorrow.", + " need to be reconsidered." + ], + "tags": [ + "conditional" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -72.15560150146484, + "mean_logprob": -9.019450187683105, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.70579528808594, + "mean_logprob": -10.676448822021484, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -78.80721282958984, + "mean_logprob": -11.258173261369977, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -76.58058166503906, + "mean_logprob": -10.94008309500558, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.656998634338379, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-037", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The policy, while defensible in principle, is difficult to", + "continuations": [ + " implement consistently.", + " principle the defense.", + " difficult implementation was.", + " consistency defending." + ], + "tags": [ + "concession" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -31.71694564819336, + "mean_logprob": -7.92923641204834, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -47.2063102722168, + "mean_logprob": -11.8015775680542, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -49.735626220703125, + "mean_logprob": -12.433906555175781, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -58.07313919067383, + "mean_logprob": -11.614627838134766, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 3.685391426086426, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-038", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Only after the archive was digitized did researchers", + "continuations": [ + " digitized only before.", + " discover the missing correspondence.", + " archive the discovery did.", + " correspondingly research." + ], + "tags": [ + "inversion" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -46.22135925292969, + "mean_logprob": -7.703559875488281, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -54.098731994628906, + "mean_logprob": -9.01645533243815, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -50.014949798583984, + "mean_logprob": -8.335824966430664, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -33.95510482788086, + "mean_logprob": -8.488776206970215, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.6322650909423828, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-039", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The editor objected less to the claim itself than to", + "continuations": [ + " claims having objections.", + " itself being less.", + " the certainty with which it was stated.", + " stating an editor." + ], + "tags": [ + "comparison" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -37.70317077636719, + "mean_logprob": -7.5406341552734375, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -32.247314453125, + "mean_logprob": -8.06182861328125, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -72.02908325195312, + "mean_logprob": -8.003231472439236, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -47.069061279296875, + "mean_logprob": -9.413812255859375, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.4625973171657982, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-040", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "So subtle was the change in tone that few listeners", + "continuations": [ + " made tone into change.", + " subtly were few.", + " had listened tomorrow.", + " noticed it." + ], + "tags": [ + "inversion" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -43.827022552490234, + "mean_logprob": -7.304503758748372, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -52.070125579833984, + "mean_logprob": -10.414025115966798, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -68.51943969726562, + "mean_logprob": -9.788491385323661, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.30699920654297, + "mean_logprob": -10.576749801635742, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 2.4839876265752894, + "context_truncated": false, + "elapsed_seconds": 0.0006322957488009706 + }, + { + "id": "language_completion-041", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The proposal is neither as costly as its critics allege nor as simple as its supporters", + "continuations": [ + " suggest.", + " allegation was cost.", + " supporting simply.", + " critics and costly." + ], + "tags": [ + "parallelism" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -23.13853645324707, + "mean_logprob": -11.569268226623535, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -51.455650329589844, + "mean_logprob": -8.575941721598307, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -31.464736938476562, + "mean_logprob": -10.488245646158854, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -43.277828216552734, + "mean_logprob": -7.212971369425456, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.3629703521728507, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-042", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Whatever reservations the panel may have had, they were not sufficient to", + "continuations": [ + " reserve a panel seat.", + " block the appointment.", + " appoint sufficient reservations.", + " having been whatever." + ], + "tags": [ + "concession" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -51.362457275390625, + "mean_logprob": -8.560409545898438, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -46.775115966796875, + "mean_logprob": -9.355023193359376, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.68230438232422, + "mean_logprob": -9.936460876464844, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -27.971242904663086, + "mean_logprob": -6.9928107261657715, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.567598819732666, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-043", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Had the data been interpreted in isolation, the apparent trend might have", + "continuations": [ + " isolated its data.", + " been interpreting apparent.", + " seemed more convincing.", + " trended the isolation." + ], + "tags": [ + "counterfactual" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.11228942871094, + "mean_logprob": -8.778072357177734, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -48.530311584472656, + "mean_logprob": -9.706062316894531, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -58.11432647705078, + "mean_logprob": -9.685721079508463, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -63.78950119018555, + "mean_logprob": -10.631583531697592, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.9076487223307286, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-044", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The mayor's apology did little to allay concerns that the inquiry had been", + "continuations": [ + " mayoral and apologetic.", + " concerned into inquiry.", + " doing little apology.", + " deliberately delayed." + ], + "tags": [ + "collocation" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -70.36294555664062, + "mean_logprob": -10.051849365234375, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -48.391632080078125, + "mean_logprob": -8.065272013346354, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -47.15665054321289, + "mean_logprob": -9.431330108642578, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -55.59307861328125, + "mean_logprob": -9.265513102213541, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.2002410888671875, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-045", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Not until the final chapter does the significance of the opening scene become", + "continuations": [ + " fully apparent.", + " final before opening.", + " chaptered significantly.", + " opening does final." + ], + "tags": [ + "inversion" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -31.14529037475586, + "mean_logprob": -10.381763458251953, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -32.91607666015625, + "mean_logprob": -8.229019165039062, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -54.82319641113281, + "mean_logprob": -10.964639282226562, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.91718101501465, + "mean_logprob": -7.979295253753662, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.2497239112854004, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-046", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The theory's appeal lies not in its novelty but in its ability to", + "continuations": [ + " appeal theoretically.", + " unify previously separate observations.", + " become novel before theory.", + " observe its ability." + ], + "tags": [ + "semantics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -54.666263580322266, + "mean_logprob": -9.111043930053711, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -63.38331604003906, + "mean_logprob": -10.563886006673178, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -56.8101692199707, + "mean_logprob": -11.36203384399414, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.78034210205078, + "mean_logprob": -8.695085525512695, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.4159584045410156, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-047", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Much as the negotiators wanted an agreement, neither side was prepared to", + "continuations": [ + " negotiate wanting.", + " agreement much as.", + " concede the central point.", + " prepare neither side." + ], + "tags": [ + "concession" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -53.09514617919922, + "mean_logprob": -10.619029235839843, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.148902893066406, + "mean_logprob": -8.787225723266602, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -66.54666900634766, + "mean_logprob": -9.506667000906807, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -44.735206604003906, + "mean_logprob": -11.183801651000977, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.7194412776402057, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-048", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Whether the discrepancy reflects measurement error or a genuine anomaly remains", + "continuations": [ + " measuring a reflection.", + " discrepant in error.", + " anomalously genuine.", + " uncertain." + ], + "tags": [ + "syntax" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.56224822998047, + "mean_logprob": -9.712449645996093, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -46.73966979980469, + "mean_logprob": -7.789944966634114, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -62.741172790527344, + "mean_logprob": -7.842646598815918, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -26.168338775634766, + "mean_logprob": -13.084169387817383, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.05270163218180368, + "context_truncated": false, + "elapsed_seconds": 0.0006548527526319958 + }, + { + "id": "language_completion-049", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The ruling established a precedent whose consequences would not become clear", + "continuations": [ + " for several years.", + " precedent had ruled.", + " consequences establishing.", + " clearly whose." + ], + "tags": [ + "relative_clause" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -42.82978820800781, + "mean_logprob": -10.707447052001953, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -49.950584411621094, + "mean_logprob": -8.32509740193685, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.713008880615234, + "mean_logprob": -9.178252220153809, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.128490447998047, + "mean_logprob": -9.042830149332682, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.7177327473958321, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "language_completion-050", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "To attribute the recovery solely to the new policy would be to", + "continuations": [ + " recover sole attribution.", + " ignore several contributing factors.", + " policy the recovery solely.", + " contribute an attribution." + ], + "tags": [ + "idiom" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -50.74481201171875, + "mean_logprob": -10.14896240234375, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -89.41334533691406, + "mean_logprob": -12.77333504813058, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -60.76581954956055, + "mean_logprob": -10.127636591593424, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -52.54154968261719, + "mean_logprob": -10.508309936523437, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.021325810750326468, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-001", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A glass slipped from the table onto a tile floor. It most likely", + "continuations": [ + " grew roots.", + " floated upward.", + " shattered.", + " became warmer than fire." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -34.72678756713867, + "mean_logprob": -11.57559585571289, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.07689666748047, + "mean_logprob": -10.815379333496093, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.843753814697266, + "mean_logprob": -10.460938453674316, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -57.52307891845703, + "mean_logprob": -9.587179819742838, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.8737586339314785, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-002", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Jules wanted to mail a letter, so he put a stamp on the", + "continuations": [ + " refrigerator.", + " envelope.", + " sidewalk.", + " spoon." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -54.02965545654297, + "mean_logprob": -10.805931091308594, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.02235794067383, + "mean_logprob": -9.005589485168457, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.04995346069336, + "mean_logprob": -9.51248836517334, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.801136016845703, + "mean_logprob": -10.267045338948568, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.5068988800048828, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-003", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The room was dark until Ana flipped the wall switch and the", + "continuations": [ + " carpet vanished.", + " window froze.", + " lights came on.", + " ceiling became wet." + ], + "tags": [ + "causality" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.067256927490234, + "mean_logprob": -9.613451385498047, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.223369598388672, + "mean_logprob": -7.805842399597168, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.22154998779297, + "mean_logprob": -8.555387496948242, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -46.99520492553711, + "mean_logprob": -9.399040985107423, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.7495450973510742, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-004", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Before slicing a loaf of bread, most people reach for", + "continuations": [ + " a paintbrush.", + " a blanket.", + " a shoe.", + " a knife." + ], + "tags": [ + "tool_use" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -50.546077728271484, + "mean_logprob": -10.109215545654298, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -49.11615753173828, + "mean_logprob": -9.823231506347657, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.34988784790039, + "mean_logprob": -10.587471961975098, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.24095916748047, + "mean_logprob": -9.060239791870117, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.7629917144775398, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-005", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a houseplant's soil is very dry, the plant probably needs", + "continuations": [ + " water.", + " a password.", + " louder music.", + " a winter coat." + ], + "tags": [ + "everyday" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -20.761451721191406, + "mean_logprob": -10.380725860595703, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -48.91697311401367, + "mean_logprob": -12.229243278503418, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -66.47871398925781, + "mean_logprob": -13.295742797851563, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -54.496803283691406, + "mean_logprob": -10.899360656738281, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.5186347961425781, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-006", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Omar put leftovers in the refrigerator to keep them", + "continuations": [ + " invisible.", + " cold.", + " expensive.", + " rectangular." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -42.035160064697266, + "mean_logprob": -10.508790016174316, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.68337345123291, + "mean_logprob": -6.841686725616455, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.693801879882812, + "mean_logprob": -10.846900939941406, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.47222137451172, + "mean_logprob": -9.36805534362793, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 2.5263686180114746, + "context_truncated": false, + "elapsed_seconds": 0.0006180367490742356 + }, + { + "id": "commonsense-007", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Someone standing outside in heavy rain without shelter will probably get", + "continuations": [ + " taller.", + " younger.", + " wet.", + " magnetic." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -16.87139320373535, + "mean_logprob": -5.62379773457845, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.571264266967773, + "mean_logprob": -10.285632133483887, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.630020141601562, + "mean_logprob": -8.815010070800781, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -27.319490432739258, + "mean_logprob": -13.659745216369629, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 3.191212336222331, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-008", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A cyclist approaching a red traffic light should", + "continuations": [ + " accelerate through every lane.", + " close both eyes.", + " remove the wheels.", + " stop." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -81.89884948730469, + "mean_logprob": -11.699835641043526, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.57212829589844, + "mean_logprob": -10.14303207397461, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -58.98305130004883, + "mean_logprob": -11.796610260009766, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -19.01034927368164, + "mean_logprob": -9.50517463684082, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.6378574371337891, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-009", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "To find out whether someone has a fever, a nurse commonly uses", + "continuations": [ + " a thermometer.", + " a ruler.", + " a compass.", + " a stapler." + ], + "tags": [ + "tool_use" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -50.211910247802734, + "mean_logprob": -8.368651707967123, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -25.776649475097656, + "mean_logprob": -6.444162368774414, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.69879150390625, + "mean_logprob": -7.6746978759765625, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -46.597991943359375, + "mean_logprob": -7.7663319905598955, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.2305355072021484, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-010", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "When the phone battery reaches zero, the phone usually", + "continuations": [ + " doubles in size.", + " turns off.", + " prints a book.", + " becomes waterproof." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -46.29652404785156, + "mean_logprob": -9.259304809570313, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.826026916503906, + "mean_logprob": -6.275342305501302, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.19274139404297, + "mean_logprob": -9.838548278808593, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.8820915222168, + "mean_logprob": -10.17641830444336, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 2.9839625040690114, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-011", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A person who has not eaten all day is likely to feel", + "continuations": [ + " polished.", + " transparent.", + " hungry.", + " metallic." + ], + "tags": [ + "human_needs" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -37.078956604003906, + "mean_logprob": -12.359652201334635, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.9520378112793, + "mean_logprob": -12.650679270426432, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.77179718017578, + "mean_logprob": -9.192949295043945, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -48.986724853515625, + "mean_logprob": -12.246681213378906, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 3.053731918334961, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-012", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "After washing wet clothes, a common next step is to", + "continuations": [ + " bury them in concrete.", + " mail them overseas.", + " paint them black.", + " dry them." + ], + "tags": [ + "everyday" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -54.26698303222656, + "mean_logprob": -9.04449717203776, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -74.298828125, + "mean_logprob": -10.614118303571429, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.858856201171875, + "mean_logprob": -10.464714050292969, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.28878402709961, + "mean_logprob": -9.42959467569987, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.3850975036621094, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-013", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "If a candle flame is blown out, the room may become", + "continuations": [ + " darker.", + " heavier than a car.", + " filled with snow.", + " twice as wide." + ], + "tags": [ + "causality" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -30.654226303100586, + "mean_logprob": -10.218075434366861, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -60.60946273803711, + "mean_logprob": -10.101577123006185, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.87702178955078, + "mean_logprob": -10.469255447387695, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.74101638793945, + "mean_logprob": -10.185254096984863, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.0836769739786778, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-014", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "To unlock a typical front door from outside, a resident may use", + "continuations": [ + " soap.", + " a key.", + " a pillow.", + " a forkful of rice." + ], + "tags": [ + "tool_use" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.832035064697266, + "mean_logprob": -9.277345021565756, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.507837295532227, + "mean_logprob": -8.502612431844076, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -45.08024978637695, + "mean_logprob": -9.01604995727539, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -72.45986938476562, + "mean_logprob": -10.351409912109375, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.5134375254313142, + "context_truncated": false, + "elapsed_seconds": 0.0005877396251889877 + }, + { + "id": "commonsense-015", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Fresh snow on a sidewalk can make the surface", + "continuations": [ + " spicy.", + " musical.", + " slippery.", + " edible." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -31.20960807800293, + "mean_logprob": -10.403202692667643, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -30.206226348876953, + "mean_logprob": -15.103113174438477, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.75737762451172, + "mean_logprob": -11.18934440612793, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -35.690391540527344, + "mean_logprob": -11.896797180175781, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.786141713460287, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-016", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "If a smoke alarm sounds and smoke is visible, people should", + "continuations": [ + " finish a long nap.", + " hide the alarm.", + " open a novel.", + " leave the building safely." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -61.992767333984375, + "mean_logprob": -8.856109619140625, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -57.99372100830078, + "mean_logprob": -9.66562016805013, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.51604461669922, + "mean_logprob": -11.129011154174805, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -53.384437561035156, + "mean_logprob": -8.897406260172525, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.04129664103190045, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-017", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A ceramic mug filled with hot tea may be too hot to", + "continuations": [ + " hold comfortably.", + " remember a name.", + " become ceramic.", + " count backward." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.182830810546875, + "mean_logprob": -9.045707702636719, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -45.322296142578125, + "mean_logprob": -11.330574035644531, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -44.28685760498047, + "mean_logprob": -8.857371520996093, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -41.87733840942383, + "mean_logprob": -10.469334602355957, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.1883361816406257, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-018", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Tara heard thunder only a few seconds after seeing lightning, suggesting the storm was", + "continuations": [ + " beneath the ocean.", + " nearby.", + " made of glass.", + " over last year." + ], + "tags": [ + "causality" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -64.37397766113281, + "mean_logprob": -10.728996276855469, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.656070709228516, + "mean_logprob": -8.828035354614258, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.638336181640625, + "mean_logprob": -9.409584045410156, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -27.173336029052734, + "mean_logprob": -6.793334007263184, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 2.034701347351074, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-019", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The freezer door was left open overnight. In the morning, the ice cream was probably", + "continuations": [ + " sharper.", + " newly packaged.", + " soft or melted.", + " louder." + ], + "tags": [ + "causality" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.03557586669922, + "mean_logprob": -9.011858622233072, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -44.705177307128906, + "mean_logprob": -11.176294326782227, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.96526336669922, + "mean_logprob": -8.193052673339844, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.96697998046875, + "mean_logprob": -9.241744995117188, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.8188059488932282, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-020", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Ravi arrived early for an interview but did not want to interrupt the meeting inside, so he", + "continuations": [ + " shouted through the door.", + " canceled every meeting.", + " moved the building.", + " waited quietly." + ], + "tags": [ + "social" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -48.8745002746582, + "mean_logprob": -6.982071467808315, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -63.00220489501953, + "mean_logprob": -10.500367482503256, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.32246398925781, + "mean_logprob": -9.080615997314453, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -44.42529296875, + "mean_logprob": -8.88505859375, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.9029871259416842, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-021", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A recipe serves four people, but eight guests are expected. A sensible cook would", + "continuations": [ + " double the ingredients.", + " halve every plate.", + " freeze the oven.", + " omit all food." + ], + "tags": [ + "planning" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.737693786621094, + "mean_logprob": -10.434423446655273, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -57.089744567871094, + "mean_logprob": -9.514957427978516, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -60.59453201293945, + "mean_logprob": -10.099088668823242, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -57.43223571777344, + "mean_logprob": -11.486447143554688, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.5841312408447266, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-022", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Mina's laptop became very hot and its fan ran constantly, so she first checked whether", + "continuations": [ + " the keyboard knew her name.", + " its vents were blocked.", + " the screen was square.", + " noon had arrived." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -58.126373291015625, + "mean_logprob": -8.303767613002233, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -50.94694519042969, + "mean_logprob": -7.278135027204241, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -49.06258010864258, + "mean_logprob": -9.812516021728516, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.51990509033203, + "mean_logprob": -10.103981018066406, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 1.0256325857979913, + "context_truncated": false, + "elapsed_seconds": 0.0006747061233909335 + }, + { + "id": "commonsense-023", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A friend says, \"I just lost my job.\" The most considerate immediate response is to", + "continuations": [ + " ask for a loan.", + " change the subject to sports.", + " express sympathy and listen.", + " congratulate them." + ], + "tags": [ + "social" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -55.9201545715332, + "mean_logprob": -9.320025761922201, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -70.57774353027344, + "mean_logprob": -11.762957255045572, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -69.07548522949219, + "mean_logprob": -9.867926461356026, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -64.53109741210938, + "mean_logprob": -10.755182902018229, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.5479006994338249, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-024", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The label says a medicine may cause drowsiness. After taking it, a person should avoid", + "continuations": [ + " drinking water.", + " reading the label.", + " sitting safely.", + " driving a car." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.007823944091797, + "mean_logprob": -8.002607981363932, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.23345184326172, + "mean_logprob": -8.80836296081543, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.051963806152344, + "mean_logprob": -9.012990951538086, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.835575103759766, + "mean_logprob": -7.958893775939941, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.04371420542399029, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-025", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Water is dripping from the ceiling directly below an upstairs bathroom. The leak likely", + "continuations": [ + " originates near that bathroom.", + " comes from sunlight.", + " proves the roof is dry.", + " makes water weightless." + ], + "tags": [ + "diagnosis" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -62.84894943237305, + "mean_logprob": -8.978421347481865, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.799449920654297, + "mean_logprob": -7.949862480163574, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -74.61406707763672, + "mean_logprob": -10.659152439662389, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.431907653808594, + "mean_logprob": -10.686381530761718, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 1.0285588673182904, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-026", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The car's fuel gauge is nearly empty and the next town is far away. The prudent action is to", + "continuations": [ + " increase tire size.", + " refuel before leaving.", + " turn off the steering wheel.", + " ignore all signs." + ], + "tags": [ + "planning" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -56.2535514831543, + "mean_logprob": -11.25071029663086, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -38.614654541015625, + "mean_logprob": -7.722930908203125, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -60.37819290161133, + "mean_logprob": -8.625456128801618, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.23211669921875, + "mean_logprob": -9.84642333984375, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.902525220598493, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-027", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A package marked \"fragile\" should generally be", + "continuations": [ + " shaken to test it.", + " left in rain.", + " handled carefully.", + " used as a chair." + ], + "tags": [ + "everyday" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -54.60768127441406, + "mean_logprob": -9.101280212402344, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -44.91283416748047, + "mean_logprob": -11.228208541870117, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -41.40422821044922, + "mean_logprob": -10.351057052612305, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -43.03417205810547, + "mean_logprob": -7.172362009684245, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.9289182027180987, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-028", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Kai cannot hear the television, but everyone else can. Before replacing it, Kai should check", + "continuations": [ + " whether the walls are blue.", + " how much the sofa weighs.", + " if tomorrow is Tuesday.", + " the volume and audio settings." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -66.21952819824219, + "mean_logprob": -11.036588033040365, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -82.47686004638672, + "mean_logprob": -8.247686004638672, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -87.34583282470703, + "mean_logprob": -9.70509253607856, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -68.52349853515625, + "mean_logprob": -9.789071219308036, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.4574065314398883, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-029", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A wet floor sign in a hallway warns pedestrians that they should", + "continuations": [ + " walk carefully.", + " run faster.", + " turn off every light.", + " remove their shoes immediately." + ], + "tags": [ + "safety" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -32.235374450683594, + "mean_logprob": -10.745124816894531, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.793033599853516, + "mean_logprob": -9.931011199951172, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -50.862831115722656, + "mean_logprob": -10.172566223144532, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -73.60811614990234, + "mean_logprob": -12.268019358317057, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.24155502319336009, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-030", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Nico promised to return a borrowed book on Friday but needs it until Monday. The responsible choice is to", + "continuations": [ + " hide the book.", + " ask the owner for an extension.", + " claim Friday was canceled.", + " lend it to a stranger." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -45.25054931640625, + "mean_logprob": -9.05010986328125, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -67.60839080810547, + "mean_logprob": -8.451048851013184, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -86.21526336669922, + "mean_logprob": -9.579473707411024, + "total_sequence_tokens": 37 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -60.31355667114258, + "mean_logprob": -7.539194583892822, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.9118542671203613, + "context_truncated": false, + "elapsed_seconds": 0.0006418982520699501 + }, + { + "id": "commonsense-031", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A refrigerator is running, but food inside is warm. One useful first check is whether", + "continuations": [ + " the food can read.", + " the kitchen is rectangular.", + " the door seals completely.", + " the shelves are named." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -32.8817138671875, + "mean_logprob": -6.5763427734375, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -61.198848724365234, + "mean_logprob": -7.649856090545654, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -61.75665283203125, + "mean_logprob": -10.292775472005209, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -52.71350860595703, + "mean_logprob": -8.785584767659506, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.0735133171081541, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-032", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A hiker sees dark clouds approaching and has several hours left on an exposed ridge. The safer choice is to", + "continuations": [ + " climb the tallest tree.", + " hold a metal pole.", + " wait for lightning.", + " descend toward shelter." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -67.6370849609375, + "mean_logprob": -9.662440708705358, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -64.07723999023438, + "mean_logprob": -10.679539998372396, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -32.11351013183594, + "mean_logprob": -6.422702026367188, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -65.48056030273438, + "mean_logprob": -10.913426717122396, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 3.23973868233817, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-033", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "An egg dropped onto a thick cushion is less likely to break because the cushion", + "continuations": [ + " reduces the sudden impact.", + " makes the egg colder.", + " removes gravity.", + " turns shell into rubber." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -46.91685485839844, + "mean_logprob": -9.383370971679687, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -51.73536682128906, + "mean_logprob": -8.62256113688151, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -63.899375915527344, + "mean_logprob": -12.779875183105469, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -60.1533203125, + "mean_logprob": -10.025553385416666, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.7608098347981773, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-034", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A metal jar lid is stuck. Running warm water over the lid can help because the metal may", + "continuations": [ + " lose all mass.", + " expand slightly.", + " stop being metal.", + " become colder than ice." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -30.374399185180664, + "mean_logprob": -7.593599796295166, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -30.879261016845703, + "mean_logprob": -10.293087005615234, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.64453887939453, + "mean_logprob": -9.411134719848633, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -68.78175354003906, + "mean_logprob": -11.46362559000651, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.8175349235534668, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-035", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Lena receives an email demanding immediate payment through gift cards and threatening arrest. The message is most likely", + "continuations": [ + " a routine tax receipt.", + " a weather alert.", + " a scam.", + " a library reminder." + ], + "tags": [ + "digital_safety" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -65.74859619140625, + "mean_logprob": -9.392656598772321, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -41.80462646484375, + "mean_logprob": -8.36092529296875, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.656436920166016, + "mean_logprob": -11.664109230041504, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.85015106201172, + "mean_logprob": -9.970030212402344, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.0317313058035715, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-036", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A sealed plastic bottle dents inward after hot air inside cools. The best explanation is that", + "continuations": [ + " the bottle gained extra air.", + " cooling erased the plastic.", + " gravity reversed briefly.", + " pressure inside decreased." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -76.91275787353516, + "mean_logprob": -10.987536839076451, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -65.80667877197266, + "mean_logprob": -10.967779795328775, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -86.39522552490234, + "mean_logprob": -10.799403190612793, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.360084533691406, + "mean_logprob": -9.340021133422852, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.4593820571899414, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-037", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A colleague repeatedly misses deadlines because requirements change without notice. The most constructive first step is to", + "continuations": [ + " document changes and clarify priorities.", + " publicly insult the colleague.", + " silently redo all work forever.", + " delete the project." + ], + "tags": [ + "workplace" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -69.97596740722656, + "mean_logprob": -8.74699592590332, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -82.23057556152344, + "mean_logprob": -10.27882194519043, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -99.58992767333984, + "mean_logprob": -11.065547519259983, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.98529815673828, + "mean_logprob": -10.197059631347656, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.450063705444336, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-038", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A carbon monoxide alarm sounds, but nobody can smell smoke. Occupants should still", + "continuations": [ + " assume silence means safety.", + " leave and contact emergency services.", + " search with a flame.", + " unplug it and sleep." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -53.699005126953125, + "mean_logprob": -8.949834187825521, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -65.96089172363281, + "mean_logprob": -10.993481953938803, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -50.97273254394531, + "mean_logprob": -8.495455423990885, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -68.3896484375, + "mean_logprob": -9.769949776785714, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.4543787638346366, + "context_truncated": false, + "elapsed_seconds": 0.0007146217503759544 + }, + { + "id": "commonsense-039", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A wooden drawer sticks only during humid weather. The most plausible reason is that the wood", + "continuations": [ + " becomes electrically charged.", + " forgets its shape.", + " swells after absorbing moisture.", + " turns into metal." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -63.998260498046875, + "mean_logprob": -12.799652099609375, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -45.84546661376953, + "mean_logprob": -9.169093322753906, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -72.93502807617188, + "mean_logprob": -9.116878509521484, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.932228088378906, + "mean_logprob": -10.733057022094727, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.052214813232421164, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-040", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "Sam hears a rumor about a coworker from someone who admits they were not present. Before repeating it, Sam should", + "continuations": [ + " add dramatic details.", + " post it anonymously.", + " treat it as confirmed.", + " verify the information." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.66705322265625, + "mean_logprob": -9.916763305664062, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -53.412288665771484, + "mean_logprob": -8.902048110961914, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -45.00975036621094, + "mean_logprob": -9.001950073242188, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -43.79171371459961, + "mean_logprob": -8.758342742919922, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.1437053680419922, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-041", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Two identical wet towels are hung up, one spread flat and one crumpled. The spread towel usually dries first because it has", + "continuations": [ + " more surface exposed to air.", + " less fabric.", + " no water initially.", + " a different mass forever." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -47.500057220458984, + "mean_logprob": -7.916676203409831, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.808265686035156, + "mean_logprob": -9.269421895345053, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.416290283203125, + "mean_logprob": -9.354072570800781, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -47.43267822265625, + "mean_logprob": -7.905446370442708, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.011229832967122988, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-042", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A meeting invite lists 3 p.m. in another time zone. To avoid arriving at the wrong time, a participant should", + "continuations": [ + " ignore the time zone.", + " convert it to local time.", + " attend at every hour.", + " change the calendar date randomly." + ], + "tags": [ + "planning" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -50.476524353027344, + "mean_logprob": -8.41275405883789, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -65.14743041992188, + "mean_logprob": -9.306775774274554, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.32004165649414, + "mean_logprob": -8.064008331298828, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -72.06877899169922, + "mean_logprob": -8.007642110188803, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.056366221110025094, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-043", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A pan of oil catches fire on a stove. The cook should turn off the heat and", + "continuations": [ + " pour water into the oil.", + " carry the burning pan outside.", + " cover the pan with a lid.", + " fan the flames." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -52.9549560546875, + "mean_logprob": -7.564993722098214, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -69.20196533203125, + "mean_logprob": -11.533660888671875, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -61.35211181640625, + "mean_logprob": -7.669013977050781, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -28.139785766601562, + "mean_logprob": -4.689964294433594, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 2.8750294276646207, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-044", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "After a software update, a printer stops appearing on one computer but works from others. The issue is most likely with", + "continuations": [ + " all paper ever made.", + " the building's plumbing.", + " the printer's physical existence.", + " that computer's configuration." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.783538818359375, + "mean_logprob": -10.356707763671874, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -55.94538497924805, + "mean_logprob": -7.9921978541782925, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -63.57121276855469, + "mean_logprob": -9.081601824079241, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -49.59595489501953, + "mean_logprob": -8.265992482503256, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.27379462832496326, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-045", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A person feels dizzy after standing up very quickly. Sitting back down helps primarily by", + "continuations": [ + " reducing the immediate risk of falling.", + " making the floor warmer.", + " stopping all circulation.", + " increasing their height." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -72.52623748779297, + "mean_logprob": -10.36089106968471, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -54.45806121826172, + "mean_logprob": -9.076343536376953, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -45.63667297363281, + "mean_logprob": -7.606112162272136, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.89968490600586, + "mean_logprob": -10.724921226501465, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.4702313741048174, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-046", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A sealed bag of chips puffs up during a drive to high altitude because outside air pressure", + "continuations": [ + " becomes much higher.", + " becomes lower.", + " changes the chips into gas.", + " disappears completely." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -38.879554748535156, + "mean_logprob": -9.719888687133789, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.16160202026367, + "mean_logprob": -11.387200673421225, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -64.91744995117188, + "mean_logprob": -9.273921421595983, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -57.85066604614258, + "mean_logprob": -14.462666511535645, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.4459672655378064, + "context_truncated": false, + "elapsed_seconds": 0.0006456185001297854 + }, + { + "id": "commonsense-047", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A manager asks for feedback but becomes defensive whenever criticism is offered. Employees will likely become", + "continuations": [ + " more candid each time.", + " unable to remember work.", + " reluctant to speak honestly.", + " certain all criticism vanished." + ], + "tags": [ + "social" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -43.501869201660156, + "mean_logprob": -8.70037384033203, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -49.85992431640625, + "mean_logprob": -9.97198486328125, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -110.88764190673828, + "mean_logprob": -11.088764190673828, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -65.48811340332031, + "mean_logprob": -9.355444771902901, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.6550709315708705, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "commonsense-048", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A parked car has one window fogged on the inside while the others are clear. This suggests moisture is concentrated", + "continuations": [ + " inside the engine only.", + " on the road beneath.", + " in the sunlight.", + " near that window." + ], + "tags": [ + "diagnosis" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -40.51490020751953, + "mean_logprob": -8.102980041503907, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -71.45691680908203, + "mean_logprob": -10.208130972726005, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.87205505371094, + "mean_logprob": -8.218013763427734, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.009666442871094, + "mean_logprob": -9.502416610717773, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.11503372192382777, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "commonsense-049", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A child apologizes only after being caught and immediately repeats the behavior. The apology alone is weak evidence of", + "continuations": [ + " genuine change.", + " having been caught.", + " the behavior occurring.", + " spoken language." + ], + "tags": [ + "social" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -49.4476318359375, + "mean_logprob": -9.8895263671875, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -21.906917572021484, + "mean_logprob": -5.476729393005371, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.92167663574219, + "mean_logprob": -9.230419158935547, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.97049331665039, + "mean_logprob": -8.242623329162598, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 2.7658939361572266, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "commonsense-050", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A rechargeable battery lasts much less time in freezing weather but recovers indoors. The temporary loss is most plausibly due to", + "continuations": [ + " permanent deletion of electricity.", + " slower chemical reactions in the cold.", + " the device becoming heavier.", + " indoor clocks running faster." + ], + "tags": [ + "physical" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -63.75188064575195, + "mean_logprob": -9.107411520821708, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -71.94821166992188, + "mean_logprob": -8.993526458740234, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -60.23455810546875, + "mean_logprob": -10.039093017578125, + "total_sequence_tokens": 39 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -57.46464920043945, + "mean_logprob": -8.20923560006278, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.7842908586774548, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "world_knowledge-001", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The capital city of France is", + "continuations": [ + " Vienna.", + " Madrid.", + " Paris.", + " Lisbon." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.01634216308594, + "mean_logprob": -10.754085540771484, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.30549240112305, + "mean_logprob": -11.768497467041016, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -22.04580307006836, + "mean_logprob": -11.02290153503418, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -48.82149887084961, + "mean_logprob": -12.205374717712402, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.2688159942626953, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "world_knowledge-002", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The planet known for its prominent rings is", + "continuations": [ + " Mercury.", + " Saturn.", + " Earth.", + " Mars." + ], + "tags": [ + "science" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -31.45173454284668, + "mean_logprob": -10.483911514282227, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.19021987915039, + "mean_logprob": -11.396739959716797, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.02161407470703, + "mean_logprob": -10.510807037353516, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -20.467140197753906, + "mean_logprob": -10.233570098876953, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.25034141540527344, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "world_knowledge-003", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Water freezes at zero degrees on the", + "continuations": [ + " Kelvin scale.", + " Fahrenheit scale.", + " Celsius scale.", + " Beaufort scale." + ], + "tags": [ + "science" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -46.31181716918945, + "mean_logprob": -9.26236343383789, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -81.42158508300781, + "mean_logprob": -11.631655011858259, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -63.840511322021484, + "mean_logprob": -12.768102264404297, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -45.31562042236328, + "mean_logprob": -9.063124084472657, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.19923934936523402, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "world_knowledge-004", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The author of the play Hamlet was", + "continuations": [ + " Charles Dickens.", + " Jane Austen.", + " Mark Twain.", + " William Shakespeare." + ], + "tags": [ + "literature" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.6100959777832, + "mean_logprob": -10.32201919555664, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -67.89935302734375, + "mean_logprob": -13.57987060546875, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.561960220336914, + "mean_logprob": -7.6404900550842285, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -45.11360168457031, + "mean_logprob": -7.518933614095052, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.12155644098917673, + "context_truncated": false, + "elapsed_seconds": 0.000664144623442553 + }, + { + "id": "world_knowledge-005", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The largest ocean on Earth is the", + "continuations": [ + " Pacific Ocean.", + " Indian Ocean.", + " Arctic Ocean.", + " Atlantic Ocean." + ], + "tags": [ + "geography" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.44324493408203, + "mean_logprob": -9.814414978027344, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -31.002723693847656, + "mean_logprob": -10.334241231282553, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.252729415893555, + "mean_logprob": -9.750909805297852, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.6279182434082, + "mean_logprob": -11.209306081136068, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.06350517272949219, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-006", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Plants primarily absorb water through their", + "continuations": [ + " flowers.", + " roots.", + " seeds.", + " fruit." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -27.795772552490234, + "mean_logprob": -13.897886276245117, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.860668182373047, + "mean_logprob": -9.430334091186523, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -23.782901763916016, + "mean_logprob": -11.891450881958008, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -23.214828491210938, + "mean_logprob": -11.607414245605469, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 2.1770801544189453, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-007", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The chemical symbol for oxygen is", + "continuations": [ + " Ox.", + " G.", + " O.", + " Og." + ], + "tags": [ + "chemistry" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -27.157772064208984, + "mean_logprob": -13.578886032104492, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.5664005279541, + "mean_logprob": -8.78320026397705, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.714336395263672, + "mean_logprob": -10.857168197631836, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -29.51443862915039, + "mean_logprob": -9.838146209716797, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.054945945739746, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-008", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The Nile River flows through the continent of", + "continuations": [ + " South America.", + " Australia.", + " Europe.", + " Africa." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.43445587158203, + "mean_logprob": -9.811485290527344, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -24.209732055664062, + "mean_logprob": -12.104866027832031, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -23.761642456054688, + "mean_logprob": -11.880821228027344, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -29.266948699951172, + "mean_logprob": -14.633474349975586, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 2.0693359375, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-009", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A triangle has", + "continuations": [ + " three sides.", + " four sides.", + " five sides.", + " six sides." + ], + "tags": [ + "mathematics" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -22.808979034423828, + "mean_logprob": -7.602993011474609, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.087766647338867, + "mean_logprob": -7.029255549112956, + "total_sequence_tokens": 7 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.66215705871582, + "mean_logprob": -8.887385686238607, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.076953887939453, + "mean_logprob": -6.025651295979817, + "total_sequence_tokens": 7 + } + ], + "best_vs_second_margin": 1.0036042531331386, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-010", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The organ that pumps blood through the human body is the", + "continuations": [ + " liver.", + " heart.", + " lung.", + " stomach." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -25.722064971923828, + "mean_logprob": -12.861032485961914, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.178516387939453, + "mean_logprob": -5.589258193969727, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.056636810302734, + "mean_logprob": -9.028318405151367, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -23.532001495361328, + "mean_logprob": -11.766000747680664, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 3.4390602111816406, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-011", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Japan is an island country in", + "continuations": [ + " South America.", + " western Africa.", + " East Asia.", + " northern Europe." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.606637954711914, + "mean_logprob": -8.53554598490397, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -39.72673797607422, + "mean_logprob": -13.24224599202474, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.66391372680664, + "mean_logprob": -8.554637908935547, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -35.04963684082031, + "mean_logprob": -11.683212280273438, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.019091924031576113, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-012", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The first month of the Gregorian calendar is", + "continuations": [ + " March.", + " June.", + " December.", + " January." + ], + "tags": [ + "calendar" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -18.689502716064453, + "mean_logprob": -9.344751358032227, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.84135627746582, + "mean_logprob": -8.92067813873291, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -20.464502334594727, + "mean_logprob": -10.232251167297363, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -23.336570739746094, + "mean_logprob": -11.668285369873047, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.4240732192993164, + "context_truncated": false, + "elapsed_seconds": 0.0005931972482358105 + }, + { + "id": "world_knowledge-013", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Bees commonly produce", + "continuations": [ + " honey.", + " salt.", + " wool.", + " paper." + ], + "tags": [ + "biology" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -26.15863037109375, + "mean_logprob": -13.079315185546875, + "total_sequence_tokens": 6 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -26.998363494873047, + "mean_logprob": -13.499181747436523, + "total_sequence_tokens": 6 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -37.27229309082031, + "mean_logprob": -12.424097696940104, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -23.649517059326172, + "mean_logprob": -11.824758529663086, + "total_sequence_tokens": 6 + } + ], + "best_vs_second_margin": 0.5993391672770176, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-014", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The Great Pyramid of Giza is located in", + "continuations": [ + " Peru.", + " Egypt.", + " India.", + " Greece." + ], + "tags": [ + "history" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.76852798461914, + "mean_logprob": -9.256175994873047, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -15.508020401000977, + "mean_logprob": -7.754010200500488, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -14.8720703125, + "mean_logprob": -7.43603515625, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -22.501968383789062, + "mean_logprob": -7.5006561279296875, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.0646209716796875, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-015", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The gas humans need to breathe for cellular respiration is", + "continuations": [ + " helium.", + " neon.", + " oxygen.", + " argon." + ], + "tags": [ + "biology" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -35.49334716796875, + "mean_logprob": -11.83111572265625, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.2120246887207, + "mean_logprob": -11.070674896240234, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.295242309570312, + "mean_logprob": -10.647621154785156, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.172832489013672, + "mean_logprob": -9.390944163004557, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.2566769917805996, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-016", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "A standard chessboard contains", + "continuations": [ + " forty squares.", + " fifty squares.", + " one hundred squares.", + " sixty-four squares." + ], + "tags": [ + "culture" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -43.63463592529297, + "mean_logprob": -8.726927185058594, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -59.85282516479492, + "mean_logprob": -11.970565032958984, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -46.47478103637695, + "mean_logprob": -9.29495620727539, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -99.9010009765625, + "mean_logprob": -11.100111219618055, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.5680290222167965, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-017", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The treaty that formally ended World War I between Germany and the Allied powers was the Treaty of", + "continuations": [ + " Versailles.", + " Utrecht.", + " Tordesillas.", + " Ghent." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -48.27860641479492, + "mean_logprob": -8.04643440246582, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.70013427734375, + "mean_logprob": -8.74002685546875, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -59.84514617919922, + "mean_logprob": -9.974191029866537, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -34.5246467590332, + "mean_logprob": -11.5082155863444, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.69359245300293, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-018", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The process by which plants convert light energy into chemical energy is", + "continuations": [ + " fermentation.", + " photosynthesis.", + " respiration only.", + " erosion." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.741458892822266, + "mean_logprob": -8.685364723205566, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -53.31855773925781, + "mean_logprob": -10.663711547851562, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -47.24535369873047, + "mean_logprob": -11.811338424682617, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -35.43144989013672, + "mean_logprob": -11.81048329671224, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.978346824645996, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-019", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The longest river entirely within Europe is the", + "continuations": [ + " Danube.", + " Rhine.", + " Volga.", + " Seine." + ], + "tags": [ + "geography" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -44.05580520629883, + "mean_logprob": -14.68526840209961, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -36.440452575683594, + "mean_logprob": -12.146817525227865, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -44.636383056640625, + "mean_logprob": -11.159095764160156, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.14091110229492, + "mean_logprob": -10.713637034098307, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.44545873006184955, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-020", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "In computing, RAM is primarily used for", + "continuations": [ + " permanent archival storage.", + " printing documents.", + " cooling the processor.", + " temporary working memory." + ], + "tags": [ + "computing" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -58.095008850097656, + "mean_logprob": -11.619001770019532, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -39.093021392822266, + "mean_logprob": -13.031007130940756, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -46.97308349609375, + "mean_logprob": -9.39461669921875, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.09212112426758, + "mean_logprob": -9.023030281066895, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.3715864181518551, + "context_truncated": false, + "elapsed_seconds": 0.0005891238724871073 + }, + { + "id": "world_knowledge-021", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The scientist who formulated the three laws of motion was", + "continuations": [ + " Isaac Newton.", + " Louis Pasteur.", + " Gregor Mendel.", + " Niels Bohr." + ], + "tags": [ + "science_history" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -68.96280670166016, + "mean_logprob": -11.49380111694336, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -44.76726150512695, + "mean_logprob": -8.95345230102539, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -70.30673217773438, + "mean_logprob": -10.043818882533483, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -74.8038558959961, + "mean_logprob": -10.686265127999443, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.0903665815080927, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-022", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The strait separating Spain from Morocco is the Strait of", + "continuations": [ + " Malacca.", + " Gibraltar.", + " Hormuz.", + " Bering." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.643043518066406, + "mean_logprob": -8.910760879516602, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -62.95368194580078, + "mean_logprob": -10.49228032430013, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -54.475440979003906, + "mean_logprob": -10.895088195800781, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.46063995361328, + "mean_logprob": -11.153546651204428, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.5815194447835292, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-023", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A word with the opposite meaning of another word is called an", + "continuations": [ + " acronym.", + " homophone.", + " antonym.", + " anagram." + ], + "tags": [ + "language" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.504695892333984, + "mean_logprob": -8.376173973083496, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.53236389160156, + "mean_logprob": -8.38309097290039, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.15300750732422, + "mean_logprob": -9.038251876831055, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.126874923706055, + "mean_logprob": -9.375624974568685, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.006916999816894531, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-024", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The part of a cell that contains most of its genetic material is the", + "continuations": [ + " ribosome.", + " cell wall.", + " vacuole.", + " nucleus." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.20899200439453, + "mean_logprob": -8.241798400878906, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -32.978702545166016, + "mean_logprob": -10.992900848388672, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.57279586791992, + "mean_logprob": -9.14319896697998, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -24.983489990234375, + "mean_logprob": -8.327829996744791, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.08603159586588482, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-025", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The Renaissance began as a major cultural movement in", + "continuations": [ + " Italy.", + " Iceland.", + " Canada.", + " Mongolia." + ], + "tags": [ + "history" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -21.837236404418945, + "mean_logprob": -10.918618202209473, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.06013488769531, + "mean_logprob": -10.515033721923828, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -20.422765731811523, + "mean_logprob": -10.211382865905762, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -47.263755798339844, + "mean_logprob": -9.452751159667969, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.758631706237793, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-026", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The SI unit of electrical resistance is the", + "continuations": [ + " watt.", + " ohm.", + " tesla.", + " joule." + ], + "tags": [ + "physics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.72850227355957, + "mean_logprob": -8.242834091186523, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -29.15107536315918, + "mean_logprob": -7.287768840789795, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -55.54030990600586, + "mean_logprob": -13.885077476501465, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.9012451171875, + "mean_logprob": -9.225311279296875, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.9550652503967285, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-027", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The novel Pride and Prejudice was written by", + "continuations": [ + " George Eliot.", + " Virginia Woolf.", + " Jane Austen.", + " Mary Shelley." + ], + "tags": [ + "literature" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -45.40501022338867, + "mean_logprob": -11.351252555847168, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -61.279937744140625, + "mean_logprob": -12.255987548828125, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -59.831077575683594, + "mean_logprob": -11.966215515136719, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.00796890258789, + "mean_logprob": -10.601593780517579, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.7496587753295891, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-028", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The layer of Earth's atmosphere containing most weather is the", + "continuations": [ + " mesosphere.", + " thermosphere.", + " exosphere.", + " troposphere." + ], + "tags": [ + "earth_science" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.2847900390625, + "mean_logprob": -9.6569580078125, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.19654083251953, + "mean_logprob": -8.299135208129883, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -56.16521453857422, + "mean_logprob": -14.041303634643555, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -54.059696197509766, + "mean_logprob": -10.811939239501953, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.3578227996826175, + "context_truncated": false, + "elapsed_seconds": 0.0006416614996851422 + }, + { + "id": "world_knowledge-029", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The Magna Carta was sealed in", + "continuations": [ + " England.", + " Brazil.", + " China.", + " Ethiopia." + ], + "tags": [ + "history" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -21.760757446289062, + "mean_logprob": -10.880378723144531, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -29.811912536621094, + "mean_logprob": -14.905956268310547, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -20.35849952697754, + "mean_logprob": -10.17924976348877, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -50.95463180541992, + "mean_logprob": -12.73865795135498, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.7011289596557617, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-030", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "In music, a group of three notes played together is commonly called a", + "continuations": [ + " scale.", + " chord.", + " tempo.", + " clef." + ], + "tags": [ + "music" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.917049407958984, + "mean_logprob": -7.958524703979492, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.868885040283203, + "mean_logprob": -9.289628346761068, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -35.710601806640625, + "mean_logprob": -8.927650451660156, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.06455993652344, + "mean_logprob": -10.688186645507812, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.9691257476806641, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-031", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The blood vessels that generally carry blood away from the heart are", + "continuations": [ + " capillaries only.", + " lymph nodes.", + " arteries.", + " alveoli." + ], + "tags": [ + "biology" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -55.068660736083984, + "mean_logprob": -11.013732147216796, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -54.90061950683594, + "mean_logprob": -13.725154876708984, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.76297378540039, + "mean_logprob": -8.92099126180013, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.80575180053711, + "mean_logprob": -9.451437950134277, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.5304466883341465, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-032", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The decimal number ten is written in binary as", + "continuations": [ + " 1001.", + " 1111.", + " 1100.", + " 1010." + ], + "tags": [ + "computing" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -31.321372985839844, + "mean_logprob": -5.220228830973308, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -26.493274688720703, + "mean_logprob": -4.41554578145345, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -27.592273712158203, + "mean_logprob": -4.5987122853597, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -29.814868927001953, + "mean_logprob": -4.969144821166992, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.18316650390625, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-033", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The mountain range commonly used as part of the boundary between Europe and Asia is the", + "continuations": [ + " Ural Mountains.", + " Andes.", + " Alps.", + " Atlas Mountains." + ], + "tags": [ + "geography" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -58.21821594238281, + "mean_logprob": -11.643643188476563, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -37.91998291015625, + "mean_logprob": -12.639994303385416, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.33523178100586, + "mean_logprob": -9.77841059366862, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.442955017089844, + "mean_logprob": -10.688591003417969, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.9101804097493496, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-034", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The principle that no two electrons in an atom can share all four quantum numbers is the", + "continuations": [ + " Doppler effect.", + " Pauli exclusion principle.", + " uncertainty of measurement.", + " Hubble law." + ], + "tags": [ + "physics" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -52.825408935546875, + "mean_logprob": -8.804234822591146, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -59.76504898071289, + "mean_logprob": -9.960841496785482, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -43.31327438354492, + "mean_logprob": -8.662654876708984, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -61.38538360595703, + "mean_logprob": -12.277076721191406, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.14157994588216205, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-035", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The 1648 agreements often associated with the modern state system are known as the Peace of", + "continuations": [ + " Augsburg.", + " Paris.", + " Westphalia.", + " Nijmegen." + ], + "tags": [ + "history" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.8560791015625, + "mean_logprob": -8.964019775390625, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.916175842285156, + "mean_logprob": -10.458087921142578, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -41.40983963012695, + "mean_logprob": -8.281967926025391, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.64494705200195, + "mean_logprob": -10.728989410400391, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.6820518493652337, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-036", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In linguistics, the smallest unit that carries meaning is a", + "continuations": [ + " phoneme in every case.", + " clause.", + " diphthong.", + " morpheme." + ], + "tags": [ + "linguistics" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -53.98424530029297, + "mean_logprob": -6.748030662536621, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.58818817138672, + "mean_logprob": -8.529396057128906, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -57.61259460449219, + "mean_logprob": -9.602099100748697, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -40.42994689941406, + "mean_logprob": -8.085989379882813, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.3379587173461918, + "context_truncated": false, + "elapsed_seconds": 0.0006063852488296106 + }, + { + "id": "world_knowledge-037", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The enzyme that unwinds the DNA double helix during replication is", + "continuations": [ + " helicase.", + " amylase.", + " lipase.", + " pepsin." + ], + "tags": [ + "biology" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.00115966796875, + "mean_logprob": -10.250289916992188, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -44.74166488647461, + "mean_logprob": -11.185416221618652, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.795265197753906, + "mean_logprob": -9.265088399251303, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.39374542236328, + "mean_logprob": -10.078749084472657, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.8136606852213539, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-038", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The economic concept describing the value of the best forgone alternative is", + "continuations": [ + " sunk cost.", + " opportunity cost.", + " marginal revenue.", + " price elasticity." + ], + "tags": [ + "economics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.13725280761719, + "mean_logprob": -10.784313201904297, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.737163543701172, + "mean_logprob": -9.91238784790039, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -59.03423309326172, + "mean_logprob": -9.839038848876953, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -47.606361389160156, + "mean_logprob": -9.521272277832031, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.31776657104492223, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-039", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The literary device in which an absent or imaginary person is directly addressed is", + "continuations": [ + " metonymy.", + " synecdoche.", + " apostrophe.", + " enjambment." + ], + "tags": [ + "literature" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.52888488769531, + "mean_logprob": -10.132221221923828, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -69.90031433105469, + "mean_logprob": -11.650052388509115, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -43.75285339355469, + "mean_logprob": -8.750570678710938, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -60.78293228149414, + "mean_logprob": -12.156586456298829, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.3816505432128903, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-040", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The ocean current that carries warm water across the North Atlantic is commonly called the", + "continuations": [ + " Humboldt Current.", + " Benguela Current.", + " Oyashio Current.", + " Gulf Stream." + ], + "tags": [ + "earth_science" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -81.15669250488281, + "mean_logprob": -13.526115417480469, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -66.29739379882812, + "mean_logprob": -11.049565633138021, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -84.59500122070312, + "mean_logprob": -14.099166870117188, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -36.76251220703125, + "mean_logprob": -7.35250244140625, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 3.697063191731772, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-041", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alloy traditionally made primarily from copper and tin is", + "continuations": [ + " bronze.", + " brass.", + " steel.", + " pewter." + ], + "tags": [ + "materials" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -37.74419021606445, + "mean_logprob": -9.436047554016113, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.6158390045166, + "mean_logprob": -8.871946334838867, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.201614379882812, + "mean_logprob": -8.600807189941406, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -38.448585510253906, + "mean_logprob": -9.612146377563477, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.27113914489746094, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-042", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The mathematical constant that is the base of natural logarithms is", + "continuations": [ + " pi.", + " e.", + " phi.", + " tau." + ], + "tags": [ + "mathematics" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.36918830871582, + "mean_logprob": -7.7897294362386065, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.064729690551758, + "mean_logprob": -8.032364845275879, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.871200561523438, + "mean_logprob": -9.290400187174479, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -20.894588470458984, + "mean_logprob": -6.964862823486328, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.8248666127522783, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-043", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In databases, a key that uniquely identifies each row in a table is a", + "continuations": [ + " foreign schema.", + " null index.", + " primary key.", + " stored procedure." + ], + "tags": [ + "computing" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -41.638694763183594, + "mean_logprob": -8.327738952636718, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -40.04630661010742, + "mean_logprob": -10.011576652526855, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.611034393310547, + "mean_logprob": -8.870344797770182, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.76321029663086, + "mean_logprob": -10.58773676554362, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.5426058451334637, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-044", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The branch of philosophy concerned with the nature and limits of knowledge is", + "continuations": [ + " aesthetics.", + " ethics.", + " ontology alone.", + " epistemology." + ], + "tags": [ + "philosophy" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -43.860450744628906, + "mean_logprob": -8.77209014892578, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.238283157348633, + "mean_logprob": -8.746094385782877, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -44.34880828857422, + "mean_logprob": -8.869761657714843, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.22525405883789, + "mean_logprob": -10.045050811767577, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.025995763142903527, + "context_truncated": false, + "elapsed_seconds": 0.0006087460023991298 + }, + { + "id": "world_knowledge-045", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The element with atomic number 74 is", + "continuations": [ + " tungsten.", + " titanium.", + " tantalum.", + " tellurium." + ], + "tags": [ + "chemistry" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -52.217193603515625, + "mean_logprob": -10.443438720703124, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -53.312171936035156, + "mean_logprob": -13.328042984008789, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -39.47490692138672, + "mean_logprob": -7.894981384277344, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -52.20709228515625, + "mean_logprob": -13.051773071289062, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.5484573364257805, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "world_knowledge-046", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The ancient writing system used for the Mycenaean Greek language is", + "continuations": [ + " cuneiform.", + " Linear B.", + " ogham.", + " hieratic." + ], + "tags": [ + "history" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -46.64458084106445, + "mean_logprob": -11.661145210266113, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -54.82438278198242, + "mean_logprob": -10.964876556396485, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.681148529052734, + "mean_logprob": -10.670287132263184, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.086017608642578, + "mean_logprob": -7.2715044021606445, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 3.398782730102539, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "world_knowledge-047", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A star's spectral class in the sequence OBAFGKM is primarily related to its", + "continuations": [ + " distance from Earth.", + " number of planets.", + " surface temperature.", + " age alone." + ], + "tags": [ + "astronomy" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.2562370300293, + "mean_logprob": -10.064059257507324, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -37.73524475097656, + "mean_logprob": -9.43381118774414, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.024974822998047, + "mean_logprob": -9.00832494099935, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -36.06626510620117, + "mean_logprob": -12.022088368733725, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.4254862467447911, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "world_knowledge-048", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The grammatical case typically marking direct address is the", + "continuations": [ + " accusative.", + " genitive.", + " instrumental.", + " vocative." + ], + "tags": [ + "linguistics" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -40.46596145629883, + "mean_logprob": -10.116490364074707, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.30987548828125, + "mean_logprob": -11.436625162760416, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.10780143737793, + "mean_logprob": -9.03593381245931, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.00151062011719, + "mean_logprob": -11.000503540039062, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.0805565516153965, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "world_knowledge-049", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The boundary around a black hole beyond which light cannot escape is the", + "continuations": [ + " event horizon.", + " heliopause.", + " Roche limit.", + " photosphere." + ], + "tags": [ + "physics" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.12716293334961, + "mean_logprob": -9.781790733337402, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -43.501773834228516, + "mean_logprob": -10.875443458557129, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -54.071624755859375, + "mean_logprob": -10.814324951171875, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.055641174316406, + "mean_logprob": -10.513910293579102, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.7321195602416992, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "world_knowledge-050", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In rhetoric, an argument that attacks the person rather than the claim is called", + "continuations": [ + " reductio ad absurdum.", + " ad hominem.", + " modus ponens.", + " false equivalence only." + ], + "tags": [ + "rhetoric" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -115.37855529785156, + "mean_logprob": -12.819839477539062, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -44.47725296020508, + "mean_logprob": -8.895450592041016, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -64.3358154296875, + "mean_logprob": -10.722635904947916, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -41.14677810668945, + "mean_logprob": -8.229355621337891, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.6660949707031243, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "context_tracking-001", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "On the shelf, the red box contains tea and the blue box contains rice. The tea is in", + "continuations": [ + " the red box.", + " the blue box.", + " both boxes.", + " neither box." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -21.665550231933594, + "mean_logprob": -5.416387557983398, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -21.91444969177246, + "mean_logprob": -5.478612422943115, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -36.89433288574219, + "mean_logprob": -9.223583221435547, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.1932487487793, + "mean_logprob": -11.0644162495931, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.0622248649597168, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "context_tracking-002", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Ivo chose the violin, while Mara chose the flute. Mara's instrument is", + "continuations": [ + " the violin.", + " unspecified.", + " the drum.", + " the flute." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -29.987411499023438, + "mean_logprob": -7.496852874755859, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -38.44994354248047, + "mean_logprob": -9.612485885620117, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -29.045717239379883, + "mean_logprob": -7.261429309844971, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.055612564086914, + "mean_logprob": -7.7639031410217285, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.23542356491088867, + "context_truncated": false, + "elapsed_seconds": 0.0006270407502597664 + }, + { + "id": "context_tracking-003", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The morning bus stops at Oak Street. The evening bus stops at Pine Street. The evening stop is", + "continuations": [ + " Oak Street.", + " Maple Street.", + " Pine Street.", + " Cedar Street." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -20.331253051757812, + "mean_logprob": -5.082813262939453, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -45.239707946777344, + "mean_logprob": -9.047941589355469, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -24.605693817138672, + "mean_logprob": -6.151423454284668, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -51.825836181640625, + "mean_logprob": -10.365167236328125, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 1.0686101913452148, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-004", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Nia's badge number is 27, and Sol's badge number is 41. Badge 41 belongs to", + "continuations": [ + " Nia.", + " both people.", + " nobody.", + " Sol." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.40528678894043, + "mean_logprob": -8.46842892964681, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.352684020996094, + "mean_logprob": -9.450894673665365, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -21.295503616333008, + "mean_logprob": -7.098501205444336, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.734949111938477, + "mean_logprob": -7.367474555969238, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.26897335052490234, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-005", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The north garden grows mint. The south garden grows basil. Mint grows in", + "continuations": [ + " the north garden.", + " the south garden.", + " both gardens.", + " the greenhouse." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -30.0717716217041, + "mean_logprob": -7.517942905426025, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -27.212425231933594, + "mean_logprob": -6.803106307983398, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -28.775711059570312, + "mean_logprob": -7.193927764892578, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -21.968202590942383, + "mean_logprob": -7.322734196980794, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.3908214569091797, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-006", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Room A is painted yellow; Room B is painted green. The green room is", + "continuations": [ + " Room A.", + " Room B.", + " both rooms.", + " Room C." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -27.517826080322266, + "mean_logprob": -6.879456520080566, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -30.88766860961914, + "mean_logprob": -7.721917152404785, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.534170150756836, + "mean_logprob": -7.633542537689209, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -33.905460357666016, + "mean_logprob": -8.476365089416504, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.7540860176086426, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-007", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "At noon, the cafe serves soup. At night, it serves noodles. The night menu features", + "continuations": [ + " soup.", + " salad.", + " noodles.", + " bread only." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.68688201904297, + "mean_logprob": -7.228960673014323, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -23.050472259521484, + "mean_logprob": -7.683490753173828, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.912574768066406, + "mean_logprob": -8.228143692016602, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.05689811706543, + "mean_logprob": -9.01896603902181, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.4545300801595049, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-008", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Paz owns the bicycle, and Ren owns the scooter. The scooter belongs to", + "continuations": [ + " Paz.", + " both Paz and Ren.", + " neither person.", + " Ren." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -32.34737014770508, + "mean_logprob": -10.782456715901693, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -68.35550689697266, + "mean_logprob": -9.765072413853236, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.473731994628906, + "mean_logprob": -8.824577331542969, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.74805450439453, + "mean_logprob": -8.91601816813151, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.09144083658854107, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-009", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The silver key opens the attic. The copper key opens the cellar. The attic key is", + "continuations": [ + " silver.", + " copper.", + " wooden.", + " black." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.632537841796875, + "mean_logprob": -8.816268920898438, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -24.880109786987305, + "mean_logprob": -12.440054893493652, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -28.485397338867188, + "mean_logprob": -9.495132446289062, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -20.347003936767578, + "mean_logprob": -10.173501968383789, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.678863525390625, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-010", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Toma ordered pears, while Eda ordered plums. Eda ordered", + "continuations": [ + " pears.", + " plums.", + " peaches.", + " both fruits." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.419046401977539, + "mean_logprob": -3.8063488006591797, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -20.709623336791992, + "mean_logprob": -6.903207778930664, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.79245376586914, + "mean_logprob": -8.597484588623047, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.69422721862793, + "mean_logprob": -9.231409072875977, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 3.0968589782714844, + "context_truncated": false, + "elapsed_seconds": 0.0006198270020831842 + }, + { + "id": "context_tracking-011", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The first train goes to Linz. The second goes to Graz. The second destination is", + "continuations": [ + " Linz.", + " Vienna.", + " Graz.", + " Salzburg." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -32.009315490722656, + "mean_logprob": -8.002328872680664, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -42.52112579345703, + "mean_logprob": -10.630281448364258, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -21.258689880371094, + "mean_logprob": -7.086229960123698, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -41.64887237548828, + "mean_logprob": -10.41221809387207, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.9160989125569658, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-012", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "File K is dated March, and File M is dated June. The June file is", + "continuations": [ + " File K.", + " both files.", + " undated.", + " File M." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -38.149391174316406, + "mean_logprob": -9.537347793579102, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.547706604003906, + "mean_logprob": -9.515902201334635, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.28473472595215, + "mean_logprob": -9.761578241984049, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.287494659423828, + "mean_logprob": -7.321873664855957, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 2.194028536478678, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-013", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Luma wears a square pin. Oren wears a round pin. Luma's pin is", + "continuations": [ + " square.", + " round.", + " triangular.", + " missing." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.215038299560547, + "mean_logprob": -7.107519149780273, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.857137680053711, + "mean_logprob": -5.9285688400268555, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.631256103515625, + "mean_logprob": -7.657814025878906, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -19.834857940673828, + "mean_logprob": -9.917428970336914, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.178950309753418, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-014", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The code for apple is 6. The code for orange is 9. Code 9 means", + "continuations": [ + " apple.", + " orange.", + " pear.", + " fruit in general." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -15.7889404296875, + "mean_logprob": -5.262980143229167, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.425079345703125, + "mean_logprob": -9.475026448567709, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -30.424522399902344, + "mean_logprob": -10.141507466634115, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.404212951660156, + "mean_logprob": -8.101053237915039, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 2.838073094685872, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-015", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Kira arrived before Bo, and Bo arrived before Luis. The last to arrive was", + "continuations": [ + " Kira.", + " Bo.", + " Luis.", + " nobody." + ], + "tags": [ + "order" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.36800765991211, + "mean_logprob": -7.122669219970703, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.666040420532227, + "mean_logprob": -9.833020210266113, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.02602005004883, + "mean_logprob": -8.006505012512207, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.889774322509766, + "mean_logprob": -10.629924774169922, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.8838357925415039, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-016", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The small parcel weighs two kilos. The large parcel weighs five. The heavier parcel is", + "continuations": [ + " the small parcel.", + " both equally.", + " unknown.", + " the large parcel." + ], + "tags": [ + "comparison" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -35.02166748046875, + "mean_logprob": -7.00433349609375, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.408485412597656, + "mean_logprob": -11.469495137532553, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -24.494003295898438, + "mean_logprob": -12.247001647949219, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -33.43800735473633, + "mean_logprob": -6.687601470947266, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.316732025146484, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-017", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Tuesday's password is comet. Wednesday's password is lantern. On Tuesday, the password is", + "continuations": [ + " comet.", + " lantern.", + " Tuesday.", + " Wednesday." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -30.36911964416504, + "mean_logprob": -10.123039881388346, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -24.700420379638672, + "mean_logprob": -6.175105094909668, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.69491195678711, + "mean_logprob": -9.423727989196777, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -38.588687896728516, + "mean_logprob": -7.7177375793457035, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 1.5426324844360355, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-018", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Three couriers carry different folders: Aya has violet, Ben has white, and Cy has black. The white folder is carried by", + "continuations": [ + " Aya.", + " Ben.", + " Cy.", + " no courier." + ], + "tags": [ + "multi_binding" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.555641174316406, + "mean_logprob": -11.185213724772135, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.72940444946289, + "mean_logprob": -9.364702224731445, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.329429626464844, + "mean_logprob": -8.164714813232422, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.569520950317383, + "mean_logprob": -7.892380237579346, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.27233457565307617, + "context_truncated": false, + "elapsed_seconds": 0.0006441263758460991 + }, + { + "id": "context_tracking-019", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The gallery opens after the cafe but before the theater. The theater opens after the library. Among the gallery, cafe, and theater, the earliest is", + "continuations": [ + " the gallery.", + " the theater.", + " the cafe.", + " impossible to know." + ], + "tags": [ + "order" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -25.357574462890625, + "mean_logprob": -6.339393615722656, + "total_sequence_tokens": 45 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -21.285125732421875, + "mean_logprob": -5.321281433105469, + "total_sequence_tokens": 45 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -31.96614646911621, + "mean_logprob": -6.393229293823242, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -35.35783386230469, + "mean_logprob": -8.839458465576172, + "total_sequence_tokens": 45 + } + ], + "best_vs_second_margin": 1.0181121826171875, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-020", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "A token moves from Box A to Box C, then from Box C to Box B. Its final location is", + "continuations": [ + " Box A.", + " Box C.", + " every box.", + " Box B." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -24.18586540222168, + "mean_logprob": -6.04646635055542, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -22.51188850402832, + "mean_logprob": -5.62797212600708, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.194377899169922, + "mean_logprob": -7.398125966389974, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.61856460571289, + "mean_logprob": -7.404641151428223, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.41849422454833984, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-021", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Rin lends her map to Dev. Dev then gives the map back to Rin. At the end, the map is with", + "continuations": [ + " Rin.", + " Dev.", + " both of them.", + " nobody." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -29.385530471801758, + "mean_logprob": -9.79517682393392, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.8027520179748535, + "mean_logprob": -3.4013760089874268, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -38.099700927734375, + "mean_logprob": -9.524925231933594, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -33.784881591796875, + "mean_logprob": -11.261627197265625, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 6.123549222946167, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-022", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The amber team scored below the teal team but above the gray team. The highest-scoring of these teams was", + "continuations": [ + " amber.", + " teal.", + " gray.", + " tied." + ], + "tags": [ + "order" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.028976440429688, + "mean_logprob": -5.6763254801432295, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.098108291625977, + "mean_logprob": -6.032702763875325, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.976871490478516, + "mean_logprob": -7.658957163492839, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -20.74127960205078, + "mean_logprob": -6.913759867350261, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.35637728373209576, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-023", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Ava's shift is Monday, Bram's is Thursday, and Cato's is Tuesday. The person working between Ava and Bram in the week is", + "continuations": [ + " Ava.", + " Bram.", + " Cato.", + " nobody." + ], + "tags": [ + "calendar" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -20.61333656311035, + "mean_logprob": -6.87111218770345, + "total_sequence_tokens": 42 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.361501693725586, + "mean_logprob": -7.120500564575195, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.36394500732422, + "mean_logprob": -7.454648335774739, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.911911010742188, + "mean_logprob": -8.970637003580729, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 0.2493883768717451, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-024", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The note began under the lamp. Jo moved it to the desk, and Uma moved it from the desk into a drawer. The note ended", + "continuations": [ + " under the lamp.", + " on the desk.", + " with Jo.", + " in the drawer." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -29.39205551147461, + "mean_logprob": -5.878411102294922, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -41.605140686035156, + "mean_logprob": -8.321028137207032, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.092979431152344, + "mean_logprob": -8.364326477050781, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -22.979764938354492, + "mean_logprob": -4.595952987670898, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 1.2824581146240233, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-025", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The labels are reassigned: north now means left, south means right, east means up, and west means down. Under this rule, north means", + "continuations": [ + " left.", + " right.", + " up.", + " down." + ], + "tags": [ + "remapping" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.174675941467285, + "mean_logprob": -5.587337970733643, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.114038467407227, + "mean_logprob": -5.557019233703613, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.463342666625977, + "mean_logprob": -6.231671333312988, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -9.946954727172852, + "mean_logprob": -4.973477363586426, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.5835418701171875, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-026", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The bronze cup is taller than the glass cup. The glass cup is taller than the clay cup. The shortest is", + "continuations": [ + " bronze.", + " clay.", + " glass.", + " not specified." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -29.293476104736328, + "mean_logprob": -7.323369026184082, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -29.958810806274414, + "mean_logprob": -9.986270268758139, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -19.335952758789062, + "mean_logprob": -9.667976379394531, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.64965057373047, + "mean_logprob": -10.662412643432617, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 2.344607353210449, + "context_truncated": false, + "elapsed_seconds": 0.0006423150007321965 + }, + { + "id": "context_tracking-027", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Nell assigns fox to 14, owl to 22, and hare to 31. The animal assigned 31 is", + "continuations": [ + " fox.", + " owl.", + " hare.", + " none." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -12.476889610290527, + "mean_logprob": -4.158963203430176, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.468511581420898, + "mean_logprob": -3.1171278953552246, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -17.676599502563477, + "mean_logprob": -5.892199834187825, + "total_sequence_tokens": 37 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -12.921156883239746, + "mean_logprob": -6.460578441619873, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 1.0418353080749512, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-028", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "A meeting moves from Room 3 to Room 8, then is moved back to its original room. It will occur in", + "continuations": [ + " Room 8.", + " both rooms.", + " no room.", + " Room 3." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -33.14497375488281, + "mean_logprob": -6.628994750976562, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -30.09564971923828, + "mean_logprob": -7.52391242980957, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.022136688232422, + "mean_logprob": -8.67404556274414, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -30.371910095214844, + "mean_logprob": -6.074382019042969, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.5546127319335934, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-029", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "In a made-up calendar, Dax follows Rul, and Vem follows Dax. The day immediately after Rul is", + "continuations": [ + " Dax.", + " Vem.", + " Rul.", + " unknown." + ], + "tags": [ + "local_rules" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -13.760643005371094, + "mean_logprob": -4.586881001790364, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -25.64159393310547, + "mean_logprob": -8.547197977701822, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.398632049560547, + "mean_logprob": -8.46621068318685, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -25.278274536132812, + "mean_logprob": -12.639137268066406, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 3.8793296813964853, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-030", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Faye is older than Gus but younger than Hana. The oldest of the three is", + "continuations": [ + " Faye.", + " Hana.", + " Gus.", + " tied." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -26.742341995239258, + "mean_logprob": -6.6855854988098145, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.811660766601562, + "mean_logprob": -9.270553588867188, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -28.449230194091797, + "mean_logprob": -9.483076731363932, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.226682662963867, + "mean_logprob": -7.742227554321289, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.0566420555114746, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-031", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The red route reaches Delta before Echo. The blue route reaches Echo before Delta. On the blue route, the first of those stops is", + "continuations": [ + " Delta.", + " neither stop.", + " Echo.", + " both together." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -12.524965286254883, + "mean_logprob": -3.1312413215637207, + "total_sequence_tokens": 42 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -34.353416442871094, + "mean_logprob": -11.451138814290365, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.63387107849121, + "mean_logprob": -7.658467769622803, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.575090408325195, + "mean_logprob": -7.8583634694417315, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 4.527226448059082, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-032", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Iris gives the ring to Juno, Juno gives it to Kato, and Kato keeps it. The ring is finally with", + "continuations": [ + " Iris.", + " Juno.", + " everyone.", + " Kato." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -31.235769271850586, + "mean_logprob": -10.411923090616861, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -28.815208435058594, + "mean_logprob": -7.203802108764648, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.31561279296875, + "mean_logprob": -6.157806396484375, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.18698501586914, + "mean_logprob": -6.062328338623047, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.09547805786132812, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-033", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "For this report, green means urgent and red means routine. A green item is therefore", + "continuations": [ + " urgent.", + " routine.", + " canceled.", + " unclassified." + ], + "tags": [ + "remapping" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.227453231811523, + "mean_logprob": -8.075817743937174, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -15.734384536743164, + "mean_logprob": -7.867192268371582, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.40005111694336, + "mean_logprob": -11.60001277923584, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -43.29175567626953, + "mean_logprob": -10.822938919067383, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.20862547556559186, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-034", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The oak is east of the pond. The bench is east of the oak. Relative to the pond, the bench is", + "continuations": [ + " west.", + " east.", + " north.", + " at the same place." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.607290267944336, + "mean_logprob": -6.303645133972168, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.309405326843262, + "mean_logprob": -5.654702663421631, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.780250549316406, + "mean_logprob": -5.390125274658203, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -24.378231048583984, + "mean_logprob": -4.8756462097167965, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.5144790649414066, + "context_truncated": false, + "elapsed_seconds": 0.0006181348726386204 + }, + { + "id": "context_tracking-035", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Four files are ordered J, K, L, M. K is moved after M, and J is then moved between L and M. The resulting order is", + "continuations": [ + " J, L, M, K.", + " L, M, J, K.", + " L, J, M, K.", + " M, L, J, K." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -56.69683837890625, + "mean_logprob": -7.087104797363281, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -54.07590103149414, + "mean_logprob": -6.759487628936768, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -55.75765609741211, + "mean_logprob": -6.969707012176514, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -56.57053756713867, + "mean_logprob": -7.071317195892334, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.2102193832397461, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-036", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "In a codebook, tulip means north, north means copper, and copper means seven. Following the chain from tulip leads to", + "continuations": [ + " north only.", + " tulip.", + " copper only.", + " seven." + ], + "tags": [ + "multi_hop" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.62894630432129, + "mean_logprob": -7.209648768107097, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.620868682861328, + "mean_logprob": -4.405217170715332, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -28.20568084716797, + "mean_logprob": -9.401893615722656, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -18.37975311279297, + "mean_logprob": -9.189876556396484, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 2.8044315973917646, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-037", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A cube starts in Bin R. It moves to S unless the flag is blue. The flag is blue, so it moves to T instead. Its final bin is", + "continuations": [ + " Bin T.", + " Bin S.", + " Bin R.", + " unknown." + ], + "tags": [ + "conditional_update" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -28.822673797607422, + "mean_logprob": -7.2056684494018555, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -18.66064453125, + "mean_logprob": -4.6651611328125, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -25.57860565185547, + "mean_logprob": -6.394651412963867, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.504266738891602, + "mean_logprob": -6.752133369445801, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 1.7294902801513672, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-038", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Mira stands left of Noor. Olin stands right of Noor but left of Peta. From left to right, the second person is", + "continuations": [ + " Mira.", + " Noor.", + " Olin.", + " Peta." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.808246612548828, + "mean_logprob": -6.602748870849609, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.040098190307617, + "mean_logprob": -7.013366063435872, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -23.75061798095703, + "mean_logprob": -7.916872660319011, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.631656646728516, + "mean_logprob": -5.877218882242839, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.7255299886067705, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-039", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The archive uses rotating labels. On day one A means red; on day two A means blue; on day three A means gold. Today is day three, so A means", + "continuations": [ + " red.", + " blue.", + " gold.", + " every color." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -10.96800422668457, + "mean_logprob": -5.484002113342285, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -16.919618606567383, + "mean_logprob": -8.459809303283691, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.494041442871094, + "mean_logprob": -6.747020721435547, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.744218826293945, + "mean_logprob": -9.581406275431315, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 1.2630186080932617, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-040", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A coin begins with Lio, passes to Mina, returns to Lio, then passes to Niko. Niko gives it to the person who held it immediately before him. The coin ends with", + "continuations": [ + " Mina.", + " Niko.", + " nobody.", + " Lio." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.332910537719727, + "mean_logprob": -8.110970179239908, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -10.777937889099121, + "mean_logprob": -2.6944844722747803, + "total_sequence_tokens": 52 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -15.620031356811523, + "mean_logprob": -5.206677118937175, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -16.392662048339844, + "mean_logprob": -5.464220682779948, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 2.5121926466623945, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-041", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "In a race, Uma finished before Vic. Wes finished after Xena but before Uma. The earliest among Uma, Vic, Wes, and Xena was", + "continuations": [ + " Xena.", + " Wes.", + " Uma.", + " Vic." + ], + "tags": [ + "transitive" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -23.04671287536621, + "mean_logprob": -7.68223762512207, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.611141204833984, + "mean_logprob": -7.537047068277995, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -25.465457916259766, + "mean_logprob": -8.488485972086588, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.703506469726562, + "mean_logprob": -6.5678354899088545, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.9692115783691406, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-042", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "A ledger maps 4 to pine, pine to west, and west to 19. Looking up 4 repeatedly until reaching a number returns", + "continuations": [ + " west.", + " 19.", + " pine.", + " 4." + ], + "tags": [ + "multi_hop" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -26.36597442626953, + "mean_logprob": -13.182987213134766, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -14.782197952270508, + "mean_logprob": -3.695549488067627, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.240020751953125, + "mean_logprob": -8.746673583984375, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.26370620727539, + "mean_logprob": -9.087902069091797, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 5.051124095916748, + "context_truncated": false, + "elapsed_seconds": 0.0007384212512988597 + }, + { + "id": "context_tracking-043", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The black card outranks the white card. The striped card ranks below white but above gray. The second-highest card is", + "continuations": [ + " black.", + " striped.", + " white.", + " gray." + ], + "tags": [ + "order" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.521472930908203, + "mean_logprob": -8.760736465454102, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -28.118751525878906, + "mean_logprob": -9.372917175292969, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.92398452758789, + "mean_logprob": -8.961992263793945, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -16.675270080566406, + "mean_logprob": -5.558423360188802, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 3.2023131052652998, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-044", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A marker is in Drawer A. Rule one moves it to B; rule two swaps the contents of B and C; rule three moves whatever is in C to D. The marker finishes in", + "continuations": [ + " Drawer A.", + " Drawer B.", + " Drawer C.", + " Drawer D." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -32.5052490234375, + "mean_logprob": -6.5010498046875, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -40.903831481933594, + "mean_logprob": -8.180766296386718, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.591636657714844, + "mean_logprob": -8.118327331542968, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -34.80595397949219, + "mean_logprob": -6.961190795898437, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.4601409912109373, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-045", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Q is north of R. S is east of Q. T is south of S and level with R. Relative to R, T is", + "continuations": [ + " east.", + " west.", + " north.", + " at the same position." + ], + "tags": [ + "spatial" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.250899314880371, + "mean_logprob": -5.6254496574401855, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -12.281564712524414, + "mean_logprob": -6.140782356262207, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.517088890075684, + "mean_logprob": -5.258544445037842, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -37.955055236816406, + "mean_logprob": -7.591011047363281, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.36690521240234375, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-046", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "There are three switches. Flipping X toggles Y; flipping Y toggles Z. Initially all are off. X is flipped once, then Y is flipped once. At the end, Z is", + "continuations": [ + " unchanged and off.", + " on.", + " removed.", + " both on and off." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -51.79859924316406, + "mean_logprob": -8.633099873860678, + "total_sequence_tokens": 55 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.457058906555176, + "mean_logprob": -5.228529453277588, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.983139991760254, + "mean_logprob": -7.991569995880127, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -24.21500587463379, + "mean_logprob": -4.843001174926758, + "total_sequence_tokens": 54 + } + ], + "best_vs_second_margin": 0.38552827835083026, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-047", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A roster initially reads Arin, Bela, Ciro, Dena. Bela and Dena swap places; then the first and third names swap. The first name becomes", + "continuations": [ + " Arin.", + " Dena.", + " Ciro.", + " Bela." + ], + "tags": [ + "permutation" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.10793685913086, + "mean_logprob": -7.03597895304362, + "total_sequence_tokens": 44 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.120004653930664, + "mean_logprob": -9.04000155131022, + "total_sequence_tokens": 44 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -25.228734970092773, + "mean_logprob": -6.307183742523193, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.59217643737793, + "mean_logprob": -5.86405881245931, + "total_sequence_tokens": 44 + } + ], + "best_vs_second_margin": 0.44312493006388376, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-048", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "On odd dates, Gate P leads to Hall 1; on even dates it leads to Hall 2. Gate Q does the opposite. The date is the 18th, and a visitor enters Gate Q, reaching", + "continuations": [ + " Hall 2.", + " both halls.", + " no hall.", + " Hall 1." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -13.423544883728027, + "mean_logprob": -3.355886220932007, + "total_sequence_tokens": 55 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -34.71681213378906, + "mean_logprob": -8.679203033447266, + "total_sequence_tokens": 55 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -26.772235870361328, + "mean_logprob": -6.693058967590332, + "total_sequence_tokens": 55 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -14.338371276855469, + "mean_logprob": -3.584592819213867, + "total_sequence_tokens": 55 + } + ], + "best_vs_second_margin": 0.22870659828186035, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-049", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Nora is assigned to Vega unless Ishan takes Vega; if he does, Nora takes Lyra. Ishan takes Vega. Nora is therefore assigned to", + "continuations": [ + " Lyra.", + " Vega.", + " both.", + " neither." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.6636962890625, + "mean_logprob": -5.887898763020833, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.486566543579102, + "mean_logprob": -3.4955221811930337, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -14.988260269165039, + "mean_logprob": -7.4941301345825195, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.169658660888672, + "mean_logprob": -6.584829330444336, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 2.3923765818277993, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "context_tracking-050", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The sequence begins amber, cobalt, jade, silver. Remove cobalt, place silver first, then insert rose immediately after amber. The final sequence is", + "continuations": [ + " silver, rose, amber, jade.", + " silver, amber, rose, jade.", + " amber, rose, silver, jade.", + " silver, amber, jade, rose." + ], + "tags": [ + "permutation" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -65.37928771972656, + "mean_logprob": -6.537928771972656, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -64.22016906738281, + "mean_logprob": -6.422016906738281, + "total_sequence_tokens": 47 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -60.2011604309082, + "mean_logprob": -6.0201160430908205, + "total_sequence_tokens": 47 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -57.426231384277344, + "mean_logprob": -5.742623138427734, + "total_sequence_tokens": 47 + } + ], + "best_vs_second_margin": 0.2774929046630863, + "context_truncated": false, + "elapsed_seconds": 0.0006987519991525915 + }, + { + "id": "quantitative-001", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Seven plus five equals", + "continuations": [ + " twelve.", + " eleven.", + " thirteen.", + " fourteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.01771926879883, + "mean_logprob": -9.754429817199707, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -30.072134017944336, + "mean_logprob": -10.024044672648111, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.73920440673828, + "mean_logprob": -11.68480110168457, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.662960052490234, + "mean_logprob": -10.220986684163412, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.26961485544840436, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-002", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A box holds nine pencils. After three are removed, it holds", + "continuations": [ + " five pencils.", + " twelve pencils.", + " seven pencils.", + " six pencils." + ], + "tags": [ + "subtraction" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -42.907142639160156, + "mean_logprob": -8.581428527832031, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -56.482383728027344, + "mean_logprob": -8.068911961146764, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -42.51361846923828, + "mean_logprob": -8.502723693847656, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -42.34406280517578, + "mean_logprob": -8.468812561035156, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.399900599888392, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-003", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Four groups of three objects contain", + "continuations": [ + " seven objects.", + " ten objects.", + " twelve objects.", + " sixteen objects." + ], + "tags": [ + "multiplication" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.539735794067383, + "mean_logprob": -8.179911931355795, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.267900466918945, + "mean_logprob": -8.755966822306315, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -52.59984588623047, + "mean_logprob": -10.519969177246093, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -42.06551742553711, + "mean_logprob": -10.516379356384277, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.5760548909505196, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-004", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Twenty divided equally among five people gives each person", + "continuations": [ + " two.", + " three.", + " five.", + " four." + ], + "tags": [ + "division" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.93777847290039, + "mean_logprob": -9.968889236450195, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.769287109375, + "mean_logprob": -10.3846435546875, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -26.34774398803711, + "mean_logprob": -13.173871994018555, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -25.147497177124023, + "mean_logprob": -12.573748588562012, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.4157543182373047, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-005", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "The number immediately after 199 is", + "continuations": [ + " 200.", + " 198.", + " 209.", + " 299." + ], + "tags": [ + "number_sense" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -37.60895538330078, + "mean_logprob": -7.521791076660156, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -42.79155349731445, + "mean_logprob": -8.55831069946289, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -45.343963623046875, + "mean_logprob": -9.068792724609375, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -37.89729690551758, + "mean_logprob": -7.579459381103516, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.05766830444335991, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-006", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Half of eighteen is", + "continuations": [ + " eight.", + " nine.", + " ten.", + " thirty-six." + ], + "tags": [ + "fractions" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.937468528747559, + "mean_logprob": -7.468734264373779, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.69548797607422, + "mean_logprob": -10.34774398803711, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -18.544708251953125, + "mean_logprob": -9.272354125976562, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -60.306358337402344, + "mean_logprob": -10.05105972290039, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.8036198616027832, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-007", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A rectangle with length five and width two has area", + "continuations": [ + " seven.", + " twelve.", + " ten.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -16.65369415283203, + "mean_logprob": -8.326847076416016, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -32.012664794921875, + "mean_logprob": -8.003166198730469, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.72419548034668, + "mean_logprob": -8.86209774017334, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -22.801551818847656, + "mean_logprob": -7.600517272949219, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.40264892578125, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-008", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Three quarters of an hour is", + "continuations": [ + " thirty minutes.", + " forty minutes.", + " sixty minutes.", + " forty-five minutes." + ], + "tags": [ + "time" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -36.928314208984375, + "mean_logprob": -9.232078552246094, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -39.46809005737305, + "mean_logprob": -9.867022514343262, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -59.20669174194336, + "mean_logprob": -11.841338348388671, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -71.41459655761719, + "mean_logprob": -10.202085222516741, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.634943962097168, + "context_truncated": false, + "elapsed_seconds": 0.0006506953759526368 + }, + { + "id": "quantitative-009", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A ten-euro item discounted by two euros costs", + "continuations": [ + " eight euros.", + " twelve euros.", + " five euros.", + " twenty euros." + ], + "tags": [ + "money" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -44.8361701965332, + "mean_logprob": -8.967234039306641, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -55.71841812133789, + "mean_logprob": -7.959774017333984, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -44.78334045410156, + "mean_logprob": -8.956668090820312, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -43.81111145019531, + "mean_logprob": -8.762222290039062, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.8024482727050781, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-010", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "The perimeter of a square with side length three is", + "continuations": [ + " nine.", + " twelve.", + " six.", + " fifteen." + ], + "tags": [ + "geometry" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -24.015613555908203, + "mean_logprob": -12.007806777954102, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.85249328613281, + "mean_logprob": -8.963123321533203, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -24.111852645874023, + "mean_logprob": -12.055926322937012, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.729595184326172, + "mean_logprob": -10.576531728108725, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.6134084065755214, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-011", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Thirty is five more than", + "continuations": [ + " twenty.", + " thirty-five.", + " twenty-five.", + " fifteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.423551559448242, + "mean_logprob": -8.711775779724121, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -52.03375244140625, + "mean_logprob": -8.672292073567709, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -48.99390411376953, + "mean_logprob": -9.798780822753907, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.6893253326416, + "mean_logprob": -6.229775110880534, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 2.4425169626871748, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-012", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "If one notebook costs four euros, two notebooks cost", + "continuations": [ + " four euros.", + " six euros.", + " twelve euros.", + " eight euros." + ], + "tags": [ + "money" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -33.19965362548828, + "mean_logprob": -6.639930725097656, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -33.784584045410156, + "mean_logprob": -6.756916809082031, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -58.228050231933594, + "mean_logprob": -8.318292890276227, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -31.576160430908203, + "mean_logprob": -6.3152320861816404, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.3246986389160158, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-013", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The difference between fifteen and nine is", + "continuations": [ + " six.", + " five.", + " seven.", + " twenty-four." + ], + "tags": [ + "subtraction" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.237096786499023, + "mean_logprob": -8.618548393249512, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.160533905029297, + "mean_logprob": -8.580266952514648, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -14.14126205444336, + "mean_logprob": -7.07063102722168, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -42.01523971557617, + "mean_logprob": -8.403047943115235, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 1.332416915893555, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-014", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Counting by tens, the number after forty is", + "continuations": [ + " forty-one.", + " fifty.", + " sixty.", + " thirty." + ], + "tags": [ + "number_sense" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -33.271385192871094, + "mean_logprob": -6.654277038574219, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.21150016784668, + "mean_logprob": -8.73716672261556, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.29025650024414, + "mean_logprob": -9.322564125061035, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -26.49156379699707, + "mean_logprob": -8.83052126566569, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 2.0828896840413416, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-015", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Two meters equal", + "continuations": [ + " twenty centimeters.", + " two hundred millimeters.", + " two hundred centimeters.", + " two thousand centimeters." + ], + "tags": [ + "units" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -60.98021697998047, + "mean_logprob": -12.196043395996094, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -79.46316528320312, + "mean_logprob": -13.243860880533854, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -62.07371520996094, + "mean_logprob": -10.345619201660156, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -63.52356719970703, + "mean_logprob": -10.587261199951172, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.24164199829101562, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-016", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The average of four and eight is", + "continuations": [ + " four.", + " five.", + " eight.", + " six." + ], + "tags": [ + "average" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.099382400512695, + "mean_logprob": -6.049691200256348, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.007711410522461, + "mean_logprob": -6.5038557052612305, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.679021835327148, + "mean_logprob": -6.339510917663574, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.147565841674805, + "mean_logprob": -6.573782920837402, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.28981971740722656, + "context_truncated": false, + "elapsed_seconds": 0.0005825380030728411 + }, + { + "id": "quantitative-017", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A dozen eggs contains", + "continuations": [ + " twelve eggs.", + " ten eggs.", + " six eggs.", + " twenty eggs." + ], + "tags": [ + "number_sense" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.757659912109375, + "mean_logprob": -10.351531982421875, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.44968032836914, + "mean_logprob": -11.81656010945638, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.541419982910156, + "mean_logprob": -7.513806660970052, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -25.864330291748047, + "mean_logprob": -8.621443430582682, + "total_sequence_tokens": 8 + } + ], + "best_vs_second_margin": 1.10763676961263, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-018", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A train travels 180 kilometers in three hours at a constant rate. Its speed is", + "continuations": [ + " forty kilometers per hour.", + " sixty kilometers per hour.", + " ninety kilometers per hour.", + " one hundred eighty kilometers per hour." + ], + "tags": [ + "rate" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -53.53334045410156, + "mean_logprob": -7.647620064871652, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -73.94090270996094, + "mean_logprob": -9.242612838745117, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -71.78278350830078, + "mean_logprob": -8.972847938537598, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -75.8760986328125, + "mean_logprob": -8.430677625868055, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.7830575609964034, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-019", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A jacket priced at 80 euros is reduced by 25 percent. The sale price is", + "continuations": [ + " fifty euros.", + " fifty-five euros.", + " sixty euros.", + " sixty-five euros." + ], + "tags": [ + "percent" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -44.37246322631836, + "mean_logprob": -7.395410537719727, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -70.76239013671875, + "mean_logprob": -7.86248779296875, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -64.83973693847656, + "mean_logprob": -9.262819562639509, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -91.8014907836914, + "mean_logprob": -9.18014907836914, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.46707725524902344, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-020", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A tank contains 36 liters. One third is drained, leaving", + "continuations": [ + " twelve liters.", + " eighteen liters.", + " thirty liters.", + " twenty-four liters." + ], + "tags": [ + "fractions" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -63.52197265625, + "mean_logprob": -10.586995442708334, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -40.83658981323242, + "mean_logprob": -8.167317962646484, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -48.406005859375, + "mean_logprob": -9.681201171875, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -72.5405502319336, + "mean_logprob": -10.362935747419085, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.5138832092285153, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-021", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Five consecutive integers begin with 12. Their sum is", + "continuations": [ + " seventy.", + " sixty.", + " seventy-five.", + " eighty." + ], + "tags": [ + "sequence" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -43.52127456665039, + "mean_logprob": -10.880318641662598, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -44.13595962524414, + "mean_logprob": -11.033989906311035, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -67.41502380371094, + "mean_logprob": -9.63071768624442, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.550472259521484, + "mean_logprob": -9.516824086507162, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.1138935997372581, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-022", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A right triangle has legs of length 6 and 8. Its hypotenuse has length", + "continuations": [ + " nine.", + " ten.", + " twelve.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -16.792171478271484, + "mean_logprob": -8.396085739135742, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -22.10921859741211, + "mean_logprob": -11.054609298706055, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.35191535949707, + "mean_logprob": -7.587978839874268, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -21.904508590698242, + "mean_logprob": -7.301502863566081, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.28647597630818655, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-023", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "If x plus 7 equals 19, then x equals", + "continuations": [ + " ten.", + " eleven.", + " twelve.", + " twenty-six." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.147296905517578, + "mean_logprob": -6.573648452758789, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.15754318237305, + "mean_logprob": -11.05251439412435, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -33.975791931152344, + "mean_logprob": -8.493947982788086, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -56.30901336669922, + "mean_logprob": -11.261802673339844, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.9202995300292969, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-024", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A worker earns 18 euros per hour for seven hours. Total earnings are", + "continuations": [ + " 108 euros.", + " 116 euros.", + " 136 euros.", + " 126 euros." + ], + "tags": [ + "multiplication" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -53.0643424987793, + "mean_logprob": -6.633042812347412, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -58.949913024902344, + "mean_logprob": -7.368739128112793, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -53.73325729370117, + "mean_logprob": -6.7166571617126465, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -56.35491943359375, + "mean_logprob": -7.044364929199219, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.08361434936523438, + "context_truncated": false, + "elapsed_seconds": 0.0006130209985713009 + }, + { + "id": "quantitative-025", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The ratio of red to blue marbles is 3 to 5. If there are 15 red marbles, there are", + "continuations": [ + " twenty-five blue marbles.", + " nine blue marbles.", + " eighteen blue marbles.", + " seventy-five blue marbles." + ], + "tags": [ + "ratio" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -61.12688446044922, + "mean_logprob": -7.640860557556152, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -33.53755569458008, + "mean_logprob": -6.707511138916016, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -39.7473030090332, + "mean_logprob": -6.624550501505534, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -78.81594848632812, + "mean_logprob": -7.881594848632813, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 0.08296063741048165, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-026", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "A circle has radius 4. Using pi as 3.14, its circumference is approximately", + "continuations": [ + " 12.56.", + " 25.12.", + " 50.24.", + " 16.00." + ], + "tags": [ + "geometry" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -50.65218734741211, + "mean_logprob": -7.236026763916016, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -46.29389953613281, + "mean_logprob": -6.613414219447544, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -52.307579040527344, + "mean_logprob": -7.472511291503906, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -50.596771240234375, + "mean_logprob": -7.228110177176339, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.6146959577287952, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-027", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "An account balance rises from 240 to 300. The percentage increase is", + "continuations": [ + " fifteen percent.", + " twenty percent.", + " twenty-five percent.", + " sixty percent." + ], + "tags": [ + "percent" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.59833908081055, + "mean_logprob": -9.899584770202637, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.54113006591797, + "mean_logprob": -9.180376688639322, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -62.243797302246094, + "mean_logprob": -10.373966217041016, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -53.93549346923828, + "mean_logprob": -10.787098693847657, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.7192080815633144, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-028", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A bus leaves at 09:35 and arrives at 11:05. The journey lasts", + "continuations": [ + " seventy minutes.", + " eighty minutes.", + " one hundred minutes.", + " ninety minutes." + ], + "tags": [ + "time" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -48.33152770996094, + "mean_logprob": -9.666305541992188, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -41.44609069824219, + "mean_logprob": -10.361522674560547, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.53441619873047, + "mean_logprob": -8.133604049682617, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.65397644042969, + "mean_logprob": -10.130795288085938, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.5327014923095703, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-029", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The median of 3, 7, 8, 12, and 20 is", + "continuations": [ + " eight.", + " seven.", + " ten.", + " twelve." + ], + "tags": [ + "statistics" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -19.214981079101562, + "mean_logprob": -9.607490539550781, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.03617286682129, + "mean_logprob": -9.018086433410645, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.760871887207031, + "mean_logprob": -6.380435943603516, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -26.134326934814453, + "mean_logprob": -6.533581733703613, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.15314579010009766, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-030", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A 48-page booklet is printed with four pages per sheet. It requires", + "continuations": [ + " ten sheets.", + " twelve sheets.", + " sixteen sheets.", + " forty-four sheets." + ], + "tags": [ + "division" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -32.071109771728516, + "mean_logprob": -8.017777442932129, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -53.00082778930664, + "mean_logprob": -8.833471298217773, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -34.53920364379883, + "mean_logprob": -6.907840728759766, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -66.54702758789062, + "mean_logprob": -8.318378448486328, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 1.1099367141723633, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-031", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Three fifths written as a decimal is", + "continuations": [ + " 0.3.", + " 0.5.", + " 0.6.", + " 0.8." + ], + "tags": [ + "fractions" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -44.197505950927734, + "mean_logprob": -8.839501190185548, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -45.50183868408203, + "mean_logprob": -9.100367736816406, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -46.43128204345703, + "mean_logprob": -9.286256408691406, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -48.77204132080078, + "mean_logprob": -9.754408264160157, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.26086654663085795, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-032", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "A shop buys an item for 30 euros and sells it for 42 euros. The profit is", + "continuations": [ + " 10 euros.", + " 11 euros.", + " 72 euros.", + " 12 euros." + ], + "tags": [ + "money" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -37.01690673828125, + "mean_logprob": -5.288129534040179, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -35.841190338134766, + "mean_logprob": -5.120170048304966, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -51.78416442871094, + "mean_logprob": -7.397737775530134, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -34.20521926879883, + "mean_logprob": -4.88645989554269, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.23371015276227602, + "context_truncated": false, + "elapsed_seconds": 0.0006680201258859597 + }, + { + "id": "quantitative-033", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "If 2x minus 4 equals 16, then x equals", + "continuations": [ + " ten.", + " six.", + " eight.", + " twelve." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.043170928955078, + "mean_logprob": -8.521585464477539, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -21.092214584350586, + "mean_logprob": -10.546107292175293, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -23.542829513549805, + "mean_logprob": -11.771414756774902, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.262779235839844, + "mean_logprob": -8.065694808959961, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.4558906555175781, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-034", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A price increases by 20 percent and is then reduced by 20 percent. Starting from 100 euros, the final price is", + "continuations": [ + " 100 euros.", + " 96 euros.", + " 80 euros.", + " 104 euros." + ], + "tags": [ + "percent" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -35.123924255371094, + "mean_logprob": -4.390490531921387, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -47.65105438232422, + "mean_logprob": -6.8072934831891745, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -42.49840545654297, + "mean_logprob": -6.071200779506138, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -48.0104866027832, + "mean_logprob": -6.0013108253479, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 1.6108202934265137, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-035", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A mixture contains milk and water in a 5 to 2 ratio. In 21 liters of mixture, the amount of water is", + "continuations": [ + " five liters.", + " seven liters.", + " six liters.", + " fifteen liters." + ], + "tags": [ + "ratio" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.98579406738281, + "mean_logprob": -8.996448516845703, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.173828125, + "mean_logprob": -8.29345703125, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.128173828125, + "mean_logprob": -8.03204345703125, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -47.30379867553711, + "mean_logprob": -9.460759735107422, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.26141357421875, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-036", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "Two numbers have sum 46 and difference 8. The larger number is", + "continuations": [ + " nineteen.", + " twenty-three.", + " thirty-five.", + " twenty-seven." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.06748962402344, + "mean_logprob": -8.26687240600586, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -55.12561798095703, + "mean_logprob": -11.025123596191406, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -53.57238006591797, + "mean_logprob": -8.928730010986328, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -54.66562271118164, + "mean_logprob": -10.933124542236328, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.6618576049804688, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-037", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A car travels the first 120 kilometers at 60 km/h and the next 120 at 40 km/h. Its average speed for the full trip is", + "continuations": [ + " 48 km/h.", + " 50 km/h.", + " 45 km/h.", + " 52 km/h." + ], + "tags": [ + "rate" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -35.52599334716797, + "mean_logprob": -5.075141906738281, + "total_sequence_tokens": 49 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -29.06087875366211, + "mean_logprob": -4.151554107666016, + "total_sequence_tokens": 49 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -43.60639190673828, + "mean_logprob": -6.229484558105469, + "total_sequence_tokens": 49 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -38.14623260498047, + "mean_logprob": -5.449461800711496, + "total_sequence_tokens": 49 + } + ], + "best_vs_second_margin": 0.9235877990722656, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-038", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A rectangle's length is three more than its width. If its perimeter is 30, its width is", + "continuations": [ + " five.", + " six.", + " seven.", + " nine." + ], + "tags": [ + "algebra" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.98013687133789, + "mean_logprob": -5.990068435668945, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.90248966217041, + "mean_logprob": -5.451244831085205, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -14.075971603393555, + "mean_logprob": -7.037985801696777, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.726163864135742, + "mean_logprob": -7.363081932067871, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.5388236045837402, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-039", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A bag has four red, three blue, and five green balls. The probability of drawing a blue ball is", + "continuations": [ + " one third.", + " three fifths.", + " one quarter.", + " three quarters." + ], + "tags": [ + "probability" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.773807525634766, + "mean_logprob": -8.257935841878256, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -28.46892547607422, + "mean_logprob": -5.693785095214844, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.721267700195312, + "mean_logprob": -9.240422566731771, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -27.875974655151367, + "mean_logprob": -6.968993663787842, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.2752085685729977, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-040", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "An investment of 2,000 euros earns simple interest at 4 percent per year for three years. The interest earned is", + "continuations": [ + " 80 euros.", + " 160 euros.", + " 2,240 euros.", + " 240 euros." + ], + "tags": [ + "finance" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -40.89963150024414, + "mean_logprob": -5.842804500034878, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -48.60980987548828, + "mean_logprob": -6.076226234436035, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -66.28264617919922, + "mean_logprob": -6.628264617919922, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -59.199981689453125, + "mean_logprob": -7.399997711181641, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 0.23342173440115754, + "context_truncated": false, + "elapsed_seconds": 0.000640897749690339 + }, + { + "id": "quantitative-041", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "The sequence 2, 6, 12, 20, 30 continues with", + "continuations": [ + " 42.", + " 36.", + " 40.", + " 44." + ], + "tags": [ + "sequence" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -15.689617156982422, + "mean_logprob": -3.9224042892456055, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -28.44967269897461, + "mean_logprob": -7.112418174743652, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -20.066699981689453, + "mean_logprob": -5.016674995422363, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -23.725341796875, + "mean_logprob": -5.93133544921875, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.0942707061767578, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-042", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A machine completes three jobs in 18 minutes at a constant rate. At that rate, eight jobs take", + "continuations": [ + " 36 minutes.", + " 48 minutes.", + " 54 minutes.", + " 144 minutes." + ], + "tags": [ + "rate" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -47.93095397949219, + "mean_logprob": -9.586190795898437, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.38521194458008, + "mean_logprob": -8.677042388916016, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -43.020896911621094, + "mean_logprob": -8.604179382324219, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -59.279239654541016, + "mean_logprob": -9.879873275756836, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.07286300659179723, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-043", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "The mean of six numbers is 14. Five of them sum to 61. The sixth number is", + "continuations": [ + " fourteen.", + " seventeen.", + " twenty-three.", + " eighty-four." + ], + "tags": [ + "statistics" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -13.87632942199707, + "mean_logprob": -4.62544314066569, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.07023811340332, + "mean_logprob": -7.76755952835083, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -46.94623947143555, + "mean_logprob": -9.38924789428711, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -45.94563293457031, + "mean_logprob": -7.657605489095052, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 3.0321623484293614, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-044", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A map scale is 1 to 50,000. A distance of 6 centimeters on the map represents", + "continuations": [ + " 300 meters.", + " 30 kilometers.", + " 300 kilometers.", + " 3 kilometers." + ], + "tags": [ + "scale" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -31.586660385131836, + "mean_logprob": -5.264443397521973, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -28.607112884521484, + "mean_logprob": -4.767852147420247, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -32.26197814941406, + "mean_logprob": -4.608854021344866, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -28.51755142211914, + "mean_logprob": -5.703510284423828, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.15899812607538077, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-045", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "If three quarters of a number is 27, the number is", + "continuations": [ + " thirty-six.", + " twenty-one.", + " thirty.", + " forty." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -54.35548400878906, + "mean_logprob": -9.059247334798178, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -28.820100784301758, + "mean_logprob": -7.2050251960754395, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.298553466796875, + "mean_logprob": -9.766184488932291, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.157215118408203, + "mean_logprob": -5.719071706136067, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.485953489939372, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-046", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A 15-liter solution is 40 percent salt. Adding 5 liters of water makes the new salt concentration", + "continuations": [ + " twenty percent.", + " thirty percent.", + " thirty-five percent.", + " forty-five percent." + ], + "tags": [ + "mixture" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.613317489624023, + "mean_logprob": -9.204439163208008, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -45.380577087402344, + "mean_logprob": -11.345144271850586, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -69.27904510498047, + "mean_logprob": -9.897006443568639, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -73.7576904296875, + "mean_logprob": -10.536812918526786, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.6925672803606311, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-047", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A square's diagonal has length 10. Its area is", + "continuations": [ + " twenty-five.", + " forty.", + " fifty.", + " one hundred." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -51.766273498535156, + "mean_logprob": -10.353254699707032, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.97705078125, + "mean_logprob": -8.992350260416666, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -38.21430206298828, + "mean_logprob": -12.73810068766276, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -25.88629913330078, + "mean_logprob": -8.628766377766928, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.3635838826497384, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-048", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A clock gains two minutes every three hours. After 24 real hours, it will be ahead by", + "continuations": [ + " eight minutes.", + " twelve minutes.", + " eighteen minutes.", + " sixteen minutes." + ], + "tags": [ + "rate" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -30.86008071899414, + "mean_logprob": -10.286693572998047, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.49627685546875, + "mean_logprob": -8.69925537109375, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -39.648780822753906, + "mean_logprob": -9.912195205688477, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.679664611816406, + "mean_logprob": -7.919916152954102, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.7793392181396488, + "context_truncated": false, + "elapsed_seconds": 0.00063447562570218 + }, + { + "id": "quantitative-049", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A number is increased by 8, doubled, and then reduced by 6 to give 30. The original number is", + "continuations": [ + " ten.", + " eight.", + " twelve.", + " fourteen." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -13.370779037475586, + "mean_logprob": -6.685389518737793, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.2511625289917, + "mean_logprob": -6.62558126449585, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -27.935344696044922, + "mean_logprob": -6.9838361740112305, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -16.376888275146484, + "mean_logprob": -5.458962758382161, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.1666185061136884, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "quantitative-050", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A store marks an item up 25 percent from a cost of 64 euros. The marked price is", + "continuations": [ + " 72 euros.", + " 80 euros.", + " 84 euros.", + " 89 euros." + ], + "tags": [ + "percent" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -42.51408767700195, + "mean_logprob": -6.073441096714565, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -39.979286193847656, + "mean_logprob": -5.711326599121094, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -37.506141662597656, + "mean_logprob": -5.3580202375139505, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -35.52503967285156, + "mean_logprob": -5.075005667550223, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.2830145699637274, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-001", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "All robins are birds. Pip is a robin. Therefore, Pip is", + "continuations": [ + " a bird.", + " not a bird.", + " a fish.", + " necessarily blue." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.873626708984375, + "mean_logprob": -8.291208902994791, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.66956329345703, + "mean_logprob": -7.917390823364258, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -20.651714324951172, + "mean_logprob": -6.883904774983724, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -28.075511932373047, + "mean_logprob": -9.358503977457682, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.0334860483805342, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-002", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No squares are circles. Shape K is a square. Shape K is therefore", + "continuations": [ + " a circle.", + " not a square.", + " both shapes.", + " not a circle." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -25.032014846801758, + "mean_logprob": -8.34400494893392, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -22.964313507080078, + "mean_logprob": -5.7410783767700195, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -29.93118667602539, + "mean_logprob": -9.977062225341797, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -28.500995635986328, + "mean_logprob": -7.125248908996582, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.3841705322265625, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-003", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If the switch is on, the lamp glows. The switch is on, so the lamp", + "continuations": [ + " must be broken.", + " is off.", + " glows.", + " has no switch." + ], + "tags": [ + "modus_ponens" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -39.44676971435547, + "mean_logprob": -9.861692428588867, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.013668060302734, + "mean_logprob": -8.671222686767578, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -7.634694576263428, + "mean_logprob": -1.908673644065857, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.90302276611328, + "mean_logprob": -8.72575569152832, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 6.762549042701721, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-004", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "A is taller than B, and B is taller than C. It follows that", + "continuations": [ + " C is taller than A.", + " A and C are equal.", + " B is shortest.", + " A is taller than C." + ], + "tags": [ + "transitive" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -44.07673263549805, + "mean_logprob": -6.296676090785435, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -45.065162658691406, + "mean_logprob": -7.510860443115234, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.98060607910156, + "mean_logprob": -8.196121215820312, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -38.7853889465332, + "mean_logprob": -5.540769849504743, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.7559062412806918, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-005", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Either the coin is in the cup or in the box. It is not in the cup. Thus it is", + "continuations": [ + " in the box.", + " in both places.", + " nowhere.", + " still in the cup." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -21.211158752441406, + "mean_logprob": -5.302789688110352, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -27.404403686523438, + "mean_logprob": -6.851100921630859, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -35.987422943115234, + "mean_logprob": -11.995807647705078, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -40.86028289794922, + "mean_logprob": -8.172056579589844, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 1.5483112335205078, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-006", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Every metal key conducts electricity. This object is a metal key, so it", + "continuations": [ + " cannot be metal.", + " conducts electricity.", + " must be wooden.", + " proves all objects conduct." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -38.250953674316406, + "mean_logprob": -9.562738418579102, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.500682830810547, + "mean_logprob": -7.875170707702637, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -44.90924072265625, + "mean_logprob": -8.98184814453125, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -52.655941009521484, + "mean_logprob": -8.77599016825358, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.9008194605509434, + "context_truncated": false, + "elapsed_seconds": 0.0006186840000736993 + }, + { + "id": "logical_reasoning-007", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If today is Monday, tomorrow is Tuesday. Today is Monday. Tomorrow is", + "continuations": [ + " Sunday.", + " Wednesday.", + " Tuesday.", + " Monday." + ], + "tags": [ + "conditional" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.522586822509766, + "mean_logprob": -6.507528940836589, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -37.20925521850586, + "mean_logprob": -7.441851043701172, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -17.979923248291016, + "mean_logprob": -4.494980812072754, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.43180465698242, + "mean_logprob": -8.107951164245605, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 2.012548128763835, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-008", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No red token is large. Token P is red. Token P is", + "continuations": [ + " definitely large.", + " both red and large.", + " not red.", + " not large." + ], + "tags": [ + "deduction" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -32.839622497558594, + "mean_logprob": -8.209905624389648, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -44.41047668457031, + "mean_logprob": -8.882095336914062, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.16616439819336, + "mean_logprob": -8.722054799397787, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.836578369140625, + "mean_logprob": -9.278859456380209, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.5121491750081386, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-009", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Only one of Door A and Door B is open. Door A is closed, so", + "continuations": [ + " Door B is open.", + " both are closed.", + " both are open.", + " Door B must be closed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -25.576068878173828, + "mean_logprob": -4.262678146362305, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -25.511682510375977, + "mean_logprob": -6.377920627593994, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -24.294939041137695, + "mean_logprob": -6.073734760284424, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -45.53675842285156, + "mean_logprob": -6.505251203264509, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 1.8110566139221191, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-010", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "All cedar trees are plants. No plant is a machine. Therefore, no cedar tree is", + "continuations": [ + " a plant.", + " a machine.", + " a tree.", + " alive." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.173559188842773, + "mean_logprob": -9.057853062947592, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -26.926116943359375, + "mean_logprob": -8.975372314453125, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.388507843017578, + "mean_logprob": -7.462835947672526, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -19.215091705322266, + "mean_logprob": -9.607545852661133, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.5125363667805987, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-011", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "If it rains, the path gets wet. The path is not wet. Therefore, it", + "continuations": [ + " certainly snowed.", + " must be night.", + " did not rain.", + " is still raining." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -48.27105712890625, + "mean_logprob": -12.067764282226562, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -41.714088439941406, + "mean_logprob": -10.428522109985352, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -32.524993896484375, + "mean_logprob": -8.131248474121094, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.294403076171875, + "mean_logprob": -7.058880615234375, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.072367858886719, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-012", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "L is before M, and M is before N. In this ordering, N is", + "continuations": [ + " first.", + " before L.", + " between L and M.", + " after L." + ], + "tags": [ + "order" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.965788841247559, + "mean_logprob": -6.482894420623779, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -23.95630645751953, + "mean_logprob": -7.985435485839844, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.408409118652344, + "mean_logprob": -8.081681823730468, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.54198455810547, + "mean_logprob": -6.180661519368489, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.30223290125529, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-013", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "Some poets are teachers. Every teacher reads. Therefore, some poets", + "continuations": [ + " read.", + " never read.", + " are not teachers.", + " are all teachers." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.911632537841797, + "mean_logprob": -7.955816268920898, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.183990478515625, + "mean_logprob": -7.394663492838542, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -25.9031982421875, + "mean_logprob": -6.475799560546875, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.038700103759766, + "mean_logprob": -7.509675025939941, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.918863932291667, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-014", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "Exactly one card is marked. Card X is not marked and Card Y is marked. Card Z is therefore", + "continuations": [ + " also marked.", + " not marked.", + " the same as Y.", + " impossible to classify." + ], + "tags": [ + "constraint" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -34.57331848144531, + "mean_logprob": -11.524439493815104, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -27.353572845458984, + "mean_logprob": -9.117857615152994, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -34.39896774291992, + "mean_logprob": -6.879793548583985, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -55.387351989746094, + "mean_logprob": -11.077470397949218, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 2.2380640665690095, + "context_truncated": false, + "elapsed_seconds": 0.0006069989976822399 + }, + { + "id": "logical_reasoning-015", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "If a number is divisible by four, it is even. Twenty is divisible by four, so twenty is", + "continuations": [ + " odd.", + " prime.", + " even.", + " negative." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -28.246990203857422, + "mean_logprob": -9.415663401285807, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -17.80933380126953, + "mean_logprob": -5.936444600423177, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.069190979003906, + "mean_logprob": -8.034595489501953, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -21.112478256225586, + "mean_logprob": -10.556239128112793, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 2.0981508890787763, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-016", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No blue item is fragile. Vase V is fragile. Therefore, Vase V is", + "continuations": [ + " definitely blue.", + " both blue and fragile.", + " not fragile.", + " not blue." + ], + "tags": [ + "contraposition" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.181365966796875, + "mean_logprob": -8.545341491699219, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -44.83970642089844, + "mean_logprob": -7.473284403483073, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -26.38355827331543, + "mean_logprob": -6.595889568328857, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -24.448780059814453, + "mean_logprob": -8.149593353271484, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.8773948351542158, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-017", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "All glims are nords. Some nords are pelts. From these facts alone, it follows that", + "continuations": [ + " no conclusion links glims to pelts.", + " all glims are pelts.", + " some glims are pelts.", + " no nords are glims." + ], + "tags": [ + "quantifiers" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -76.02500915527344, + "mean_logprob": -6.911364468661222, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -45.16609191894531, + "mean_logprob": -5.01845465766059, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -43.778587341308594, + "mean_logprob": -4.864287482367621, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -50.21465301513672, + "mean_logprob": -6.27683162689209, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.15416717529296875, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-018", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "If R then S. If S then T. R is true. Therefore,", + "continuations": [ + " T is false.", + " T is true.", + " S is false.", + " R is false." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.201358795166016, + "mean_logprob": -8.550339698791504, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.71513366699219, + "mean_logprob": -8.928783416748047, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -39.636077880859375, + "mean_logprob": -9.909019470214844, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -35.04518127441406, + "mean_logprob": -8.761295318603516, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.21095561981201172, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-019", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Exactly two of A, B, and C are true. A is true and B is false. Therefore, C is", + "continuations": [ + " false.", + " unknown.", + " true.", + " both true and false." + ], + "tags": [ + "constraint" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.874483108520508, + "mean_logprob": -8.937241554260254, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -19.214202880859375, + "mean_logprob": -9.607101440429688, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.62352466583252, + "mean_logprob": -5.81176233291626, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -45.28544998168945, + "mean_logprob": -9.05708999633789, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 3.125479221343994, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-020", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Kai sits somewhere left of Lio. Mina sits right of Lio. Which statement must be true?", + "continuations": [ + " Mina sits left of Kai.", + " Kai sits beside Mina.", + " Lio sits left of Kai.", + " Kai sits left of Mina." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -63.91800308227539, + "mean_logprob": -7.102000342475043, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -85.18739318847656, + "mean_logprob": -8.518739318847656, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -60.826236724853516, + "mean_logprob": -6.758470747205946, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -54.75862121582031, + "mean_logprob": -6.084291246202257, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.6741795010036888, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-021", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Every zor is a vek, and no vek is a lum. Therefore,", + "continuations": [ + " no zor is a lum.", + " every lum is a zor.", + " some zors are lums.", + " no zor is a vek." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -57.47293472290039, + "mean_logprob": -7.184116840362549, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -69.03628540039062, + "mean_logprob": -8.629535675048828, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -62.94375991821289, + "mean_logprob": -8.991965702601842, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -58.7258415222168, + "mean_logprob": -7.3407301902771, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.15661334991455078, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-022", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alarm rings if and only if the sensor is active. The alarm is silent. Thus the sensor is", + "continuations": [ + " active.", + " inactive.", + " both states.", + " unrelated to the alarm." + ], + "tags": [ + "biconditional" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.18564224243164, + "mean_logprob": -6.09282112121582, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -19.06684112548828, + "mean_logprob": -6.355613708496094, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.932445526123047, + "mean_logprob": -8.977481842041016, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -46.79875183105469, + "mean_logprob": -6.685535975864956, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.26279258728027344, + "context_truncated": false, + "elapsed_seconds": 0.0006800847513659392 + }, + { + "id": "logical_reasoning-023", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "At least one of P or Q is false. P is true. It follows that Q is", + "continuations": [ + " true.", + " undetermined.", + " false.", + " equal to P." + ], + "tags": [ + "propositional" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -9.893251419067383, + "mean_logprob": -4.946625709533691, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -59.21479034423828, + "mean_logprob": -11.842958068847656, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.830623626708984, + "mean_logprob": -6.415311813354492, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -40.95244598388672, + "mean_logprob": -10.23811149597168, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.4686861038208008, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-024", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Three runners finish with no ties. Ada is not first. Bo finishes after Cy. The only possible first-place runner is", + "continuations": [ + " Ada.", + " Bo.", + " either Ada or Bo.", + " Cy." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -15.50674057006836, + "mean_logprob": -5.16891352335612, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -17.830760955810547, + "mean_logprob": -8.915380477905273, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -39.374019622802734, + "mean_logprob": -6.562336603800456, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -19.24895668029785, + "mean_logprob": -9.624478340148926, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 1.393423080444336, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-025", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "If the report is approved, funding continues. Funding did not continue. Therefore, the report was", + "continuations": [ + " not approved.", + " approved late.", + " necessarily reviewed.", + " unrelated to funding." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -27.070613861083984, + "mean_logprob": -9.023537953694662, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -35.65403747558594, + "mean_logprob": -11.884679158528646, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -42.93102264404297, + "mean_logprob": -10.732755661010742, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -49.94780349731445, + "mean_logprob": -9.98956069946289, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.9660227457682282, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-026", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Some artists are engineers, and no engineer is careless. Therefore, some artists are", + "continuations": [ + " careless.", + " not careless.", + " not engineers.", + " all engineers." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -12.718421936035156, + "mean_logprob": -4.239473978678386, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -24.674175262451172, + "mean_logprob": -6.168543815612793, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -26.798019409179688, + "mean_logprob": -8.932673136393229, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.27902603149414, + "mean_logprob": -10.093008677164713, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.9290698369344073, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-027", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A statement and its negation cannot both be true. The statement is true, so its negation is", + "continuations": [ + " true.", + " unknown.", + " false.", + " equivalent." + ], + "tags": [ + "negation" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.916280746459961, + "mean_logprob": -6.4581403732299805, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.257645606994629, + "mean_logprob": -6.6288228034973145, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.068572998046875, + "mean_logprob": -6.5342864990234375, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -22.75235366821289, + "mean_logprob": -11.376176834106445, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.07614612579345703, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-028", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "If X is selected, Y must be selected. If Y is selected, Z cannot be selected. X is selected, so Z", + "continuations": [ + " must be selected.", + " may equal X.", + " makes Y unselected.", + " cannot be selected." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -26.869823455810547, + "mean_logprob": -6.717455863952637, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.514991760253906, + "mean_logprob": -9.128747940063477, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -61.94955062866211, + "mean_logprob": -8.849935804094587, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.948558807373047, + "mean_logprob": -7.487139701843262, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.769683837890625, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-029", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "No historians are timeless beings. Some writers are historians. Therefore,", + "continuations": [ + " some writers are not timeless beings.", + " all writers are timeless.", + " no writers are historians.", + " all historians are writers." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -66.81134033203125, + "mean_logprob": -8.351417541503906, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -48.70854187011719, + "mean_logprob": -8.118090311686197, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -47.97731018066406, + "mean_logprob": -7.996218363444011, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -54.248802185058594, + "mean_logprob": -9.0414670308431, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.12187194824218661, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-030", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Either J or K, but not both, is open. If J is open, L is closed. J is open. Hence", + "continuations": [ + " K and L are open.", + " K and L are closed.", + " K is open and L closed.", + " K is closed and L open." + ], + "tags": [ + "propositional" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -49.69562530517578, + "mean_logprob": -8.282604217529297, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -50.68724822998047, + "mean_logprob": -8.447874704996744, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -43.10638427734375, + "mean_logprob": -6.158054896763393, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -54.44194030761719, + "mean_logprob": -7.7774200439453125, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 1.6193651471819193, + "context_truncated": false, + "elapsed_seconds": 0.0006101742510509212 + }, + { + "id": "logical_reasoning-031", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Rhea is before Sami. Taro is after Sami. Ula is before Rhea. The second person in order is", + "continuations": [ + " Ula.", + " Sami.", + " Rhea.", + " Taro." + ], + "tags": [ + "order" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -13.59786605834961, + "mean_logprob": -4.53262201944987, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.953633785247803, + "mean_logprob": -2.6512112617492676, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -17.270219802856445, + "mean_logprob": -5.7567399342854815, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -21.673748016357422, + "mean_logprob": -5.4184370040893555, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 1.8814107577006025, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-032", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "All valid tickets have a stamp. This ticket has no stamp. It is therefore", + "continuations": [ + " valid.", + " stamped invisibly.", + " both valid and invalid.", + " not valid." + ], + "tags": [ + "contraposition" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.947935104370117, + "mean_logprob": -7.973967552185059, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -65.82256317138672, + "mean_logprob": -9.403223310198102, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -51.00755310058594, + "mean_logprob": -7.286793300083706, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -16.660537719726562, + "mean_logprob": -5.5535125732421875, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.7332807268415182, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-033", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Exactly one of the claims \"A is true\" and \"B is false\" is true. A is false. Therefore, B is", + "continuations": [ + " false.", + " true.", + " both.", + " undetermined." + ], + "tags": [ + "propositional" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -17.428428649902344, + "mean_logprob": -8.714214324951172, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -12.954870223999023, + "mean_logprob": -6.477435111999512, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.632009506225586, + "mean_logprob": -6.816004753112793, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -55.83190155029297, + "mean_logprob": -11.166380310058594, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.33856964111328125, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-034", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Every quill is a renn. No renn is a soth. Some varks are quills. It follows that some varks are", + "continuations": [ + " soths.", + " not soths.", + " not quills.", + " all renns." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -17.164167404174805, + "mean_logprob": -4.291041851043701, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -22.488882064819336, + "mean_logprob": -4.4977764129638675, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -24.69638442993164, + "mean_logprob": -6.17409610748291, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.89082336425781, + "mean_logprob": -7.178164672851563, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 0.20673456192016637, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-035", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Exactly two of P, Q, R, and S are true. P and Q have opposite truth values. R is true and S is false. Therefore,", + "continuations": [ + " P and Q are both true.", + " P and Q are both false.", + " exactly one of P and Q is true.", + " neither P nor Q matters." + ], + "tags": [ + "constraint" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -43.365089416503906, + "mean_logprob": -6.195012773786273, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -40.65787124633789, + "mean_logprob": -5.808267320905413, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -58.00055694580078, + "mean_logprob": -6.444506327311198, + "total_sequence_tokens": 43 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -55.50283432006836, + "mean_logprob": -9.25047238667806, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.3867454528808594, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-036", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "If A then B. If not C then not B. A is true. Which conclusion must follow?", + "continuations": [ + " C is false.", + " B is false.", + " A is false.", + " C is true." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -41.0430908203125, + "mean_logprob": -10.260772705078125, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -45.18550491333008, + "mean_logprob": -11.29637622833252, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.483055114746094, + "mean_logprob": -8.620763778686523, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -33.449745178222656, + "mean_logprob": -8.362436294555664, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.2583274841308594, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-037", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Five seats are numbered 1 through 5. Iva sits immediately left of Jae. Kian sits in seat 5. Iva sits in seat 2. Jae sits in", + "continuations": [ + " seat 3.", + " seat 1.", + " seat 4.", + " seat 5." + ], + "tags": [ + "constraint" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -14.242015838623047, + "mean_logprob": -3.5605039596557617, + "total_sequence_tokens": 51 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.0240478515625, + "mean_logprob": -3.756011962890625, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.26809024810791, + "mean_logprob": -3.8170225620269775, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -13.651144981384277, + "mean_logprob": -3.4127862453460693, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.14771771430969238, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-038", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No fal is a mer. Every tov is a fal. Some riks are mers. Which statement must be true?", + "continuations": [ + " Some riks are tovs.", + " No tov is a mer.", + " Every rik is a fal.", + " Some mers are fals." + ], + "tags": [ + "quantifiers" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -42.4434928894043, + "mean_logprob": -4.715943654378255, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -45.260196685791016, + "mean_logprob": -6.4657423836844305, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -49.88524627685547, + "mean_logprob": -6.235655784606934, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -42.0517463684082, + "mean_logprob": -6.007392338344029, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 1.2914486839657737, + "context_truncated": false, + "elapsed_seconds": 0.0006751877481292468 + }, + { + "id": "logical_reasoning-039", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "A safe opens exactly when both keys are turned. The safe did not open, and the first key was turned. Therefore, the second key was", + "continuations": [ + " turned.", + " broken necessarily.", + " not turned.", + " unrelated." + ], + "tags": [ + "conjunction" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -22.176910400390625, + "mean_logprob": -11.088455200195312, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -38.554718017578125, + "mean_logprob": -12.851572672526041, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -28.202688217163086, + "mean_logprob": -9.400896072387695, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -24.825214385986328, + "mean_logprob": -8.275071461995443, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 1.125824610392252, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-040", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A committee selects exactly three of J, K, L, and M. If J is selected, K is not. L and M are selected. Therefore, the third member must be", + "continuations": [ + " J.", + " L again.", + " M again.", + " K." + ], + "tags": [ + "constraint" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.81520938873291, + "mean_logprob": -7.907604694366455, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.83157730102539, + "mean_logprob": -7.27719243367513, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.861717224121094, + "mean_logprob": -5.620572408040364, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -10.19455337524414, + "mean_logprob": -5.09727668762207, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.523295720418294, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-041", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Some nels are paks. All paks are wims. No wim is a dor. Therefore,", + "continuations": [ + " some nels are not dors.", + " all nels are wims.", + " some dors are paks.", + " no nels are paks." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -49.954856872558594, + "mean_logprob": -6.244357109069824, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -58.172035217285156, + "mean_logprob": -7.2715044021606445, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -41.25672149658203, + "mean_logprob": -5.8938173566545755, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -46.443702697753906, + "mean_logprob": -6.6348146711077005, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.3505397524152487, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-042", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Four tasks W, X, Y, Z are scheduled once each. W precedes X. Y immediately precedes Z. X is first. These conditions are", + "continuations": [ + " satisfied by X, W, Y, Z.", + " impossible to satisfy.", + " satisfied by W, X, Y, Z.", + " satisfied by Y, Z, W, X." + ], + "tags": [ + "consistency" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -71.8221664428711, + "mean_logprob": -6.529287858442827, + "total_sequence_tokens": 46 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -42.760982513427734, + "mean_logprob": -8.552196502685547, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -65.78887939453125, + "mean_logprob": -5.980807217684659, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -68.23700714111328, + "mean_logprob": -6.203364285555753, + "total_sequence_tokens": 46 + } + ], + "best_vs_second_margin": 0.22255706787109375, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-043", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If either M or N is true, P is true. P is false. Therefore,", + "continuations": [ + " M is true.", + " N is true.", + " both M and N are false.", + " exactly one is false." + ], + "tags": [ + "contraposition" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -28.238346099853516, + "mean_logprob": -7.059586524963379, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -22.21750259399414, + "mean_logprob": -5.554375648498535, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -48.798484802246094, + "mean_logprob": -6.971212114606585, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.372314453125, + "mean_logprob": -7.074462890625, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.41683646610805, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-044", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "All A-things are B-things. Some B-things are C-things. No C-thing is a D-thing. Which claim is guaranteed?", + "continuations": [ + " Some A-things are C-things.", + " No A-thing is a D-thing.", + " All B-things are A-things.", + " Some B-things are not D-things." + ], + "tags": [ + "quantifiers" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -53.35425567626953, + "mean_logprob": -4.850386879660866, + "total_sequence_tokens": 50 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -47.036956787109375, + "mean_logprob": -4.703695678710938, + "total_sequence_tokens": 49 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -68.14971923828125, + "mean_logprob": -6.195429021661932, + "total_sequence_tokens": 50 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -66.75942993164062, + "mean_logprob": -5.563285827636719, + "total_sequence_tokens": 51 + } + ], + "best_vs_second_margin": 0.14669120094992838, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-045", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A code is valid if exactly one of conditions X and Y holds. X holds and the code is valid. Therefore, Y", + "continuations": [ + " does not hold.", + " also holds.", + " is equivalent to X.", + " cannot be discussed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -26.49679183959961, + "mean_logprob": -6.624197959899902, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -20.20181655883789, + "mean_logprob": -6.733938852945964, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -48.03401184082031, + "mean_logprob": -9.606802368164063, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.50175476074219, + "mean_logprob": -8.625438690185547, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.1097408930460615, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-046", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "R is earlier than S. T is later than U. S is earlier than U. Which ordering must hold?", + "continuations": [ + " T is earlier than R.", + " R is earlier than T.", + " U is earlier than S.", + " S is later than T." + ], + "tags": [ + "transitive" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -37.85660934448242, + "mean_logprob": -6.30943489074707, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -36.548095703125, + "mean_logprob": -6.091349283854167, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -35.97064208984375, + "mean_logprob": -5.995107014973958, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -34.77587127685547, + "mean_logprob": -5.795978546142578, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.1991284688313799, + "context_truncated": false, + "elapsed_seconds": 0.0006743826270394493 + }, + { + "id": "logical_reasoning-047", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If the engine starts, either fuel or battery power is available. The engine starts, and no fuel is available. Thus", + "continuations": [ + " no power is available.", + " the engine did not start.", + " battery power is available.", + " fuel is available." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -36.68482971191406, + "mean_logprob": -7.336965942382813, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -46.194793701171875, + "mean_logprob": -7.6991322835286455, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -38.529563903808594, + "mean_logprob": -7.705912780761719, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.73880958557129, + "mean_logprob": -7.434702396392822, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.09773645401000941, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "logical_reasoning-048", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "Exactly one person among A, B, and C lies. A says B tells the truth. B says C tells the truth. C says A lies. The liar is", + "continuations": [ + " A.", + " B.", + " impossible to determine.", + " C." + ], + "tags": [ + "truth_tellers" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.947668075561523, + "mean_logprob": -6.473834037780762, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.734720230102539, + "mean_logprob": -6.8673601150512695, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -34.79273223876953, + "mean_logprob": -8.698183059692383, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.884757041931152, + "mean_logprob": -5.942378520965576, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 0.5314555168151855, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "logical_reasoning-049", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No glens are haps. Some jors are glens. Every jor is a mip. It follows that some mips are", + "continuations": [ + " not haps.", + " haps.", + " not jors.", + " all glens." + ], + "tags": [ + "syllogism" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -27.398021697998047, + "mean_logprob": -6.849505424499512, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.93634033203125, + "mean_logprob": -7.64544677734375, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -25.17153549194336, + "mean_logprob": -6.29288387298584, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -23.41002655029297, + "mean_logprob": -5.852506637573242, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.44037723541259766, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "logical_reasoning-050", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A proposition is true only if Q is true, and Q is true only if R is false. The proposition is true. Therefore, R is", + "continuations": [ + " true.", + " false.", + " both true and false.", + " independent of Q." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -8.509860038757324, + "mean_logprob": -4.254930019378662, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -7.415702819824219, + "mean_logprob": -3.7078514099121094, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -29.241044998168945, + "mean_logprob": -5.848208999633789, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -35.482688903808594, + "mean_logprob": -8.870672225952148, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.5470786094665527, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "code_completion-001", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def square(x):\n", + "continuations": [ + " return x * x\n", + " return x + x\n", + " print(x)\n", + " return 2\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -64.4998779296875, + "mean_logprob": -8.062484741210938, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -67.82811737060547, + "mean_logprob": -8.478514671325684, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -71.71414184570312, + "mean_logprob": -8.96426773071289, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -56.271183013916016, + "mean_logprob": -8.03874043055943, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.02374431065150695, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "code_completion-002", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def is_empty(items):\n", + "continuations": [ + " return items[0]\n", + " return len(items) == 0\n", + " return len(items) + 1\n", + " items.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -58.50013732910156, + "mean_logprob": -6.5000152587890625, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -100.62899017333984, + "mean_logprob": -6.708599344889323, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -102.07505798339844, + "mean_logprob": -7.291075570242746, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -80.3046875, + "mean_logprob": -8.03046875, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.20858408610026036, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "code_completion-003", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def greet(name):\n", + "continuations": [ + " return name - \"Hello\"\n", + " print = name\n", + " return \"Hello, \" + name\n", + " return len(\"Hello\")\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -94.52261352539062, + "mean_logprob": -7.876884460449219, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -68.80657196044922, + "mean_logprob": -9.829510280064174, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -116.4942855834961, + "mean_logprob": -8.96109889103816, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -126.24462890625, + "mean_logprob": -9.017473493303571, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.0842144305889416, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "code_completion-004", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def first(items):\n", + "continuations": [ + " return items[-1]\n", + " return len(items)\n", + " return items\n", + " return items[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -81.32862091064453, + "mean_logprob": -8.132862091064453, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -70.47511291503906, + "mean_logprob": -6.406828446821733, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -41.42377853393555, + "mean_logprob": -6.903963088989258, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -72.35987854003906, + "mean_logprob": -8.039986504448784, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.49713464216752445, + "context_truncated": false, + "elapsed_seconds": 0.0006179792508191895 + }, + { + "id": "code_completion-005", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def add(a, b):\n", + "continuations": [ + " return a + b\n", + " return a - b\n", + " return a == b\n", + " return [a, b]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -59.59634017944336, + "mean_logprob": -7.44954252243042, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -55.10966873168945, + "mean_logprob": -6.888708591461182, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -69.24526977539062, + "mean_logprob": -7.693918863932292, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -80.50672149658203, + "mean_logprob": -8.050672149658203, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.5608339309692383, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-006", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def last(items):\n", + "continuations": [ + " return items[0]\n", + " return items[-1]\n", + " return items[1:]\n", + " return -len(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -68.88465118408203, + "mean_logprob": -7.6538501315646705, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -79.58338165283203, + "mean_logprob": -7.9583381652832035, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -74.48210906982422, + "mean_logprob": -7.448210906982422, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -94.82456970214844, + "mean_logprob": -7.902047475179036, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.20563922458224848, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-007", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def is_even(n):\n", + "continuations": [ + " return n / 2\n", + " return n % 2 == 1\n", + " return n % 2 == 0\n", + " return n * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -66.42578125, + "mean_logprob": -7.380642361111111, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -111.68634033203125, + "mean_logprob": -7.977595738002232, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -113.54393005371094, + "mean_logprob": -8.11028071812221, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -60.2064094543457, + "mean_logprob": -6.689601050482856, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.6910413106282549, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-008", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def absolute_value(x):\n", + "continuations": [ + " return -abs(x)\n", + " return x * 0\n", + " return bool(x)\n", + " return abs(x)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -81.40156555175781, + "mean_logprob": -7.400142322887074, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -67.146728515625, + "mean_logprob": -7.460747612847222, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -78.09366607666016, + "mean_logprob": -7.809366607666016, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -65.0159912109375, + "mean_logprob": -7.2239990234375, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.1761432994495742, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-009", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def double_all(numbers):\n", + "continuations": [ + " return [n * 2 for n in numbers]\n", + " return numbers * 0\n", + " return [2 for n in numbers]\n", + " return len(numbers) * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -103.17900848388672, + "mean_logprob": -6.878600565592448, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -60.89836883544922, + "mean_logprob": -6.766485426161024, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -83.21892547607422, + "mean_logprob": -6.9349104563395185, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -107.17425537109375, + "mean_logprob": -7.144950358072917, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.11211513943142393, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-010", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def contains(items, value):\n", + "continuations": [ + " return items == value\n", + " return value in items\n", + " return items + value\n", + " return items.index\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -55.86119079589844, + "mean_logprob": -6.206798977322048, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -57.433311462402344, + "mean_logprob": -7.179163932800293, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -53.57707977294922, + "mean_logprob": -6.697134971618652, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -65.32173919677734, + "mean_logprob": -7.257971021864149, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.4903359942966041, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-011", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def maximum(a, b):\n", + "continuations": [ + " return a + b\n", + " return a < b\n", + " return a if a > b else b\n", + " return None\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -58.671287536621094, + "mean_logprob": -7.333910942077637, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -59.07000732421875, + "mean_logprob": -7.383750915527344, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -103.42295837402344, + "mean_logprob": -8.618579864501953, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -61.86029815673828, + "mean_logprob": -8.837185450962611, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.04983997344970703, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-012", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def repeat(text, count):\n", + "continuations": [ + " return count / text\n", + " return len(text)\n", + " return text + count\n", + " return text * count\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -54.93566131591797, + "mean_logprob": -6.866957664489746, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -88.8314437866211, + "mean_logprob": -8.075585798783736, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -61.172508239746094, + "mean_logprob": -7.646563529968262, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -53.27174377441406, + "mean_logprob": -6.658967971801758, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.20798969268798828, + "context_truncated": false, + "elapsed_seconds": 0.000646987999061821 + }, + { + "id": "code_completion-013", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def keys_list(mapping):\n", + "continuations": [ + " return list(mapping.keys())\n", + " return mapping.values\n", + " return [mapping]\n", + " return mapping.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -139.94610595703125, + "mean_logprob": -9.329740397135417, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -74.72267150878906, + "mean_logprob": -7.472267150878906, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -92.17547607421875, + "mean_logprob": -10.241719563802084, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -100.77813720703125, + "mean_logprob": -8.398178100585938, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.9259109497070312, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-014", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def starts_with_a(text):\n", + "continuations": [ + " return text[-1] == \"a\"\n", + " return text.startswith(\"a\")\n", + " return \"a\" + text\n", + " return text.count\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -150.28900146484375, + "mean_logprob": -10.019266764322916, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 16, + "sum_logprob": -166.7127685546875, + "mean_logprob": -10.419548034667969, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -77.43402099609375, + "mean_logprob": -7.743402099609375, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -83.32905578613281, + "mean_logprob": -9.258783976236979, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.515381876627604, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-015", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def total(numbers):\n", + "continuations": [ + " return len(numbers)\n", + " return max(numbers)\n", + " return sum(numbers)\n", + " return numbers[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -86.95172882080078, + "mean_logprob": -7.245977401733398, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -80.07550048828125, + "mean_logprob": -7.279590953480113, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -79.93278503417969, + "mean_logprob": -7.2666168212890625, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -66.48738861083984, + "mean_logprob": -7.387487623426649, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.020639419555664062, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-016", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def clamp_zero(x):\n", + "continuations": [ + " return min(0, x)\n", + " return x == 0\n", + " return -x\n", + " return max(0, x)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -83.23368072509766, + "mean_logprob": -7.56669824773615, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -77.92828369140625, + "mean_logprob": -7.792828369140625, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -53.59979248046875, + "mean_logprob": -7.657113211495536, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -81.6349868774414, + "mean_logprob": -7.421362443403765, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.1453358043323858, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-017", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def reversed_copy(items):\n", + "continuations": [ + " return items[::-1]\n", + " return items[::1]\n", + " return items.reverse\n", + " return sorted(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -97.65043640136719, + "mean_logprob": -8.1375363667806, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -91.92560577392578, + "mean_logprob": -8.35687325217507, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -72.24282836914062, + "mean_logprob": -8.026980929904514, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -78.83255004882812, + "mean_logprob": -7.166595458984375, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.8603854709201393, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-018", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def count_positive(numbers):\n", + "continuations": [ + " return sum(numbers)\n", + " return sum(1 for n in numbers if n > 0)\n", + " return [n > 0 for n in numbers]\n", + " return len(numbers) > 0\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -84.54900360107422, + "mean_logprob": -7.68627305464311, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -141.5113525390625, + "mean_logprob": -7.861741807725695, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 15, + "sum_logprob": -126.06310272216797, + "mean_logprob": -8.404206848144531, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -105.72014617919922, + "mean_logprob": -7.0480097452799475, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.6382633093631629, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-019", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def unique_in_order(items):\n seen = set()\n", + "continuations": [ + " return set(items)\n", + " return sorted(items)\n", + " return [x for x in items if not (x in seen or seen.add(x))]\n", + " return list(reversed(items))\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -62.2546272277832, + "mean_logprob": -6.22546272277832, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -85.78536987304688, + "mean_logprob": -7.798669988458807, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 26, + "sum_logprob": -218.25479125976562, + "mean_logprob": -8.394415048452524, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -107.25091552734375, + "mean_logprob": -7.15006103515625, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.9245983123779302, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-020", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def safe_divide(a, b):\n if b == 0:\n", + "continuations": [ + " return a\n return b / a\n", + " b = 1\n return b\n", + " raise StopIteration\n return a + b\n", + " return None\n return a / b\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 18, + "sum_logprob": -73.69779205322266, + "mean_logprob": -4.094321780734592, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -58.988121032714844, + "mean_logprob": -3.2771178351508246, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -121.98308563232422, + "mean_logprob": -5.544685710560191, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -98.77484130859375, + "mean_logprob": -5.198675858347039, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 0.8172039455837679, + "context_truncated": false, + "elapsed_seconds": 0.0006332248740363866 + }, + { + "id": "code_completion-021", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def word_lengths(words):\n", + "continuations": [ + " return {word: len(word) for word in words}\n", + " return {len(words): word for word in words}\n", + " return [words, len(words)]\n", + " return words.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 19, + "sum_logprob": -158.77627563476562, + "mean_logprob": -8.356646086040296, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 19, + "sum_logprob": -163.4668426513672, + "mean_logprob": -8.603518034282484, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 16, + "sum_logprob": -130.91583251953125, + "mean_logprob": -8.182239532470703, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -100.77430725097656, + "mean_logprob": -9.161300659179688, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.17440655356959311, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-022", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def flatten_once(groups):\n", + "continuations": [ + " return list(groups)\n", + " return [item for group in groups for item in group]\n", + " return [groups[0]]\n", + " return sum(len(g) for g in groups)\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -71.32257080078125, + "mean_logprob": -6.483870072798296, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -134.9700469970703, + "mean_logprob": -7.93941452923943, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -88.4291000366211, + "mean_logprob": -6.802238464355469, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 17, + "sum_logprob": -115.90776824951172, + "mean_logprob": -6.81810401467716, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.3183683915571729, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-023", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def factorial(n):\n result = 1\n for value in range(2, n + 1):\n", + "continuations": [ + " result += value\n return n\n", + " result = value\n return value\n", + " result *= value\n return result\n", + " n -= value\n return n\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 18, + "sum_logprob": -72.63949584960938, + "mean_logprob": -4.0355275472005205, + "total_sequence_tokens": 49 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -61.173919677734375, + "mean_logprob": -3.5984658633961395, + "total_sequence_tokens": 48 + }, + { + "index": 2, + "tokens": 18, + "sum_logprob": -75.02532958984375, + "mean_logprob": -4.16807386610243, + "total_sequence_tokens": 49 + }, + { + "index": 3, + "tokens": 18, + "sum_logprob": -68.0163803100586, + "mean_logprob": -3.7786877950032554, + "total_sequence_tokens": 49 + } + ], + "best_vs_second_margin": 0.18022193160711586, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-024", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def first_index(items, target):\n for index, item in enumerate(items):\n if item == target:\n", + "continuations": [ + " continue\n return 0\n", + " return item\n return target\n", + " break\n return index\n", + " return index\n return -1\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 20, + "sum_logprob": -59.487579345703125, + "mean_logprob": -2.974378967285156, + "total_sequence_tokens": 62 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -62.084495544433594, + "mean_logprob": -3.1042247772216798, + "total_sequence_tokens": 62 + }, + { + "index": 2, + "tokens": 19, + "sum_logprob": -68.74903869628906, + "mean_logprob": -3.6183704576994242, + "total_sequence_tokens": 61 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -62.67036437988281, + "mean_logprob": -2.9843030657087053, + "total_sequence_tokens": 63 + } + ], + "best_vs_second_margin": 0.009924098423549133, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-025", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def merge_counts(left, right):\n result = dict(left)\n for key, value in right.items():\n", + "continuations": [ + " result[key] = result.get(key, 0) + value\n return result\n", + " result[value] = key\n return left\n", + " result.clear()\n return right\n", + " key += value\n return result.values()\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 30, + "sum_logprob": -164.9541015625, + "mean_logprob": -5.498470052083333, + "total_sequence_tokens": 71 + }, + { + "index": 1, + "tokens": 21, + "sum_logprob": -107.4541244506836, + "mean_logprob": -5.116863069080171, + "total_sequence_tokens": 62 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -88.68522644042969, + "mean_logprob": -4.434261322021484, + "total_sequence_tokens": 61 + }, + { + "index": 3, + "tokens": 23, + "sum_logprob": -109.01890563964844, + "mean_logprob": -4.739952419115149, + "total_sequence_tokens": 64 + } + ], + "best_vs_second_margin": 0.30569109709366504, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-026", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def chunks(items, size):\n", + "continuations": [ + " return items[:size]\n", + " return [items[i:i + size] for i in range(0, len(items), size)]\n", + " return [size] * len(items)\n", + " return items[::size][0]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -85.84294128417969, + "mean_logprob": -7.803903753107244, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 31, + "sum_logprob": -260.35479736328125, + "mean_logprob": -8.398541850428428, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 16, + "sum_logprob": -118.6962661743164, + "mean_logprob": -7.418516635894775, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 15, + "sum_logprob": -112.9700927734375, + "mean_logprob": -7.531339518229166, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.11282288233439086, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-027", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def balanced_parentheses(text):\n depth = 0\n for char in text:\n if char == \"(\":\n depth += 1\n elif char == \")\":\n depth -= 1\n if depth < 0:\n return False\n", + "continuations": [ + " return True\n", + " return depth > 0\n", + " return depth == 0\n", + " return len(text) == depth\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -31.38246726989746, + "mean_logprob": -3.9228084087371826, + "total_sequence_tokens": 134 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -47.16850662231445, + "mean_logprob": -5.240945180257161, + "total_sequence_tokens": 135 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -46.837989807128906, + "mean_logprob": -4.683798980712891, + "total_sequence_tokens": 136 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -81.84320831298828, + "mean_logprob": -5.845943450927734, + "total_sequence_tokens": 140 + } + ], + "best_vs_second_margin": 0.7609905719757082, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-028", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def rotate_left(items, amount):\n if not items:\n return []\n", + "continuations": [ + " return items[::-1]\n", + " return items[:amount]\n", + " return items + amount\n", + " amount %= len(items)\n return items[amount:] + items[:amount]\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 12, + "sum_logprob": -73.76446533203125, + "mean_logprob": -6.1470387776692705, + "total_sequence_tokens": 44 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -82.20048522949219, + "mean_logprob": -7.47277138449929, + "total_sequence_tokens": 43 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -52.64512634277344, + "mean_logprob": -6.58064079284668, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -218.92623901367188, + "mean_logprob": -6.841444969177246, + "total_sequence_tokens": 64 + } + ], + "best_vs_second_margin": 0.43360201517740915, + "context_truncated": false, + "elapsed_seconds": 0.0008992002512968611 + }, + { + "id": "code_completion-029", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def group_by_first_letter(words):\n groups = {}\n for word in words:\n", + "continuations": [ + " groups.setdefault(word[0], []).append(word)\n return groups\n", + " groups[word] = word[0]\n return words\n", + " groups.append(word[0])\n return groups\n", + " word[0] = groups\n return groups.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 32, + "sum_logprob": -225.76681518554688, + "mean_logprob": -7.05521297454834, + "total_sequence_tokens": 65 + }, + { + "index": 1, + "tokens": 23, + "sum_logprob": -134.1617431640625, + "mean_logprob": -5.833119268002718, + "total_sequence_tokens": 56 + }, + { + "index": 2, + "tokens": 24, + "sum_logprob": -157.0426025390625, + "mean_logprob": -6.5434417724609375, + "total_sequence_tokens": 57 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -152.34860229492188, + "mean_logprob": -6.093944091796875, + "total_sequence_tokens": 58 + } + ], + "best_vs_second_margin": 0.26082482379415683, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-030", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def moving_pairs(items):\n", + "continuations": [ + " return zip(items, items)\n", + " return list(zip(items, items[1:]))\n", + " return items[::2]\n", + " return [(items[0], items[-1])]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -93.85356140136719, + "mean_logprob": -7.219504723182092, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -145.56588745117188, + "mean_logprob": -7.278294372558594, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -76.40733337402344, + "mean_logprob": -6.9461212158203125, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -143.17666625976562, + "mean_logprob": -7.535614013671875, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.2733835073617792, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-031", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def invert_mapping(mapping):\n", + "continuations": [ + " return dict(mapping)\n", + " return list(mapping.items())\n", + " return {value: key for key, value in mapping.items()}\n", + " return mapping[::-1]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -96.83506774902344, + "mean_logprob": -8.80318797718395, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -126.3913345336914, + "mean_logprob": -8.42608896891276, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 24, + "sum_logprob": -187.07003784179688, + "mean_logprob": -7.79458491007487, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -99.59236145019531, + "mean_logprob": -7.660950880784255, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.13363402929061508, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-032", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def all_equal(items):\n iterator = iter(items)\n try:\n first = next(iterator)\n except StopIteration:\n", + "continuations": [ + " return False\n return first == iterator\n", + " first = None\n return items == first\n", + " return []\n return len(items)\n", + " return True\n return all(item == first for item in iterator)\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -83.51063537597656, + "mean_logprob": -3.7959379716352983, + "total_sequence_tokens": 78 + }, + { + "index": 1, + "tokens": 21, + "sum_logprob": -92.9236068725586, + "mean_logprob": -4.424933660598028, + "total_sequence_tokens": 77 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -76.58889770507812, + "mean_logprob": -3.481313532049006, + "total_sequence_tokens": 78 + }, + { + "index": 3, + "tokens": 31, + "sum_logprob": -150.43948364257812, + "mean_logprob": -4.852886569115423, + "total_sequence_tokens": 87 + } + ], + "best_vs_second_margin": 0.3146244395862925, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-033", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def transpose(rows):\n", + "continuations": [ + " return [list(column) for column in zip(*rows)]\n", + " return rows[::-1]\n", + " return list(zip(rows))\n", + " return [sum(row) for row in rows]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -187.91241455078125, + "mean_logprob": -8.541473388671875, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -90.9638671875, + "mean_logprob": -6.997220552884615, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -85.95217895507812, + "mean_logprob": -6.611706073467548, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 17, + "sum_logprob": -141.53628540039062, + "mean_logprob": -8.3256638470818, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.3855144794170666, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-034", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def running_totals(numbers):\n totals = []\n current = 0\n for number in numbers:\n", + "continuations": [ + " totals.append(number)\n return current\n", + " current += number\n totals.append(current)\n return totals\n", + " current = number\n return [current]\n", + " totals += current\n return numbers\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 23, + "sum_logprob": -135.65884399414062, + "mean_logprob": -5.8982106084408965, + "total_sequence_tokens": 61 + }, + { + "index": 1, + "tokens": 36, + "sum_logprob": -172.3142547607422, + "mean_logprob": -4.786507076687283, + "total_sequence_tokens": 74 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -112.49333953857422, + "mean_logprob": -5.624666976928711, + "total_sequence_tokens": 58 + }, + { + "index": 3, + "tokens": 19, + "sum_logprob": -83.2471923828125, + "mean_logprob": -4.381431178042763, + "total_sequence_tokens": 57 + } + ], + "best_vs_second_margin": 0.40507589864451976, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-035", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def binary_search(items, target):\n low, high = 0, len(items) - 1\n while low <= high:\n mid = (low + high) // 2\n if items[mid] == target:\n return mid\n if items[mid] < target:\n low = mid + 1\n else:\n", + "continuations": [ + " low = mid - 1\n return low\n", + " high = mid + 1\n return high\n", + " high = mid - 1\n return -1\n", + " return mid\n return None\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 24, + "sum_logprob": -62.32008361816406, + "mean_logprob": -2.596670150756836, + "total_sequence_tokens": 163 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -60.67481231689453, + "mean_logprob": -2.5281171798706055, + "total_sequence_tokens": 163 + }, + { + "index": 2, + "tokens": 25, + "sum_logprob": -59.0748291015625, + "mean_logprob": -2.3629931640625, + "total_sequence_tokens": 164 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -63.36119079589844, + "mean_logprob": -3.0171995617094494, + "total_sequence_tokens": 160 + } + ], + "best_vs_second_margin": 0.16512401580810554, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-036", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def merge_sorted(left, right):\n result = []\n i = j = 0\n while i < len(left) and j < len(right):\n if left[i] <= right[j]:\n result.append(left[i])\n i += 1\n else:\n result.append(right[j])\n j += 1\n", + "continuations": [ + " return result\n", + " result.extend(left)\n return result\n", + " return sorted(left)\n", + " result.extend(left[i:])\n result.extend(right[j:])\n return result\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -23.438350677490234, + "mean_logprob": -3.9063917795817056, + "total_sequence_tokens": 167 + }, + { + "index": 1, + "tokens": 18, + "sum_logprob": -90.318359375, + "mean_logprob": -5.017686631944445, + "total_sequence_tokens": 179 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -60.692161560058594, + "mean_logprob": -5.5174692327326, + "total_sequence_tokens": 172 + }, + { + "index": 3, + "tokens": 37, + "sum_logprob": -180.72010803222656, + "mean_logprob": -4.884327244114232, + "total_sequence_tokens": 198 + } + ], + "best_vs_second_margin": 0.9779354645325262, + "context_truncated": false, + "elapsed_seconds": 0.0011890407513419632 + }, + { + "id": "code_completion-037", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def topological_visit(node, graph, visiting, visited, order):\n if node in visiting:\n raise ValueError(\"cycle\")\n if node in visited:\n return\n visiting.add(node)\n for neighbor in graph.get(node, []):\n topological_visit(neighbor, graph, visiting, visited, order)\n", + "continuations": [ + " visiting.remove(node)\n visited.add(node)\n order.append(node)\n", + " order.append(node)\n visiting.add(node)\n", + " visited.remove(node)\n order.clear()\n", + " return graph[node]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 36, + "sum_logprob": -226.78045654296875, + "mean_logprob": -6.299457126193577, + "total_sequence_tokens": 162 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -152.7591094970703, + "mean_logprob": -6.364962895711263, + "total_sequence_tokens": 150 + }, + { + "index": 2, + "tokens": 22, + "sum_logprob": -151.3779754638672, + "mean_logprob": -6.880817066539418, + "total_sequence_tokens": 148 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -65.12138366699219, + "mean_logprob": -6.512138366699219, + "total_sequence_tokens": 136 + } + ], + "best_vs_second_margin": 0.06550576951768594, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-038", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def longest_common_prefix(words):\n if not words:\n return \"\"\n prefix = words[0]\n for word in words[1:]:\n while not word.startswith(prefix):\n", + "continuations": [ + " prefix += word\n", + " prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n", + " word = word[:-1]\n return word\n", + " return words[0]\n return prefix + word\n" + ], + "tags": [ + "python", + "string" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 17, + "sum_logprob": -61.9388427734375, + "mean_logprob": -3.6434613396139706, + "total_sequence_tokens": 98 + }, + { + "index": 1, + "tokens": 65, + "sum_logprob": -199.16104125976562, + "mean_logprob": -3.0640160193810098, + "total_sequence_tokens": 146 + }, + { + "index": 2, + "tokens": 26, + "sum_logprob": -93.62850952148438, + "mean_logprob": -3.6010965200570912, + "total_sequence_tokens": 107 + }, + { + "index": 3, + "tokens": 26, + "sum_logprob": -86.75059509277344, + "mean_logprob": -3.3365613497220554, + "total_sequence_tokens": 107 + } + ], + "best_vs_second_margin": 0.27254533034104567, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-039", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def memoized_fibonacci(n, cache=None):\n if cache is None:\n cache = {0: 0, 1: 1}\n if n not in cache:\n", + "continuations": [ + " cache[n] = n\n", + " return cache\n", + " cache[n] = memoized_fibonacci(n - 1, cache) + memoized_fibonacci(n - 2, cache)\n return cache[n]\n", + " cache.clear()\n return n\n" + ], + "tags": [ + "python", + "recursion" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -58.41175842285156, + "mean_logprob": -3.894117228190104, + "total_sequence_tokens": 81 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -22.842281341552734, + "mean_logprob": -2.0765710310502485, + "total_sequence_tokens": 77 + }, + { + "index": 2, + "tokens": 61, + "sum_logprob": -296.2841796875, + "mean_logprob": -4.857117699795082, + "total_sequence_tokens": 127 + }, + { + "index": 3, + "tokens": 21, + "sum_logprob": -91.00448608398438, + "mean_logprob": -4.333546956380208, + "total_sequence_tokens": 87 + } + ], + "best_vs_second_margin": 1.8175461971398557, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-040", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def intervals_overlap(a_start, a_end, b_start, b_end):\n", + "continuations": [ + " return a_end < b_start\n", + " return a_start == b_end\n", + " return a_start + b_start\n", + " return max(a_start, b_start) <= min(a_end, b_end)\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -71.48335266113281, + "mean_logprob": -5.498719435471755, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -75.27032470703125, + "mean_logprob": -5.376451764787946, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -80.02064514160156, + "mean_logprob": -5.715760367257254, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 29, + "sum_logprob": -174.2914276123047, + "mean_logprob": -6.010049228010507, + "total_sequence_tokens": 57 + } + ], + "best_vs_second_margin": 0.12226767068380884, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-041", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def breadth_first_distances(graph, start):\n from collections import deque\n distances = {start: 0}\n queue = deque([start])\n while queue:\n node = queue.popleft()\n for neighbor in graph.get(node, []):\n if neighbor not in distances:\n", + "continuations": [ + " distances[neighbor] = distances[node] + 1\n queue.append(neighbor)\n return distances\n", + " distances[node] += 1\n return queue\n", + " queue.append(node)\n return graph\n", + " return neighbor\n return distances[start]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 67, + "sum_logprob": -265.82781982421875, + "mean_logprob": -3.967579400361474, + "total_sequence_tokens": 194 + }, + { + "index": 1, + "tokens": 34, + "sum_logprob": -139.9459228515625, + "mean_logprob": -4.116056554457721, + "total_sequence_tokens": 161 + }, + { + "index": 2, + "tokens": 32, + "sum_logprob": -132.1974639892578, + "mean_logprob": -4.131170749664307, + "total_sequence_tokens": 159 + }, + { + "index": 3, + "tokens": 29, + "sum_logprob": -96.3578109741211, + "mean_logprob": -3.3226831370386583, + "total_sequence_tokens": 156 + } + ], + "best_vs_second_margin": 0.6448962633228157, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-042", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def deep_get(mapping, path, default=None):\n current = mapping\n for key in path:\n if not isinstance(current, dict) or key not in current:\n return default\n", + "continuations": [ + " current = mapping[key]\n return mapping\n", + " current = current[key]\n return current\n", + " path = current\n return key\n", + " return current.get(path)\n return default\n" + ], + "tags": [ + "python", + "nested_data" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -111.19834899902344, + "mean_logprob": -5.05447040904652, + "total_sequence_tokens": 98 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -102.3375015258789, + "mean_logprob": -5.116875076293946, + "total_sequence_tokens": 96 + }, + { + "index": 2, + "tokens": 17, + "sum_logprob": -67.2765121459961, + "mean_logprob": -3.9574418909409466, + "total_sequence_tokens": 93 + }, + { + "index": 3, + "tokens": 23, + "sum_logprob": -110.58541870117188, + "mean_logprob": -4.8080616826596465, + "total_sequence_tokens": 99 + } + ], + "best_vs_second_margin": 0.8506197917187, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-043", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def stable_partition(items, predicate):\n matching = []\n other = []\n for item in items:\n if predicate(item):\n matching.append(item)\n else:\n other.append(item)\n", + "continuations": [ + " return sorted(items)\n", + " return matching or other\n", + " return matching + other\n", + " return zip(matching, other)\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -63.75392150878906, + "mean_logprob": -5.795811046253551, + "total_sequence_tokens": 116 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -48.059043884277344, + "mean_logprob": -5.3398937649197045, + "total_sequence_tokens": 114 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -51.10919952392578, + "mean_logprob": -5.678799947102864, + "total_sequence_tokens": 114 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -105.3359375, + "mean_logprob": -7.523995535714286, + "total_sequence_tokens": 119 + } + ], + "best_vs_second_margin": 0.3389061821831598, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-044", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def parse_key_value(lines):\n result = {}\n for line in lines:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n key, separator, value = line.partition(\"=\")\n if not separator:\n raise ValueError(\"missing equals sign\")\n", + "continuations": [ + " result[value] = separator\n return lines\n", + " result[key] = key\n return value\n", + " result.clear()\n return key\n", + " result[key.strip()] = value.strip()\n return result\n" + ], + "tags": [ + "python", + "parsing" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 22, + "sum_logprob": -106.98391723632812, + "mean_logprob": -4.862905328924006, + "total_sequence_tokens": 170 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -90.57785034179688, + "mean_logprob": -4.528892517089844, + "total_sequence_tokens": 168 + }, + { + "index": 2, + "tokens": 20, + "sum_logprob": -90.25782775878906, + "mean_logprob": -4.512891387939453, + "total_sequence_tokens": 168 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -145.58416748046875, + "mean_logprob": -4.549505233764648, + "total_sequence_tokens": 180 + } + ], + "best_vs_second_margin": 0.01600112915039098, + "context_truncated": false, + "elapsed_seconds": 0.0011421438757679425 + }, + { + "id": "code_completion-045", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def sliding_window_sums(numbers, width):\n if width <= 0 or width > len(numbers):\n return []\n current = sum(numbers[:width])\n result = [current]\n for index in range(width, len(numbers)):\n", + "continuations": [ + " current += numbers[index] - numbers[index - width]\n result.append(current)\n return result\n", + " current = numbers[index]\n return [current]\n", + " result.append(sum(numbers))\n return numbers\n", + " width += index\n return result[:width]\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 47, + "sum_logprob": -207.30294799804688, + "mean_logprob": -4.41070102123504, + "total_sequence_tokens": 148 + }, + { + "index": 1, + "tokens": 24, + "sum_logprob": -97.65843200683594, + "mean_logprob": -4.069101333618164, + "total_sequence_tokens": 125 + }, + { + "index": 2, + "tokens": 27, + "sum_logprob": -118.31358337402344, + "mean_logprob": -4.381984569408275, + "total_sequence_tokens": 128 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -108.87945556640625, + "mean_logprob": -4.35517822265625, + "total_sequence_tokens": 126 + } + ], + "best_vs_second_margin": 0.28607688903808626, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + }, + { + "id": "code_completion-046", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def retry(operation, attempts):\n last_error = None\n for _ in range(attempts):\n try:\n return operation()\n except Exception as error:\n last_error = error\n", + "continuations": [ + " return last_error\n", + " if last_error is not None:\n raise last_error\n raise ValueError(\"attempts must be positive\")\n", + " operation()\n return None\n", + " except last_error:\n return attempts\n" + ], + "tags": [ + "python", + "exceptions" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -40.72242736816406, + "mean_logprob": -4.5247141520182295, + "total_sequence_tokens": 101 + }, + { + "index": 1, + "tokens": 47, + "sum_logprob": -256.4456787109375, + "mean_logprob": -5.456291036402925, + "total_sequence_tokens": 139 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -56.00478744506836, + "mean_logprob": -4.000341960362026, + "total_sequence_tokens": 106 + }, + { + "index": 3, + "tokens": 20, + "sum_logprob": -70.40243530273438, + "mean_logprob": -3.520121765136719, + "total_sequence_tokens": 112 + } + ], + "best_vs_second_margin": 0.4802201952253071, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + }, + { + "id": "code_completion-047", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def normalize_rows(matrix):\n result = []\n for row in matrix:\n total = sum(row)\n if total == 0:\n result.append([0.0 for _ in row])\n else:\n", + "continuations": [ + " result.append(row)\n", + " result.append([total for _ in row])\n", + " result.append([value / total for value in row])\n return result\n", + " result = row / total\n return matrix\n" + ], + "tags": [ + "python", + "numeric" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 19, + "sum_logprob": -51.1166877746582, + "mean_logprob": -2.6903519881399056, + "total_sequence_tokens": 116 + }, + { + "index": 1, + "tokens": 27, + "sum_logprob": -86.87263488769531, + "mean_logprob": -3.2175049958405673, + "total_sequence_tokens": 124 + }, + { + "index": 2, + "tokens": 34, + "sum_logprob": -143.69935607910156, + "mean_logprob": -4.22645164938534, + "total_sequence_tokens": 131 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -86.53683471679688, + "mean_logprob": -3.461473388671875, + "total_sequence_tokens": 122 + } + ], + "best_vs_second_margin": 0.5271530077006616, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + }, + { + "id": "code_completion-048", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def merge_ranges(ranges):\n if not ranges:\n return []\n ordered = sorted(ranges)\n merged = [list(ordered[0])]\n for start, end in ordered[1:]:\n previous = merged[-1]\n if start <= previous[1]:\n previous[1] = max(previous[1], end)\n else:\n", + "continuations": [ + " previous[0] = start\n return ranges\n", + " merged = [[start, end]]\n return ordered\n", + " return previous\n return merged\n", + " merged.append([start, end])\n return [tuple(pair) for pair in merged]\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 25, + "sum_logprob": -77.07347106933594, + "mean_logprob": -3.0829388427734377, + "total_sequence_tokens": 171 + }, + { + "index": 1, + "tokens": 29, + "sum_logprob": -130.03201293945312, + "mean_logprob": -4.483862515153556, + "total_sequence_tokens": 175 + }, + { + "index": 2, + "tokens": 21, + "sum_logprob": -66.81196594238281, + "mean_logprob": -3.181522187732515, + "total_sequence_tokens": 167 + }, + { + "index": 3, + "tokens": 44, + "sum_logprob": -235.27816772460938, + "mean_logprob": -5.347231084650213, + "total_sequence_tokens": 190 + } + ], + "best_vs_second_margin": 0.09858334495907739, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + }, + { + "id": "code_completion-049", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def count_paths(grid, row=0, col=0, memo=None):\n if memo is None:\n memo = {}\n if row >= len(grid) or col >= len(grid[0]) or grid[row][col] == 1:\n return 0\n if (row, col) == (len(grid) - 1, len(grid[0]) - 1):\n return 1\n if (row, col) not in memo:\n", + "continuations": [ + " memo[(row, col)] = count_paths(grid, row + 1, col, memo) + count_paths(grid, row, col + 1, memo)\n return memo[(row, col)]\n", + " memo.clear()\n return 0\n", + " return count_paths(grid, row - 1, col - 1, memo)\n return memo\n", + " memo[(row, col)] = 1\n return len(memo)\n" + ], + "tags": [ + "python", + "dynamic_programming" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 71, + "sum_logprob": -327.5510559082031, + "mean_logprob": -4.613395153636664, + "total_sequence_tokens": 234 + }, + { + "index": 1, + "tokens": 22, + "sum_logprob": -87.46721649169922, + "mean_logprob": -3.97578256780451, + "total_sequence_tokens": 185 + }, + { + "index": 2, + "tokens": 38, + "sum_logprob": -154.33082580566406, + "mean_logprob": -4.061337521201686, + "total_sequence_tokens": 201 + }, + { + "index": 3, + "tokens": 32, + "sum_logprob": -135.2390899658203, + "mean_logprob": -4.226221561431885, + "total_sequence_tokens": 195 + } + ], + "best_vs_second_margin": 0.08555495339717645, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + }, + { + "id": "code_completion-050", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def compose(*functions):\n def composed(value):\n result = value\n for function in reversed(functions):\n", + "continuations": [ + " result = functions(result)\n return value\n", + " result = function(result)\n return result\n return composed\n", + " return function\n return functions\n return result\n", + " result.append(function)\n return result\n return functions\n" + ], + "tags": [ + "python", + "higher_order" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 29, + "sum_logprob": -117.11097717285156, + "mean_logprob": -4.038309557684537, + "total_sequence_tokens": 79 + }, + { + "index": 1, + "tokens": 35, + "sum_logprob": -135.05490112304688, + "mean_logprob": -3.858711460658482, + "total_sequence_tokens": 85 + }, + { + "index": 2, + "tokens": 30, + "sum_logprob": -94.80081939697266, + "mean_logprob": -3.1600273132324217, + "total_sequence_tokens": 80 + }, + { + "index": 3, + "tokens": 36, + "sum_logprob": -163.29800415039062, + "mean_logprob": -4.536055670844184, + "total_sequence_tokens": 86 + } + ], + "best_vs_second_margin": 0.6986841474260603, + "context_truncated": false, + "elapsed_seconds": 0.001319195003209946 + } + ] +}