| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 28.57142857 | |
| 8 37.50000000 | |
| 9 33.33333333 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 33.33333333 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 26.66666667 | |
| 16 31.25000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 36.36363636 | |
| 23 39.13043478 | |
| 24 41.66666667 | |
| 25 44.00000000 | |
| 26 42.30769231 | |
| 27 44.44444444 | |
| 28 46.42857143 | |
| 29 44.82758621 | |
| 30 43.33333333 | |
| 31 45.16129032 | |
| 32 46.87500000 | |
| 33 45.45454545 | |
| 34 47.05882353 | |
| 35 48.57142857 | |
| 36 47.22222222 | |
| 37 45.94594595 | |
| 38 47.36842105 | |
| 39 46.15384615 | |
| 40 45.00000000 | |
| 41 43.90243902 | |
| 42 42.85714286 | |
| 43 44.18604651 | |
| 44 43.18181818 | |
| 45 44.44444444 | |
| 46 43.47826087 | |
| 47 42.55319149 | |
| 48 43.75000000 | |
| 49 42.85714286 | |
| 50 44.00000000 | |
| 51 43.13725490 | |
| 52 42.30769231 | |
| 53 41.50943396 | |
| 54 40.74074074 | |
| 55 41.81818182 | |
| 56 42.85714286 | |
| 57 43.85964912 | |
| 58 43.10344828 | |
| 59 44.06779661 | |
| 60 43.33333333 | |
| 61 44.26229508 | |
| 62 43.54838710 | |
| 63 42.85714286 | |
| 64 43.75000000 | |
| 65 43.07692308 | |
| 66 43.93939394 | |
| 67 44.77611940 | |
| 68 44.11764706 | |
| 69 43.47826087 | |
| 70 42.85714286 | |
| 71 42.25352113 | |
| 72 43.05555556 | |
| 73 42.46575342 | |
| 74 43.24324324 | |
| 75 44.00000000 | |
| 76 44.73684211 | |
| 77 45.45454545 | |
| 78 46.15384615 | |
| 79 45.56962025 | |
| 80 45.00000000 | |
| 81 45.67901235 | |
| 82 45.12195122 | |
| 83 45.78313253 | |
| 84 45.23809524 | |
| 85 44.70588235 | |
| 86 45.34883721 | |
| 87 45.97701149 | |
| 88 46.59090909 | |
| 89 46.06741573 | |
| 90 45.55555556 | |
| 91 45.05494505 | |
| 92 44.56521739 | |
| 93 44.08602151 | |
| 94 44.68085106 | |
| 95 44.21052632 | |
| 96 44.79166667 | |
| 97 44.32989691 | |
| 98 44.89795918 | |
| 99 44.44444444 | |
| 100 44.00000000 | |
| 101 44.55445545 | |
| 102 44.11764706 | |
| 103 43.68932039 | |
| 104 44.23076923 | |
| 105 44.76190476 | |
| 106 44.33962264 | |
| 107 43.92523364 | |
| 108 43.51851852 | |
| 109 43.11926606 | |
| 110 43.63636364 | |
| 111 44.14414414 | |
| 112 44.64285714 | |
| 113 44.24778761 | |
| 114 44.73684211 | |
| 115 45.21739130 | |
| 116 45.68965517 | |
| 117 45.29914530 | |
| 118 44.91525424 | |
| 119 44.53781513 | |
| 120 44.16666667 | |
| 121 44.62809917 | |
| 122 44.26229508 | |
| 123 43.90243902 | |
| 124 44.35483871 | |
| 125 44.80000000 | |
| 126 44.44444444 | |
| 127 44.09448819 | |
| 128 44.53125000 | |
| 129 44.18604651 | |
| 130 44.61538462 | |
| 131 45.03816794 | |
| 132 45.45454545 | |
| 133 45.11278195 | |
| 134 44.77611940 | |
| 135 44.44444444 | |
| 136 44.85294118 | |
| 137 45.25547445 | |
| 138 44.92753623 | |
| 139 45.32374101 | |
| 140 45.71428571 | |
| 141 46.09929078 | |
| 142 46.47887324 | |
| 143 46.85314685 | |
| 144 46.52777778 | |
| 145 46.89655172 | |
| 146 47.26027397 | |
| 147 46.93877551 | |
| 148 47.29729730 | |
| 149 47.65100671 | |
| 150 47.33333333 | |
| 151 47.01986755 | |
| 152 47.36842105 | |
| 153 47.05882353 | |
| 154 46.75324675 | |
| 155 46.45161290 | |
| 156 46.15384615 | |
| 157 46.49681529 | |
| 158 46.83544304 | |
| 159 46.54088050 | |
| 160 46.87500000 | |
| 161 46.58385093 | |
| 162 46.29629630 | |
| 163 46.01226994 | |
| 164 46.34146341 | |
| 165 46.06060606 | |
| 166 45.78313253 | |
| 167 46.10778443 | |
| 168 45.83333333 | |
| 169 46.15384615 | |
| 170 45.88235294 | |
| 171 46.19883041 | |
| 172 45.93023256 | |
| 173 45.66473988 | |
| 174 45.40229885 | |
| 175 45.14285714 | |
| 176 45.45454545 | |
| 177 45.19774011 | |
| 178 45.50561798 | |
| 179 45.25139665 | |
| 180 45.00000000 | |
| 181 44.75138122 | |
| 182 44.50549451 | |
| 183 44.80874317 | |
| 184 44.56521739 | |
| 185 44.86486486 | |
| 186 45.16129032 | |
| 187 45.45454545 | |
| 188 45.21276596 | |
| 189 45.50264550 | |
| 190 45.26315789 | |
| 191 45.54973822 | |
| 192 45.31250000 | |
| 193 45.07772021 | |
| 194 44.84536082 | |
| 195 45.12820513 | |
| 196 45.40816327 | |
| 197 45.17766497 | |
| 198 45.45454545 | |
| 199 45.22613065 | |
| 200 45.00000000 | |
| 201 44.77611940 | |
| 202 44.55445545 | |
| 203 44.33497537 | |
| 204 44.11764706 | |
| 205 44.39024390 | |
| 206 44.17475728 | |
| 207 44.44444444 | |
| 208 44.23076923 | |
| 209 44.01913876 | |
| 210 43.80952381 | |
| 211 43.60189573 | |
| 212 43.39622642 | |
| 213 43.19248826 | |
| 214 42.99065421 | |
| 215 43.25581395 | |
| 216 43.05555556 | |
| 217 42.85714286 | |
| 218 43.11926606 | |
| 219 43.37899543 | |
| 220 43.18181818 | |
| 221 43.43891403 | |
| 222 43.24324324 | |
| 223 43.49775785 | |
| 224 43.75000000 | |
| 225 44.00000000 | |
| 226 44.24778761 | |
| 227 44.05286344 | |
| 228 44.29824561 | |
| 229 44.10480349 | |
| 230 43.91304348 | |
| 231 43.72294372 | |
| 232 43.96551724 | |
| 233 43.77682403 | |
| 234 44.01709402 | |
| 235 43.82978723 | |
| 236 43.64406780 | |
| 237 43.45991561 | |
| 238 43.27731092 | |
| 239 43.09623431 | |
| 240 43.33333333 | |
| 241 43.15352697 | |
| 242 43.38842975 | |
| 243 43.62139918 | |
| 244 43.85245902 | |
| 245 44.08163265 | |
| 246 43.90243902 | |
| 247 43.72469636 | |
| 248 43.95161290 | |
| 249 43.77510040 | |
| 250 43.60000000 | |
| 251 43.42629482 | |
| 252 43.25396825 | |
| 253 43.47826087 | |
| 254 43.30708661 | |
| 255 43.13725490 | |
| 256 43.35937500 | |
| 257 43.57976654 | |
| 258 43.41085271 | |
| 259 43.24324324 | |
| 260 43.46153846 | |
| 261 43.67816092 | |
| 262 43.51145038 | |
| 263 43.34600760 | |
| 264 43.56060606 | |
| 265 43.39622642 | |
| 266 43.60902256 | |
| 267 43.82022472 | |
| 268 43.65671642 | |
| 269 43.86617100 | |
| 270 44.07407407 | |
| 271 43.91143911 | |
| 272 44.11764706 | |
| 273 43.95604396 | |
| 274 43.79562044 | |
| 275 43.63636364 | |
| 276 43.84057971 | |
| 277 44.04332130 | |
| 278 44.24460432 | |
| 279 44.08602151 | |
| 280 44.28571429 | |
| 281 44.12811388 | |
| 282 43.97163121 | |
| 283 43.81625442 | |
| 284 43.66197183 | |
| 285 43.85964912 | |
| 286 43.70629371 | |
| 287 43.90243902 | |
| 288 43.75000000 | |
| 289 43.59861592 | |
| 290 43.79310345 | |
| 291 43.98625430 | |
| 292 44.17808219 | |
| 293 44.36860068 | |
| 294 44.21768707 | |
| 295 44.06779661 | |
| 296 43.91891892 | |
| 297 43.77104377 | |
| 298 43.95973154 | |
| 299 44.14715719 | |
| 300 44.33333333 | |
| 301 44.51827243 | |
| 302 44.70198675 | |
| 303 44.55445545 | |
| 304 44.73684211 | |
| 305 44.91803279 | |
| 306 45.09803922 | |
| 307 45.27687296 | |
| 308 45.45454545 | |
| 309 45.63106796 | |
| 310 45.48387097 | |
| 311 45.65916399 | |
| 312 45.83333333 | |
| 313 46.00638978 | |
| 314 46.17834395 | |
| 315 46.03174603 | |
| 316 45.88607595 | |
| 317 45.74132492 | |
| 318 45.59748428 | |
| 319 45.76802508 | |
| 320 45.62500000 | |
| 321 45.48286604 | |
| 322 45.65217391 | |
| 323 45.51083591 | |
| 324 45.37037037 | |
| 325 45.53846154 | |
| 326 45.39877301 | |
| 327 45.56574924 | |
| 328 45.73170732 | |
| 329 45.89665653 | |
| 330 45.75757576 | |
| 331 45.61933535 | |
| 332 45.48192771 | |
| 333 45.64564565 | |
| 334 45.50898204 | |
| 335 45.67164179 | |
| 336 45.83333333 | |
| 337 45.69732938 | |
| 338 45.56213018 | |
| 339 45.42772861 | |
| 340 45.58823529 | |
| 341 45.74780059 | |
| 342 45.90643275 | |
| 343 45.77259475 | |
| 344 45.93023256 | |
| 345 45.79710145 | |
| 346 45.95375723 | |
| 347 46.10951009 | |
| 348 45.97701149 | |
| 349 45.84527221 | |
| 350 46.00000000 | |
| 351 45.86894587 | |
| 352 45.73863636 | |
| 353 45.89235127 | |
| 354 45.76271186 | |
| 355 45.63380282 | |
| 356 45.78651685 | |
| 357 45.65826331 | |
| 358 45.53072626 | |
| 359 45.40389972 | |
| 360 45.55555556 | |
| 361 45.70637119 | |
| 362 45.85635359 | |
| 363 45.73002755 | |
| 364 45.87912088 | |
| 365 45.75342466 | |
| 366 45.62841530 | |
| 367 45.50408719 | |
| 368 45.38043478 | |
| 369 45.25745257 | |
| 370 45.40540541 | |
| 371 45.55256065 | |
| 372 45.43010753 | |
| 373 45.30831099 | |
| 374 45.45454545 | |
| 375 45.60000000 | |
| 376 45.47872340 | |
| 377 45.35809019 | |
| 378 45.50264550 | |
| 379 45.38258575 | |
| 380 45.26315789 | |
| 381 45.40682415 | |
| 382 45.54973822 | |
| 383 45.69190601 | |
| 384 45.57291667 | |
| 385 45.71428571 | |
| 386 45.59585492 | |
| 387 45.47803618 | |
| 388 45.36082474 | |
| 389 45.50128535 | |
| 390 45.38461538 | |
| 391 45.52429668 | |
| 392 45.66326531 | |
| 393 45.54707379 | |
| 394 45.43147208 | |
| 395 45.56962025 | |
| 396 45.70707071 | |
| 397 45.84382872 | |
| 398 45.72864322 | |
| 399 45.86466165 | |
| 400 45.75000000 | |
| 401 45.88528678 | |
| 402 45.77114428 | |
| 403 45.65756824 | |
| 404 45.79207921 | |
| 405 45.92592593 | |
| 406 45.81280788 | |
| 407 45.70024570 | |
| 408 45.58823529 | |
| 409 45.72127139 | |
| 410 45.60975610 | |
| 411 45.49878345 | |
| 412 45.38834951 | |
| 413 45.27845036 | |
| 414 45.41062802 | |
| 415 45.54216867 | |
| 416 45.43269231 | |
| 417 45.32374101 | |
| 418 45.21531100 | |
| 419 45.34606205 | |
| 420 45.47619048 | |
| 421 45.60570071 | |
| 422 45.49763033 | |
| 423 45.39007092 | |
| 424 45.51886792 | |
| 425 45.41176471 | |
| 426 45.30516432 | |
| 427 45.19906323 | |
| 428 45.09345794 | |
| 429 45.22144522 | |
| 430 45.11627907 | |
| 431 45.24361949 | |
| 432 45.37037037 | |
| 433 45.26558891 | |
| 434 45.16129032 | |
| 435 45.05747126 | |
| 436 44.95412844 | |
| 437 44.85125858 | |
| 438 44.97716895 | |
| 439 45.10250569 | |
| 440 45.00000000 | |
| 441 44.89795918 | |
| 442 44.79638009 | |
| 443 44.69525959 | |
| 444 44.59459459 | |
| 445 44.49438202 | |
| 446 44.39461883 | |
| 447 44.51901566 | |
| 448 44.41964286 | |
| 449 44.32071269 | |
| 450 44.44444444 | |
| 451 44.56762749 | |
| 452 44.46902655 | |
| 453 44.59161148 | |
| 454 44.49339207 | |
| 455 44.39560440 | |
| 456 44.51754386 | |
| 457 44.42013129 | |
| 458 44.32314410 | |
| 459 44.22657952 | |
| 460 44.34782609 | |
| 461 44.46854664 | |
| 462 44.58874459 | |
| 463 44.70842333 | |
| 464 44.82758621 | |
| 465 44.73118280 | |
| 466 44.63519313 | |
| 467 44.75374732 | |
| 468 44.87179487 | |
| 469 44.98933902 | |
| 470 44.89361702 | |
| 471 44.79830149 | |
| 472 44.70338983 | |
| 473 44.60887949 | |
| 474 44.51476793 | |
| 475 44.42105263 | |
| 476 44.53781513 | |
| 477 44.44444444 | |
| 478 44.56066946 | |
| 479 44.46764092 | |
| 480 44.58333333 | |
| 481 44.49064449 | |
| 482 44.39834025 | |
| 483 44.30641822 | |
| 484 44.21487603 | |
| 485 44.12371134 | |
| 486 44.03292181 | |
| 487 44.14784394 | |
| 488 44.05737705 | |
| 489 43.96728016 | |
| 490 44.08163265 | |
| 491 43.99185336 | |
| 492 43.90243902 | |
| 493 44.01622718 | |
| 494 44.12955466 | |
| 495 44.24242424 | |
| 496 44.35483871 | |
| 497 44.26559356 | |
| 498 44.17670683 | |
| 499 44.28857715 | |
| 500 44.20000000 | |
| 501 44.11177645 | |
| 502 44.02390438 | |
| 503 44.13518887 | |
| 504 44.24603175 | |
| 505 44.15841584 | |
| 506 44.26877470 | |
| 507 44.18145957 | |
| 508 44.09448819 | |
| 509 44.00785855 | |
| 510 43.92156863 | |
| 511 43.83561644 | |
| 512 43.75000000 | |
| 513 43.66471735 | |
| 514 43.57976654 | |
| 515 43.68932039 | |
| 516 43.79844961 | |
| 517 43.90715667 | |
| 518 44.01544402 | |
| 519 43.93063584 | |
| 520 44.03846154 | |
| 521 43.95393474 | |
| 522 44.06130268 | |
| 523 43.97705545 | |
| 524 44.08396947 | |
| 525 44.00000000 | |
| 526 43.91634981 | |
| 527 44.02277040 | |
| 528 44.12878788 | |
| 529 44.04536862 | |
| 530 44.15094340 | |
| 531 44.06779661 | |
| 532 43.98496241 | |
| 533 43.90243902 | |
| 534 44.00749064 | |
| 535 44.11214953 | |
| 536 44.02985075 | |
| 537 44.13407821 | |
| 538 44.23791822 | |
| 539 44.15584416 | |
| 540 44.07407407 | |
| 541 44.17744917 | |
| 542 44.28044280 | |
| 543 44.19889503 | |
| 544 44.30147059 | |
| 545 44.22018349 | |
| 546 44.13919414 | |
| 547 44.24131627 | |
| 548 44.34306569 | |
| 549 44.26229508 | |
| 550 44.18181818 | |
| 551 44.28312160 | |
| 552 44.38405797 | |
| 553 44.30379747 | |
| 554 44.40433213 | |
| 555 44.32432432 | |
| 556 44.42446043 | |
| 557 44.34470377 | |
| 558 44.44444444 | |
| 559 44.54382826 | |
| 560 44.46428571 | |
| 561 44.38502674 | |
| 562 44.48398577 | |
| 563 44.58259325 | |
| 564 44.50354610 | |
| 565 44.60176991 | |
| 566 44.52296820 | |
| 567 44.44444444 | |
| 568 44.36619718 | |
| 569 44.28822496 | |
| 570 44.38596491 | |
| 571 44.30823117 | |
| 572 44.40559441 | |
| 573 44.32809773 | |
| 574 44.25087108 | |
| 575 44.17391304 | |
| 576 44.27083333 | |
| 577 44.36741768 | |
| 578 44.46366782 | |
| 579 44.38687392 | |
| 580 44.48275862 | |
| 581 44.57831325 | |
| 582 44.50171821 | |
| 583 44.59691252 | |
| 584 44.69178082 | |
| 585 44.78632479 | |
| 586 44.88054608 | |
| 587 44.80408859 | |
| 588 44.72789116 | |
| 589 44.65195246 | |
| 590 44.57627119 | |
| 591 44.50084602 | |
| 592 44.59459459 | |
| 593 44.68802698 | |
| 594 44.78114478 | |
| 595 44.70588235 | |
| 596 44.79865772 | |
| 597 44.72361809 | |
| 598 44.64882943 | |
| 599 44.74123539 | |
| 600 44.66666667 | |
| 601 44.75873544 | |
| 602 44.68438538 | |
| 603 44.77611940 | |
| 604 44.70198675 | |
| 605 44.62809917 | |
| 606 44.55445545 | |
| 607 44.64579901 | |
| 608 44.57236842 | |
| 609 44.49917898 | |
| 610 44.42622951 | |
| 611 44.51718494 | |
| 612 44.44444444 | |
| 613 44.53507341 | |
| 614 44.62540717 | |
| 615 44.55284553 | |
| 616 44.64285714 | |
| 617 44.57050243 | |
| 618 44.66019417 | |
| 619 44.58804523 | |
| 620 44.51612903 | |
| 621 44.44444444 | |
| 622 44.37299035 | |
| 623 44.30176565 | |
| 624 44.23076923 | |
| 625 44.16000000 | |
| 626 44.24920128 | |
| 627 44.33811802 | |
| 628 44.26751592 | |
| 629 44.35612083 | |
| 630 44.28571429 | |
| 631 44.21553090 | |
| 632 44.30379747 | |
| 633 44.23380727 | |
| 634 44.16403785 | |
| 635 44.25196850 | |
| 636 44.18238994 | |
| 637 44.11302983 | |
| 638 44.20062696 | |
| 639 44.28794992 | |
| 640 44.37500000 | |
| 641 44.30577223 | |
| 642 44.39252336 | |
| 643 44.32348367 | |
| 644 44.40993789 | |
| 645 44.34108527 | |
| 646 44.27244582 | |
| 647 44.20401855 | |
| 648 44.13580247 | |
| 649 44.06779661 | |
| 650 44.15384615 | |
| 651 44.08602151 | |
| 652 44.17177914 | |
| 653 44.10413476 | |
| 654 44.18960245 | |
| 655 44.12213740 | |
| 656 44.20731707 | |
| 657 44.29223744 | |
| 658 44.37689970 | |
| 659 44.30955994 | |
| 660 44.24242424 | |
| 661 44.17549168 | |
| 662 44.10876133 | |
| 663 44.04223228 | |
| 664 43.97590361 | |
| 665 43.90977444 | |
| 666 43.84384384 | |
| 667 43.92803598 | |
| 668 44.01197605 | |
| 669 44.09566517 | |
| 670 44.02985075 | |
| 671 43.96423249 | |
| 672 43.89880952 | |
| 673 43.83358098 | |
| 674 43.76854599 | |
| 675 43.70370370 | |
| 676 43.63905325 | |
| 677 43.57459380 | |
| 678 43.65781711 | |
| 679 43.59351988 | |
| 680 43.67647059 | |
| 681 43.75917768 | |
| 682 43.69501466 | |
| 683 43.63103953 | |
| 684 43.71345029 | |
| 685 43.64963504 | |
| 686 43.58600583 | |
| 687 43.52256186 | |
| 688 43.45930233 | |
| 689 43.39622642 | |
| 690 43.33333333 | |
| 691 43.41534009 | |
| 692 43.35260116 | |
| 693 43.29004329 | |
| 694 43.22766571 | |
| 695 43.16546763 | |
| 696 43.24712644 | |
| 697 43.18507891 | |
| 698 43.12320917 | |
| 699 43.20457797 | |
| 700 43.28571429 | |
| 701 43.36661912 | |
| 702 43.44729345 | |
| 703 43.52773826 | |
| 704 43.46590909 | |
| 705 43.40425532 | |
| 706 43.48441926 | |
| 707 43.42291372 | |
| 708 43.36158192 | |
| 709 43.44146685 | |
| 710 43.52112676 | |
| 711 43.60056259 | |
| 712 43.67977528 | |
| 713 43.75876578 | |
| 714 43.83753501 | |
| 715 43.77622378 | |
| 716 43.71508380 | |
| 717 43.65411437 | |
| 718 43.59331476 | |
| 719 43.67176634 | |
| 720 43.61111111 | |
| 721 43.55062413 | |
| 722 43.62880886 | |
| 723 43.70677732 | |
| 724 43.64640884 | |
| 725 43.72413793 | |
| 726 43.80165289 | |
| 727 43.74140303 | |
| 728 43.81868132 | |
| 729 43.75857339 | |
| 730 43.69863014 | |
| 731 43.63885089 | |
| 732 43.57923497 | |
| 733 43.65620737 | |
| 734 43.73297003 | |
| 735 43.67346939 | |
| 736 43.61413043 | |
| 737 43.55495251 | |
| 738 43.63143631 | |
| 739 43.70771313 | |
| 740 43.78378378 | |
| 741 43.72469636 | |
| 742 43.80053908 | |
| 743 43.87617766 | |
| 744 43.95161290 | |
| 745 43.89261745 | |
| 746 43.83378016 | |
| 747 43.77510040 | |
| 748 43.71657754 | |
| 749 43.65821095 | |
| 750 43.73333333 | |
| Final result: 43.7333 +/- 1.8126 | |
| Random chance: 25.0000 +/- 1.5822 | |