| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 31.57894737 | |
| 20 30.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 26.08695652 | |
| 24 29.16666667 | |
| 25 28.00000000 | |
| 26 26.92307692 | |
| 27 25.92592593 | |
| 28 25.00000000 | |
| 29 27.58620690 | |
| 30 26.66666667 | |
| 31 25.80645161 | |
| 32 28.12500000 | |
| 33 30.30303030 | |
| 34 29.41176471 | |
| 35 28.57142857 | |
| 36 27.77777778 | |
| 37 29.72972973 | |
| 38 28.94736842 | |
| 39 30.76923077 | |
| 40 30.00000000 | |
| 41 29.26829268 | |
| 42 28.57142857 | |
| 43 30.23255814 | |
| 44 31.81818182 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 30.61224490 | |
| 50 32.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 32.14285714 | |
| 57 31.57894737 | |
| 58 31.03448276 | |
| 59 30.50847458 | |
| 60 31.66666667 | |
| 61 31.14754098 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 32.81250000 | |
| 65 32.30769231 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 32.43243243 | |
| 75 32.00000000 | |
| 76 32.89473684 | |
| 77 32.46753247 | |
| 78 32.05128205 | |
| 79 31.64556962 | |
| 80 32.50000000 | |
| 81 32.09876543 | |
| 82 32.92682927 | |
| 83 32.53012048 | |
| 84 32.14285714 | |
| 85 31.76470588 | |
| 86 31.39534884 | |
| 87 31.03448276 | |
| 88 30.68181818 | |
| 89 31.46067416 | |
| 90 31.11111111 | |
| 91 30.76923077 | |
| 92 30.43478261 | |
| 93 30.10752688 | |
| 94 29.78723404 | |
| 95 29.47368421 | |
| 96 30.20833333 | |
| 97 30.92783505 | |
| 98 30.61224490 | |
| 99 31.31313131 | |
| 100 32.00000000 | |
| 101 32.67326733 | |
| 102 32.35294118 | |
| 103 33.00970874 | |
| 104 32.69230769 | |
| 105 32.38095238 | |
| 106 32.07547170 | |
| 107 32.71028037 | |
| 108 33.33333333 | |
| 109 33.94495413 | |
| 110 34.54545455 | |
| 111 34.23423423 | |
| 112 34.82142857 | |
| 113 35.39823009 | |
| 114 35.96491228 | |
| 115 35.65217391 | |
| 116 36.20689655 | |
| 117 36.75213675 | |
| 118 36.44067797 | |
| 119 36.13445378 | |
| 120 35.83333333 | |
| 121 36.36363636 | |
| 122 36.06557377 | |
| 123 36.58536585 | |
| 124 36.29032258 | |
| 125 36.80000000 | |
| 126 36.50793651 | |
| 127 36.22047244 | |
| 128 35.93750000 | |
| 129 36.43410853 | |
| 130 36.92307692 | |
| 131 36.64122137 | |
| 132 36.36363636 | |
| 133 36.09022556 | |
| 134 35.82089552 | |
| 135 36.29629630 | |
| 136 36.02941176 | |
| 137 36.49635036 | |
| 138 36.95652174 | |
| 139 37.41007194 | |
| 140 37.14285714 | |
| 141 37.58865248 | |
| 142 37.32394366 | |
| 143 37.06293706 | |
| 144 37.50000000 | |
| 145 37.93103448 | |
| 146 37.67123288 | |
| 147 37.41496599 | |
| 148 37.16216216 | |
| 149 36.91275168 | |
| 150 36.66666667 | |
| 151 37.08609272 | |
| 152 36.84210526 | |
| 153 36.60130719 | |
| 154 36.36363636 | |
| 155 36.12903226 | |
| 156 36.53846154 | |
| 157 36.94267516 | |
| 158 37.34177215 | |
| 159 37.73584906 | |
| 160 37.50000000 | |
| 161 37.88819876 | |
| 162 37.65432099 | |
| 163 37.42331288 | |
| 164 37.19512195 | |
| 165 37.57575758 | |
| 166 37.95180723 | |
| 167 37.72455090 | |
| 168 37.50000000 | |
| 169 37.27810651 | |
| 170 37.05882353 | |
| 171 36.84210526 | |
| 172 36.62790698 | |
| 173 36.41618497 | |
| 174 36.78160920 | |
| 175 36.57142857 | |
| 176 36.36363636 | |
| 177 36.15819209 | |
| 178 35.95505618 | |
| 179 35.75418994 | |
| 180 35.55555556 | |
| 181 35.91160221 | |
| 182 35.71428571 | |
| 183 35.51912568 | |
| 184 35.32608696 | |
| 185 35.67567568 | |
| 186 36.02150538 | |
| 187 35.82887701 | |
| 188 35.63829787 | |
| 189 35.44973545 | |
| 190 35.26315789 | |
| 191 35.07853403 | |
| 192 34.89583333 | |
| 193 34.71502591 | |
| 194 35.05154639 | |
| 195 34.87179487 | |
| 196 34.69387755 | |
| 197 35.02538071 | |
| 198 35.35353535 | |
| 199 35.17587940 | |
| 200 35.50000000 | |
| 201 35.82089552 | |
| 202 36.13861386 | |
| 203 35.96059113 | |
| 204 35.78431373 | |
| 205 36.09756098 | |
| 206 35.92233010 | |
| 207 36.23188406 | |
| 208 36.05769231 | |
| 209 35.88516746 | |
| 210 35.71428571 | |
| 211 35.54502370 | |
| 212 35.37735849 | |
| 213 35.21126761 | |
| 214 35.04672897 | |
| 215 34.88372093 | |
| 216 35.18518519 | |
| 217 35.02304147 | |
| 218 34.86238532 | |
| 219 34.70319635 | |
| 220 34.54545455 | |
| 221 34.84162896 | |
| 222 34.68468468 | |
| 223 34.97757848 | |
| 224 35.26785714 | |
| 225 35.55555556 | |
| 226 35.39823009 | |
| 227 35.68281938 | |
| 228 35.52631579 | |
| 229 35.37117904 | |
| 230 35.21739130 | |
| 231 35.06493506 | |
| 232 35.34482759 | |
| 233 35.19313305 | |
| 234 35.04273504 | |
| 235 34.89361702 | |
| 236 35.16949153 | |
| 237 35.02109705 | |
| 238 35.29411765 | |
| 239 35.14644351 | |
| 240 35.00000000 | |
| 241 34.85477178 | |
| 242 34.71074380 | |
| 243 34.56790123 | |
| 244 34.42622951 | |
| 245 34.28571429 | |
| 246 34.14634146 | |
| 247 34.41295547 | |
| 248 34.67741935 | |
| 249 34.93975904 | |
| 250 34.80000000 | |
| 251 35.05976096 | |
| 252 34.92063492 | |
| 253 34.78260870 | |
| 254 35.03937008 | |
| 255 35.29411765 | |
| 256 35.15625000 | |
| 257 35.01945525 | |
| 258 34.88372093 | |
| 259 34.74903475 | |
| 260 34.61538462 | |
| 261 34.86590038 | |
| 262 34.73282443 | |
| 263 34.60076046 | |
| 264 34.84848485 | |
| 265 35.09433962 | |
| 266 34.96240602 | |
| 267 35.20599251 | |
| 268 35.07462687 | |
| 269 34.94423792 | |
| 270 34.81481481 | |
| 271 34.68634686 | |
| 272 34.55882353 | |
| 273 34.79853480 | |
| 274 35.03649635 | |
| 275 34.90909091 | |
| 276 34.78260870 | |
| 277 34.65703971 | |
| 278 34.53237410 | |
| 279 34.76702509 | |
| 280 35.00000000 | |
| 281 35.23131673 | |
| 282 35.10638298 | |
| 283 34.98233216 | |
| 284 35.21126761 | |
| 285 35.08771930 | |
| 286 35.31468531 | |
| 287 35.54006969 | |
| 288 35.76388889 | |
| 289 35.64013841 | |
| 290 35.51724138 | |
| 291 35.73883162 | |
| 292 35.61643836 | |
| 293 35.83617747 | |
| 294 35.71428571 | |
| 295 35.93220339 | |
| 296 36.14864865 | |
| 297 36.02693603 | |
| 298 35.90604027 | |
| 299 35.78595318 | |
| 300 35.66666667 | |
| 301 35.54817276 | |
| 302 35.43046358 | |
| 303 35.64356436 | |
| 304 35.52631579 | |
| 305 35.40983607 | |
| 306 35.29411765 | |
| 307 35.17915309 | |
| 308 35.38961039 | |
| 309 35.27508091 | |
| 310 35.16129032 | |
| 311 35.04823151 | |
| 312 35.25641026 | |
| 313 35.14376997 | |
| 314 35.03184713 | |
| 315 35.23809524 | |
| 316 35.12658228 | |
| 317 35.01577287 | |
| 318 34.90566038 | |
| 319 34.79623824 | |
| 320 34.68750000 | |
| 321 34.57943925 | |
| 322 34.47204969 | |
| 323 34.36532508 | |
| 324 34.25925926 | |
| 325 34.15384615 | |
| 326 34.04907975 | |
| 327 34.25076453 | |
| 328 34.45121951 | |
| 329 34.34650456 | |
| 330 34.54545455 | |
| 331 34.44108761 | |
| 332 34.63855422 | |
| 333 34.53453453 | |
| 334 34.43113772 | |
| 335 34.62686567 | |
| 336 34.82142857 | |
| 337 34.71810089 | |
| 338 34.61538462 | |
| 339 34.80825959 | |
| 340 34.70588235 | |
| 341 34.89736070 | |
| 342 35.08771930 | |
| 343 34.98542274 | |
| 344 35.17441860 | |
| 345 35.36231884 | |
| 346 35.26011561 | |
| 347 35.44668588 | |
| 348 35.34482759 | |
| 349 35.24355301 | |
| 350 35.14285714 | |
| 351 35.04273504 | |
| 352 34.94318182 | |
| 353 34.84419263 | |
| 354 34.74576271 | |
| 355 34.92957746 | |
| 356 34.83146067 | |
| 357 35.01400560 | |
| 358 34.91620112 | |
| 359 34.81894150 | |
| 360 34.72222222 | |
| 361 34.90304709 | |
| 362 34.80662983 | |
| 363 34.71074380 | |
| 364 34.61538462 | |
| 365 34.52054795 | |
| 366 34.42622951 | |
| 367 34.60490463 | |
| 368 34.51086957 | |
| 369 34.41734417 | |
| 370 34.32432432 | |
| 371 34.23180593 | |
| 372 34.40860215 | |
| 373 34.31635389 | |
| 374 34.49197861 | |
| 375 34.40000000 | |
| 376 34.30851064 | |
| 377 34.48275862 | |
| 378 34.39153439 | |
| 379 34.30079156 | |
| 380 34.21052632 | |
| 381 34.12073491 | |
| 382 34.29319372 | |
| 383 34.20365535 | |
| 384 34.11458333 | |
| 385 34.02597403 | |
| 386 33.93782383 | |
| 387 33.85012920 | |
| 388 34.02061856 | |
| 389 33.93316195 | |
| 390 33.84615385 | |
| 391 33.75959079 | |
| 392 33.67346939 | |
| 393 33.58778626 | |
| 394 33.75634518 | |
| 395 33.67088608 | |
| 396 33.83838384 | |
| 397 34.00503778 | |
| 398 33.91959799 | |
| 399 33.83458647 | |
| 400 33.75000000 | |
| 401 33.91521197 | |
| 402 34.07960199 | |
| 403 34.24317618 | |
| 404 34.15841584 | |
| 405 34.07407407 | |
| 406 33.99014778 | |
| 407 33.90663391 | |
| 408 33.82352941 | |
| 409 33.74083130 | |
| 410 33.65853659 | |
| 411 33.57664234 | |
| 412 33.73786408 | |
| 413 33.89830508 | |
| 414 34.05797101 | |
| 415 34.21686747 | |
| 416 34.13461538 | |
| 417 34.05275779 | |
| 418 33.97129187 | |
| 419 33.89021480 | |
| 420 34.04761905 | |
| 421 34.20427553 | |
| 422 34.12322275 | |
| 423 34.04255319 | |
| 424 33.96226415 | |
| 425 33.88235294 | |
| 426 33.80281690 | |
| 427 33.72365340 | |
| 428 33.64485981 | |
| 429 33.56643357 | |
| 430 33.48837209 | |
| 431 33.41067285 | |
| 432 33.56481481 | |
| 433 33.71824480 | |
| 434 33.87096774 | |
| 435 33.79310345 | |
| 436 33.71559633 | |
| 437 33.63844394 | |
| 438 33.78995434 | |
| 439 33.94077449 | |
| 440 33.86363636 | |
| 441 33.78684807 | |
| 442 33.71040724 | |
| 443 33.63431151 | |
| 444 33.78378378 | |
| 445 33.70786517 | |
| 446 33.63228700 | |
| 447 33.55704698 | |
| 448 33.48214286 | |
| 449 33.63028953 | |
| 450 33.77777778 | |
| 451 33.70288248 | |
| 452 33.62831858 | |
| 453 33.77483444 | |
| 454 33.70044053 | |
| 455 33.62637363 | |
| 456 33.55263158 | |
| 457 33.47921225 | |
| 458 33.40611354 | |
| 459 33.33333333 | |
| 460 33.26086957 | |
| 461 33.18872017 | |
| 462 33.11688312 | |
| 463 33.26133909 | |
| 464 33.40517241 | |
| 465 33.54838710 | |
| 466 33.47639485 | |
| 467 33.40471092 | |
| 468 33.54700855 | |
| 469 33.47547974 | |
| 470 33.40425532 | |
| 471 33.54564756 | |
| 472 33.47457627 | |
| 473 33.40380550 | |
| 474 33.33333333 | |
| 475 33.47368421 | |
| 476 33.40336134 | |
| 477 33.54297694 | |
| 478 33.47280335 | |
| 479 33.40292276 | |
| 480 33.33333333 | |
| 481 33.26403326 | |
| 482 33.40248963 | |
| 483 33.33333333 | |
| 484 33.47107438 | |
| 485 33.60824742 | |
| 486 33.53909465 | |
| 487 33.47022587 | |
| 488 33.40163934 | |
| 489 33.53783231 | |
| 490 33.67346939 | |
| 491 33.80855397 | |
| 492 33.94308943 | |
| 493 33.87423935 | |
| 494 34.00809717 | |
| 495 34.14141414 | |
| 496 34.07258065 | |
| 497 34.00402414 | |
| 498 33.93574297 | |
| 499 34.06813627 | |
| 500 34.00000000 | |
| 501 34.13173653 | |
| 502 34.06374502 | |
| 503 33.99602386 | |
| 504 34.12698413 | |
| 505 34.05940594 | |
| 506 33.99209486 | |
| 507 34.12228797 | |
| 508 34.25196850 | |
| 509 34.38113949 | |
| 510 34.31372549 | |
| 511 34.24657534 | |
| 512 34.17968750 | |
| 513 34.11306043 | |
| 514 34.04669261 | |
| 515 34.17475728 | |
| 516 34.10852713 | |
| 517 34.23597679 | |
| 518 34.16988417 | |
| 519 34.10404624 | |
| 520 34.03846154 | |
| 521 33.97312860 | |
| 522 33.90804598 | |
| 523 33.84321224 | |
| 524 33.77862595 | |
| 525 33.90476190 | |
| 526 33.84030418 | |
| 527 33.96584440 | |
| 528 33.90151515 | |
| 529 33.83742911 | |
| 530 33.77358491 | |
| 531 33.70998117 | |
| 532 33.64661654 | |
| 533 33.77110694 | |
| 534 33.70786517 | |
| 535 33.64485981 | |
| 536 33.58208955 | |
| 537 33.70577281 | |
| 538 33.64312268 | |
| 539 33.58070501 | |
| 540 33.51851852 | |
| 541 33.64140481 | |
| 542 33.57933579 | |
| 543 33.51749540 | |
| 544 33.63970588 | |
| 545 33.76146789 | |
| 546 33.69963370 | |
| 547 33.82084095 | |
| 548 33.94160584 | |
| 549 33.87978142 | |
| 550 33.81818182 | |
| 551 33.75680581 | |
| 552 33.87681159 | |
| 553 33.99638336 | |
| 554 33.93501805 | |
| 555 34.05405405 | |
| 556 33.99280576 | |
| 557 33.93177738 | |
| 558 33.87096774 | |
| 559 33.81037567 | |
| 560 33.92857143 | |
| 561 34.04634581 | |
| 562 33.98576512 | |
| 563 33.92539964 | |
| 564 34.04255319 | |
| 565 34.15929204 | |
| 566 34.27561837 | |
| 567 34.21516755 | |
| 568 34.33098592 | |
| 569 34.27065026 | |
| 570 34.21052632 | |
| 571 34.32574431 | |
| 572 34.26573427 | |
| 573 34.20593368 | |
| 574 34.14634146 | |
| 575 34.26086957 | |
| 576 34.20138889 | |
| 577 34.31542461 | |
| 578 34.42906574 | |
| 579 34.54231434 | |
| 580 34.65517241 | |
| 581 34.59552496 | |
| 582 34.70790378 | |
| 583 34.64837050 | |
| 584 34.76027397 | |
| 585 34.70085470 | |
| 586 34.81228669 | |
| 587 34.75298126 | |
| 588 34.86394558 | |
| 589 34.80475382 | |
| 590 34.74576271 | |
| 591 34.68697124 | |
| 592 34.62837838 | |
| 593 34.73861720 | |
| 594 34.84848485 | |
| 595 34.78991597 | |
| 596 34.73154362 | |
| 597 34.67336683 | |
| 598 34.61538462 | |
| 599 34.55759599 | |
| 600 34.50000000 | |
| 601 34.60898502 | |
| 602 34.71760797 | |
| 603 34.82587065 | |
| 604 34.76821192 | |
| 605 34.71074380 | |
| 606 34.81848185 | |
| 607 34.76112026 | |
| 608 34.70394737 | |
| 609 34.81116585 | |
| 610 34.75409836 | |
| 611 34.69721768 | |
| 612 34.80392157 | |
| 613 34.74714519 | |
| 614 34.69055375 | |
| 615 34.63414634 | |
| 616 34.74025974 | |
| 617 34.84602917 | |
| 618 34.78964401 | |
| 619 34.89499192 | |
| 620 34.83870968 | |
| 621 34.94363929 | |
| 622 35.04823151 | |
| 623 34.99197432 | |
| 624 34.93589744 | |
| 625 34.88000000 | |
| 626 34.82428115 | |
| 627 34.92822967 | |
| 628 35.03184713 | |
| 629 35.13513514 | |
| 630 35.07936508 | |
| 631 35.18225040 | |
| 632 35.12658228 | |
| 633 35.22906793 | |
| 634 35.33123028 | |
| 635 35.27559055 | |
| 636 35.22012579 | |
| 637 35.16483516 | |
| 638 35.10971787 | |
| 639 35.05477308 | |
| 640 35.15625000 | |
| 641 35.10140406 | |
| 642 35.20249221 | |
| 643 35.14774495 | |
| 644 35.09316770 | |
| 645 35.19379845 | |
| 646 35.13931889 | |
| 647 35.23956723 | |
| 648 35.18518519 | |
| 649 35.13097072 | |
| 650 35.23076923 | |
| 651 35.33026114 | |
| 652 35.42944785 | |
| 653 35.52833078 | |
| 654 35.47400612 | |
| 655 35.41984733 | |
| 656 35.51829268 | |
| 657 35.61643836 | |
| 658 35.56231003 | |
| 659 35.50834598 | |
| 660 35.45454545 | |
| 661 35.40090772 | |
| 662 35.34743202 | |
| 663 35.44494721 | |
| 664 35.39156627 | |
| 665 35.33834586 | |
| 666 35.28528529 | |
| 667 35.23238381 | |
| 668 35.17964072 | |
| 669 35.27653214 | |
| 670 35.37313433 | |
| 671 35.32041729 | |
| 672 35.26785714 | |
| 673 35.21545319 | |
| 674 35.16320475 | |
| 675 35.25925926 | |
| 676 35.35502959 | |
| 677 35.30280650 | |
| 678 35.25073746 | |
| 679 35.19882180 | |
| 680 35.29411765 | |
| 681 35.38913363 | |
| 682 35.33724340 | |
| 683 35.28550512 | |
| 684 35.38011696 | |
| 685 35.32846715 | |
| 686 35.27696793 | |
| 687 35.22561863 | |
| 688 35.31976744 | |
| 689 35.41364296 | |
| 690 35.36231884 | |
| 691 35.45586107 | |
| 692 35.54913295 | |
| 693 35.49783550 | |
| 694 35.44668588 | |
| 695 35.53956835 | |
| 696 35.63218391 | |
| 697 35.58106169 | |
| 698 35.53008596 | |
| 699 35.47925608 | |
| 700 35.42857143 | |
| 701 35.37803138 | |
| 702 35.32763533 | |
| 703 35.27738265 | |
| 704 35.22727273 | |
| 705 35.17730496 | |
| 706 35.12747875 | |
| 707 35.21923621 | |
| 708 35.16949153 | |
| 709 35.11988717 | |
| 710 35.07042254 | |
| 711 35.16174402 | |
| 712 35.11235955 | |
| 713 35.06311360 | |
| 714 35.15406162 | |
| 715 35.24475524 | |
| 716 35.19553073 | |
| 717 35.14644351 | |
| 718 35.09749304 | |
| 719 35.18776078 | |
| 720 35.13888889 | |
| 721 35.09015257 | |
| 722 35.04155125 | |
| 723 34.99308437 | |
| 724 34.94475138 | |
| 725 34.89655172 | |
| 726 34.98622590 | |
| 727 34.93810179 | |
| 728 34.89010989 | |
| 729 34.97942387 | |
| 730 35.06849315 | |
| 731 35.02051984 | |
| 732 35.10928962 | |
| 733 35.06139154 | |
| 734 35.01362398 | |
| 735 34.96598639 | |
| 736 34.91847826 | |
| 737 35.00678426 | |
| 738 34.95934959 | |
| 739 35.04736130 | |
| 740 35.00000000 | |
| 741 35.08771930 | |
| 742 35.04043127 | |
| 743 34.99327052 | |
| 744 35.08064516 | |
| 745 35.03355705 | |
| 746 34.98659517 | |
| 747 35.07362784 | |
| 748 35.02673797 | |
| 749 35.11348465 | |
| 750 35.20000000 | |
| Final result: 35.2000 +/- 1.7451 | |
| Random chance: 19.8992 +/- 1.4588 | |