common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 31.81818182 45 31.11111111 46 30.43478261 47 29.78723404 48 31.25000000 49 30.61224490 50 32.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 32.14285714 57 31.57894737 58 31.03448276 59 30.50847458 60 31.66666667 61 31.14754098 62 32.25806452 63 31.74603175 64 32.81250000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 32.43243243 75 32.00000000 76 32.89473684 77 32.46753247 78 32.05128205 79 31.64556962 80 32.50000000 81 32.09876543 82 32.92682927 83 32.53012048 84 32.14285714 85 31.76470588 86 31.39534884 87 31.03448276 88 30.68181818 89 31.46067416 90 31.11111111 91 30.76923077 92 30.43478261 93 30.10752688 94 29.78723404 95 29.47368421 96 30.20833333 97 30.92783505 98 30.61224490 99 31.31313131 100 32.00000000 101 32.67326733 102 32.35294118 103 33.00970874 104 32.69230769 105 32.38095238 106 32.07547170 107 32.71028037 108 33.33333333 109 33.94495413 110 34.54545455 111 34.23423423 112 34.82142857 113 35.39823009 114 35.96491228 115 35.65217391 116 36.20689655 117 36.75213675 118 36.44067797 119 36.13445378 120 35.83333333 121 36.36363636 122 36.06557377 123 36.58536585 124 36.29032258 125 36.80000000 126 36.50793651 127 36.22047244 128 35.93750000 129 36.43410853 130 36.92307692 131 36.64122137 132 36.36363636 133 36.09022556 134 35.82089552 135 36.29629630 136 36.02941176 137 36.49635036 138 36.95652174 139 37.41007194 140 37.14285714 141 37.58865248 142 37.32394366 143 37.06293706 144 37.50000000 145 37.93103448 146 37.67123288 147 37.41496599 148 37.16216216 149 36.91275168 150 36.66666667 151 37.08609272 152 36.84210526 153 36.60130719 154 36.36363636 155 36.12903226 156 36.53846154 157 36.94267516 158 37.34177215 159 37.73584906 160 37.50000000 161 37.88819876 162 37.65432099 163 37.42331288 164 37.19512195 165 37.57575758 166 37.95180723 167 37.72455090 168 37.50000000 169 37.27810651 170 37.05882353 171 36.84210526 172 36.62790698 173 36.41618497 174 36.78160920 175 36.57142857 176 36.36363636 177 36.15819209 178 35.95505618 179 35.75418994 180 35.55555556 181 35.91160221 182 35.71428571 183 35.51912568 184 35.32608696 185 35.67567568 186 36.02150538 187 35.82887701 188 35.63829787 189 35.44973545 190 35.26315789 191 35.07853403 192 34.89583333 193 34.71502591 194 35.05154639 195 34.87179487 196 34.69387755 197 35.02538071 198 35.35353535 199 35.17587940 200 35.50000000 201 35.82089552 202 36.13861386 203 35.96059113 204 35.78431373 205 36.09756098 206 35.92233010 207 36.23188406 208 36.05769231 209 35.88516746 210 35.71428571 211 35.54502370 212 35.37735849 213 35.21126761 214 35.04672897 215 34.88372093 216 35.18518519 217 35.02304147 218 34.86238532 219 34.70319635 220 34.54545455 221 34.84162896 222 34.68468468 223 34.97757848 224 35.26785714 225 35.55555556 226 35.39823009 227 35.68281938 228 35.52631579 229 35.37117904 230 35.21739130 231 35.06493506 232 35.34482759 233 35.19313305 234 35.04273504 235 34.89361702 236 35.16949153 237 35.02109705 238 35.29411765 239 35.14644351 240 35.00000000 241 34.85477178 242 34.71074380 243 34.56790123 244 34.42622951 245 34.28571429 246 34.14634146 247 34.41295547 248 34.67741935 249 34.93975904 250 34.80000000 251 35.05976096 252 34.92063492 253 34.78260870 254 35.03937008 255 35.29411765 256 35.15625000 257 35.01945525 258 34.88372093 259 34.74903475 260 34.61538462 261 34.86590038 262 34.73282443 263 34.60076046 264 34.84848485 265 35.09433962 266 34.96240602 267 35.20599251 268 35.07462687 269 34.94423792 270 34.81481481 271 34.68634686 272 34.55882353 273 34.79853480 274 35.03649635 275 34.90909091 276 34.78260870 277 34.65703971 278 34.53237410 279 34.76702509 280 35.00000000 281 35.23131673 282 35.10638298 283 34.98233216 284 35.21126761 285 35.08771930 286 35.31468531 287 35.54006969 288 35.76388889 289 35.64013841 290 35.51724138 291 35.73883162 292 35.61643836 293 35.83617747 294 35.71428571 295 35.93220339 296 36.14864865 297 36.02693603 298 35.90604027 299 35.78595318 300 35.66666667 301 35.54817276 302 35.43046358 303 35.64356436 304 35.52631579 305 35.40983607 306 35.29411765 307 35.17915309 308 35.38961039 309 35.27508091 310 35.16129032 311 35.04823151 312 35.25641026 313 35.14376997 314 35.03184713 315 35.23809524 316 35.12658228 317 35.01577287 318 34.90566038 319 34.79623824 320 34.68750000 321 34.57943925 322 34.47204969 323 34.36532508 324 34.25925926 325 34.15384615 326 34.04907975 327 34.25076453 328 34.45121951 329 34.34650456 330 34.54545455 331 34.44108761 332 34.63855422 333 34.53453453 334 34.43113772 335 34.62686567 336 34.82142857 337 34.71810089 338 34.61538462 339 34.80825959 340 34.70588235 341 34.89736070 342 35.08771930 343 34.98542274 344 35.17441860 345 35.36231884 346 35.26011561 347 35.44668588 348 35.34482759 349 35.24355301 350 35.14285714 351 35.04273504 352 34.94318182 353 34.84419263 354 34.74576271 355 34.92957746 356 34.83146067 357 35.01400560 358 34.91620112 359 34.81894150 360 34.72222222 361 34.90304709 362 34.80662983 363 34.71074380 364 34.61538462 365 34.52054795 366 34.42622951 367 34.60490463 368 34.51086957 369 34.41734417 370 34.32432432 371 34.23180593 372 34.40860215 373 34.31635389 374 34.49197861 375 34.40000000 376 34.30851064 377 34.48275862 378 34.39153439 379 34.30079156 380 34.21052632 381 34.12073491 382 34.29319372 383 34.20365535 384 34.11458333 385 34.02597403 386 33.93782383 387 33.85012920 388 34.02061856 389 33.93316195 390 33.84615385 391 33.75959079 392 33.67346939 393 33.58778626 394 33.75634518 395 33.67088608 396 33.83838384 397 34.00503778 398 33.91959799 399 33.83458647 400 33.75000000 401 33.91521197 402 34.07960199 403 34.24317618 404 34.15841584 405 34.07407407 406 33.99014778 407 33.90663391 408 33.82352941 409 33.74083130 410 33.65853659 411 33.57664234 412 33.73786408 413 33.89830508 414 34.05797101 415 34.21686747 416 34.13461538 417 34.05275779 418 33.97129187 419 33.89021480 420 34.04761905 421 34.20427553 422 34.12322275 423 34.04255319 424 33.96226415 425 33.88235294 426 33.80281690 427 33.72365340 428 33.64485981 429 33.56643357 430 33.48837209 431 33.41067285 432 33.56481481 433 33.71824480 434 33.87096774 435 33.79310345 436 33.71559633 437 33.63844394 438 33.78995434 439 33.94077449 440 33.86363636 441 33.78684807 442 33.71040724 443 33.63431151 444 33.78378378 445 33.70786517 446 33.63228700 447 33.55704698 448 33.48214286 449 33.63028953 450 33.77777778 451 33.70288248 452 33.62831858 453 33.77483444 454 33.70044053 455 33.62637363 456 33.55263158 457 33.47921225 458 33.40611354 459 33.33333333 460 33.26086957 461 33.18872017 462 33.11688312 463 33.26133909 464 33.40517241 465 33.54838710 466 33.47639485 467 33.40471092 468 33.54700855 469 33.47547974 470 33.40425532 471 33.54564756 472 33.47457627 473 33.40380550 474 33.33333333 475 33.47368421 476 33.40336134 477 33.54297694 478 33.47280335 479 33.40292276 480 33.33333333 481 33.26403326 482 33.40248963 483 33.33333333 484 33.47107438 485 33.60824742 486 33.53909465 487 33.47022587 488 33.40163934 489 33.53783231 490 33.67346939 491 33.80855397 492 33.94308943 493 33.87423935 494 34.00809717 495 34.14141414 496 34.07258065 497 34.00402414 498 33.93574297 499 34.06813627 500 34.00000000 501 34.13173653 502 34.06374502 503 33.99602386 504 34.12698413 505 34.05940594 506 33.99209486 507 34.12228797 508 34.25196850 509 34.38113949 510 34.31372549 511 34.24657534 512 34.17968750 513 34.11306043 514 34.04669261 515 34.17475728 516 34.10852713 517 34.23597679 518 34.16988417 519 34.10404624 520 34.03846154 521 33.97312860 522 33.90804598 523 33.84321224 524 33.77862595 525 33.90476190 526 33.84030418 527 33.96584440 528 33.90151515 529 33.83742911 530 33.77358491 531 33.70998117 532 33.64661654 533 33.77110694 534 33.70786517 535 33.64485981 536 33.58208955 537 33.70577281 538 33.64312268 539 33.58070501 540 33.51851852 541 33.64140481 542 33.57933579 543 33.51749540 544 33.63970588 545 33.76146789 546 33.69963370 547 33.82084095 548 33.94160584 549 33.87978142 550 33.81818182 551 33.75680581 552 33.87681159 553 33.99638336 554 33.93501805 555 34.05405405 556 33.99280576 557 33.93177738 558 33.87096774 559 33.81037567 560 33.92857143 561 34.04634581 562 33.98576512 563 33.92539964 564 34.04255319 565 34.15929204 566 34.27561837 567 34.21516755 568 34.33098592 569 34.27065026 570 34.21052632 571 34.32574431 572 34.26573427 573 34.20593368 574 34.14634146 575 34.26086957 576 34.20138889 577 34.31542461 578 34.42906574 579 34.54231434 580 34.65517241 581 34.59552496 582 34.70790378 583 34.64837050 584 34.76027397 585 34.70085470 586 34.81228669 587 34.75298126 588 34.86394558 589 34.80475382 590 34.74576271 591 34.68697124 592 34.62837838 593 34.73861720 594 34.84848485 595 34.78991597 596 34.73154362 597 34.67336683 598 34.61538462 599 34.55759599 600 34.50000000 601 34.60898502 602 34.71760797 603 34.82587065 604 34.76821192 605 34.71074380 606 34.81848185 607 34.76112026 608 34.70394737 609 34.81116585 610 34.75409836 611 34.69721768 612 34.80392157 613 34.74714519 614 34.69055375 615 34.63414634 616 34.74025974 617 34.84602917 618 34.78964401 619 34.89499192 620 34.83870968 621 34.94363929 622 35.04823151 623 34.99197432 624 34.93589744 625 34.88000000 626 34.82428115 627 34.92822967 628 35.03184713 629 35.13513514 630 35.07936508 631 35.18225040 632 35.12658228 633 35.22906793 634 35.33123028 635 35.27559055 636 35.22012579 637 35.16483516 638 35.10971787 639 35.05477308 640 35.15625000 641 35.10140406 642 35.20249221 643 35.14774495 644 35.09316770 645 35.19379845 646 35.13931889 647 35.23956723 648 35.18518519 649 35.13097072 650 35.23076923 651 35.33026114 652 35.42944785 653 35.52833078 654 35.47400612 655 35.41984733 656 35.51829268 657 35.61643836 658 35.56231003 659 35.50834598 660 35.45454545 661 35.40090772 662 35.34743202 663 35.44494721 664 35.39156627 665 35.33834586 666 35.28528529 667 35.23238381 668 35.17964072 669 35.27653214 670 35.37313433 671 35.32041729 672 35.26785714 673 35.21545319 674 35.16320475 675 35.25925926 676 35.35502959 677 35.30280650 678 35.25073746 679 35.19882180 680 35.29411765 681 35.38913363 682 35.33724340 683 35.28550512 684 35.38011696 685 35.32846715 686 35.27696793 687 35.22561863 688 35.31976744 689 35.41364296 690 35.36231884 691 35.45586107 692 35.54913295 693 35.49783550 694 35.44668588 695 35.53956835 696 35.63218391 697 35.58106169 698 35.53008596 699 35.47925608 700 35.42857143 701 35.37803138 702 35.32763533 703 35.27738265 704 35.22727273 705 35.17730496 706 35.12747875 707 35.21923621 708 35.16949153 709 35.11988717 710 35.07042254 711 35.16174402 712 35.11235955 713 35.06311360 714 35.15406162 715 35.24475524 716 35.19553073 717 35.14644351 718 35.09749304 719 35.18776078 720 35.13888889 721 35.09015257 722 35.04155125 723 34.99308437 724 34.94475138 725 34.89655172 726 34.98622590 727 34.93810179 728 34.89010989 729 34.97942387 730 35.06849315 731 35.02051984 732 35.10928962 733 35.06139154 734 35.01362398 735 34.96598639 736 34.91847826 737 35.00678426 738 34.95934959 739 35.04736130 740 35.00000000 741 35.08771930 742 35.04043127 743 34.99327052 744 35.08064516 745 35.03355705 746 34.98659517 747 35.07362784 748 35.02673797 749 35.11348465 750 35.20000000 Final result: 35.2000 +/- 1.7451 Random chance: 19.8992 +/- 1.4588