common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 27.90697674 44 29.54545455 45 31.11111111 46 30.43478261 47 31.91489362 48 33.33333333 49 32.65306122 50 34.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 32.81250000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 31.42857143 71 30.98591549 72 31.94444444 73 31.50684932 74 31.08108108 75 30.66666667 76 31.57894737 77 31.16883117 78 30.76923077 79 30.37974684 80 30.00000000 81 29.62962963 82 30.48780488 83 30.12048193 84 29.76190476 85 29.41176471 86 29.06976744 87 28.73563218 88 28.40909091 89 29.21348315 90 28.88888889 91 28.57142857 92 28.26086957 93 27.95698925 94 27.65957447 95 27.36842105 96 28.12500000 97 28.86597938 98 28.57142857 99 29.29292929 100 30.00000000 101 30.69306931 102 30.39215686 103 31.06796117 104 30.76923077 105 30.47619048 106 30.18867925 107 30.84112150 108 31.48148148 109 32.11009174 110 32.72727273 111 32.43243243 112 33.03571429 113 32.74336283 114 33.33333333 115 33.04347826 116 33.62068966 117 34.18803419 118 33.89830508 119 33.61344538 120 34.16666667 121 34.71074380 122 34.42622951 123 34.95934959 124 34.67741935 125 35.20000000 126 34.92063492 127 34.64566929 128 34.37500000 129 34.88372093 130 35.38461538 131 35.11450382 132 34.84848485 133 34.58646617 134 34.32835821 135 34.81481481 136 34.55882353 137 35.03649635 138 35.50724638 139 35.97122302 140 35.71428571 141 36.17021277 142 35.91549296 143 35.66433566 144 36.11111111 145 36.55172414 146 36.30136986 147 36.73469388 148 36.48648649 149 36.24161074 150 36.00000000 151 36.42384106 152 36.18421053 153 35.94771242 154 35.71428571 155 35.48387097 156 35.89743590 157 36.30573248 158 36.70886076 159 36.47798742 160 36.25000000 161 36.02484472 162 35.80246914 163 36.19631902 164 35.97560976 165 36.36363636 166 36.74698795 167 36.52694611 168 36.30952381 169 36.09467456 170 35.88235294 171 35.67251462 172 35.46511628 173 35.26011561 174 35.63218391 175 35.42857143 176 35.22727273 177 35.02824859 178 34.83146067 179 34.63687151 180 34.44444444 181 34.25414365 182 34.06593407 183 33.87978142 184 33.69565217 185 34.05405405 186 34.40860215 187 34.22459893 188 34.04255319 189 33.86243386 190 33.68421053 191 33.50785340 192 33.33333333 193 33.16062176 194 33.50515464 195 33.33333333 196 33.16326531 197 33.50253807 198 33.83838384 199 33.66834171 200 34.00000000 201 33.83084577 202 34.15841584 203 33.99014778 204 33.82352941 205 34.14634146 206 33.98058252 207 34.29951691 208 34.13461538 209 33.97129187 210 33.80952381 211 33.64928910 212 33.49056604 213 33.33333333 214 33.17757009 215 33.02325581 216 32.87037037 217 32.71889401 218 32.56880734 219 32.42009132 220 32.27272727 221 32.57918552 222 32.43243243 223 32.28699552 224 32.58928571 225 32.44444444 226 32.74336283 227 33.03964758 228 32.89473684 229 32.75109170 230 32.60869565 231 32.46753247 232 32.75862069 233 32.61802575 234 32.47863248 235 32.34042553 236 32.62711864 237 32.48945148 238 32.77310924 239 32.63598326 240 32.50000000 241 32.36514523 242 32.23140496 243 32.09876543 244 31.96721311 245 31.83673469 246 31.70731707 247 31.98380567 248 32.25806452 249 32.53012048 250 32.40000000 251 32.66932271 252 32.53968254 253 32.41106719 254 32.67716535 255 32.94117647 256 32.81250000 257 32.68482490 258 32.55813953 259 32.43243243 260 32.30769231 261 32.56704981 262 32.44274809 263 32.31939163 264 32.57575758 265 32.83018868 266 32.70676692 267 32.58426966 268 32.46268657 269 32.34200743 270 32.22222222 271 32.10332103 272 31.98529412 273 31.86813187 274 32.11678832 275 32.00000000 276 31.88405797 277 31.76895307 278 31.65467626 279 31.54121864 280 31.78571429 281 32.02846975 282 32.26950355 283 32.15547703 284 32.39436620 285 32.28070175 286 32.51748252 287 32.75261324 288 32.98611111 289 32.87197232 290 32.75862069 291 32.98969072 292 32.87671233 293 33.10580205 294 32.99319728 295 33.22033898 296 33.44594595 297 33.33333333 298 33.22147651 299 33.11036789 300 33.00000000 301 32.89036545 302 32.78145695 303 32.67326733 304 32.56578947 305 32.45901639 306 32.35294118 307 32.24755700 308 32.46753247 309 32.36245955 310 32.25806452 311 32.15434084 312 32.37179487 313 32.26837061 314 32.16560510 315 32.06349206 316 31.96202532 317 31.86119874 318 31.76100629 319 31.66144201 320 31.56250000 321 31.46417445 322 31.67701863 323 31.57894737 324 31.48148148 325 31.38461538 326 31.28834356 327 31.49847095 328 31.70731707 329 31.61094225 330 31.81818182 331 31.72205438 332 31.92771084 333 31.83183183 334 31.73652695 335 31.94029851 336 32.14285714 337 32.04747774 338 31.95266272 339 32.15339233 340 32.05882353 341 32.25806452 342 32.45614035 343 32.36151603 344 32.55813953 345 32.75362319 346 32.65895954 347 32.85302594 348 32.75862069 349 32.66475645 350 32.57142857 351 32.47863248 352 32.38636364 353 32.29461756 354 32.20338983 355 32.39436620 356 32.30337079 357 32.49299720 358 32.68156425 359 32.59052925 360 32.50000000 361 32.68698061 362 32.59668508 363 32.50688705 364 32.41758242 365 32.32876712 366 32.24043716 367 32.42506812 368 32.60869565 369 32.52032520 370 32.43243243 371 32.34501348 372 32.52688172 373 32.43967828 374 32.62032086 375 32.53333333 376 32.44680851 377 32.62599469 378 32.53968254 379 32.45382586 380 32.63157895 381 32.54593176 382 32.72251309 383 32.63707572 384 32.55208333 385 32.46753247 386 32.38341969 387 32.29974160 388 32.47422680 389 32.39074550 390 32.30769231 391 32.22506394 392 32.14285714 393 32.31552163 394 32.23350254 395 32.15189873 396 32.07070707 397 32.24181360 398 32.16080402 399 32.08020050 400 32.00000000 401 32.16957606 402 32.33830846 403 32.50620347 404 32.42574257 405 32.34567901 406 32.26600985 407 32.18673219 408 32.10784314 409 32.02933985 410 31.95121951 411 31.87347932 412 32.03883495 413 32.20338983 414 32.36714976 415 32.53012048 416 32.45192308 417 32.37410072 418 32.29665072 419 32.21957041 420 32.38095238 421 32.54156770 422 32.46445498 423 32.38770686 424 32.31132075 425 32.23529412 426 32.15962441 427 32.08430913 428 32.00934579 429 31.93473193 430 32.09302326 431 32.01856148 432 32.17592593 433 32.33256351 434 32.48847926 435 32.41379310 436 32.33944954 437 32.26544622 438 32.42009132 439 32.57403189 440 32.50000000 441 32.42630385 442 32.35294118 443 32.27990971 444 32.43243243 445 32.35955056 446 32.28699552 447 32.21476510 448 32.14285714 449 32.29398664 450 32.44444444 451 32.37250554 452 32.30088496 453 32.45033113 454 32.37885463 455 32.30769231 456 32.23684211 457 32.16630197 458 32.09606987 459 32.02614379 460 31.95652174 461 31.88720174 462 31.81818182 463 31.96544276 464 32.11206897 465 32.25806452 466 32.18884120 467 32.11991435 468 32.05128205 469 31.98294243 470 31.91489362 471 32.05944798 472 32.20338983 473 32.13530655 474 32.06751055 475 32.21052632 476 32.14285714 477 32.28511530 478 32.21757322 479 32.15031315 480 32.08333333 481 32.01663202 482 32.15767635 483 32.09109731 484 32.23140496 485 32.37113402 486 32.30452675 487 32.23819302 488 32.17213115 489 32.31083845 490 32.44897959 491 32.58655804 492 32.72357724 493 32.65720081 494 32.79352227 495 32.72727273 496 32.66129032 497 32.59557344 498 32.73092369 499 32.86573146 500 32.80000000 501 32.73453094 502 32.66932271 503 32.60437376 504 32.73809524 505 32.67326733 506 32.60869565 507 32.74161736 508 32.87401575 509 33.00589391 510 32.94117647 511 32.87671233 512 32.81250000 513 32.74853801 514 32.68482490 515 32.81553398 516 32.75193798 517 32.88201161 518 32.81853282 519 32.75529865 520 32.69230769 521 32.62955854 522 32.56704981 523 32.50478011 524 32.44274809 525 32.57142857 526 32.50950570 527 32.63757116 528 32.57575758 529 32.70321361 530 32.64150943 531 32.58003766 532 32.51879699 533 32.64540338 534 32.58426966 535 32.52336449 536 32.46268657 537 32.58845438 538 32.52788104 539 32.46753247 540 32.40740741 541 32.53234750 542 32.47232472 543 32.41252302 544 32.35294118 545 32.47706422 546 32.60073260 547 32.72394881 548 32.84671533 549 32.78688525 550 32.72727273 551 32.66787659 552 32.78985507 553 32.91139241 554 32.85198556 555 32.97297297 556 32.91366906 557 32.85457810 558 32.79569892 559 32.73703041 560 32.85714286 561 32.97682709 562 32.91814947 563 32.85968028 564 32.97872340 565 33.09734513 566 33.21554770 567 33.15696649 568 33.27464789 569 33.39191564 570 33.33333333 571 33.45008757 572 33.39160839 573 33.33333333 574 33.44947735 575 33.56521739 576 33.50694444 577 33.62218371 578 33.73702422 579 33.85146805 580 33.96551724 581 33.90705680 582 34.02061856 583 33.96226415 584 34.07534247 585 34.01709402 586 34.12969283 587 34.07155026 588 34.01360544 589 33.95585739 590 33.89830508 591 33.84094755 592 33.78378378 593 33.89544688 594 34.00673401 595 33.94957983 596 33.89261745 597 34.00335008 598 33.94648829 599 33.88981636 600 33.83333333 601 33.94342762 602 34.05315615 603 34.16252073 604 34.10596026 605 34.21487603 606 34.32343234 607 34.26688633 608 34.21052632 609 34.31855501 610 34.26229508 611 34.20621931 612 34.31372549 613 34.25774878 614 34.20195440 615 34.14634146 616 34.25324675 617 34.35980551 618 34.30420712 619 34.41033926 620 34.35483871 621 34.29951691 622 34.40514469 623 34.34991974 624 34.29487179 625 34.24000000 626 34.18530351 627 34.29027113 628 34.39490446 629 34.34022258 630 34.28571429 631 34.23137876 632 34.17721519 633 34.28120063 634 34.38485804 635 34.33070866 636 34.27672956 637 34.22291994 638 34.16927900 639 34.11580595 640 34.21875000 641 34.16536661 642 34.26791277 643 34.21461897 644 34.16149068 645 34.26356589 646 34.21052632 647 34.31221020 648 34.25925926 649 34.20647149 650 34.30769231 651 34.40860215 652 34.50920245 653 34.60949464 654 34.55657492 655 34.50381679 656 34.60365854 657 34.70319635 658 34.65045593 659 34.59787557 660 34.54545455 661 34.49319213 662 34.44108761 663 34.53996983 664 34.48795181 665 34.58646617 666 34.68468468 667 34.63268366 668 34.58083832 669 34.52914798 670 34.47761194 671 34.42622951 672 34.37500000 673 34.32392273 674 34.27299703 675 34.37037037 676 34.46745562 677 34.41654357 678 34.36578171 679 34.31516937 680 34.41176471 681 34.50807636 682 34.45747801 683 34.40702782 684 34.50292398 685 34.45255474 686 34.40233236 687 34.35225619 688 34.44767442 689 34.54281567 690 34.49275362 691 34.58755427 692 34.68208092 693 34.63203463 694 34.58213256 695 34.67625899 696 34.77011494 697 34.72022956 698 34.67048711 699 34.62088698 700 34.71428571 701 34.66476462 702 34.61538462 703 34.56614509 704 34.51704545 705 34.46808511 706 34.56090652 707 34.65346535 708 34.60451977 709 34.55571227 710 34.64788732 711 34.73980309 712 34.69101124 713 34.64235624 714 34.73389356 715 34.82517483 716 34.77653631 717 34.86750349 718 34.81894150 719 34.90959666 720 34.86111111 721 34.81276006 722 34.76454294 723 34.71645920 724 34.66850829 725 34.62068966 726 34.71074380 727 34.66299862 728 34.61538462 729 34.70507545 730 34.79452055 731 34.74692202 732 34.83606557 733 34.78854025 734 34.74114441 735 34.69387755 736 34.64673913 737 34.73541384 738 34.68834688 739 34.77672530 740 34.72972973 741 34.81781377 742 34.90566038 743 34.85868102 744 34.94623656 745 34.89932886 746 34.85254692 747 34.93975904 748 34.89304813 749 34.97997330 750 35.06666667 Final result: 35.0667 +/- 1.7436 Random chance: 19.8992 +/- 1.4588