common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 46.15384615 14 42.85714286 15 46.66666667 16 50.00000000 17 52.94117647 18 50.00000000 19 52.63157895 20 55.00000000 21 52.38095238 22 50.00000000 23 47.82608696 24 50.00000000 25 52.00000000 26 50.00000000 27 51.85185185 28 53.57142857 29 51.72413793 30 50.00000000 31 48.38709677 32 50.00000000 33 48.48484848 34 50.00000000 35 51.42857143 36 50.00000000 37 48.64864865 38 50.00000000 39 48.71794872 40 47.50000000 41 46.34146341 42 45.23809524 43 46.51162791 44 45.45454545 45 46.66666667 46 45.65217391 47 44.68085106 48 45.83333333 49 46.93877551 50 48.00000000 51 47.05882353 52 46.15384615 53 45.28301887 54 44.44444444 55 45.45454545 56 44.64285714 57 45.61403509 58 46.55172414 59 47.45762712 60 46.66666667 61 47.54098361 62 46.77419355 63 46.03174603 64 46.87500000 65 46.15384615 66 45.45454545 67 44.77611940 68 44.11764706 69 43.47826087 70 44.28571429 71 45.07042254 72 45.83333333 73 45.20547945 74 45.94594595 75 46.66666667 76 47.36842105 77 48.05194805 78 48.71794872 79 48.10126582 80 48.75000000 81 49.38271605 82 48.78048780 83 49.39759036 84 48.80952381 85 49.41176471 86 48.83720930 87 49.42528736 88 50.00000000 89 49.43820225 90 48.88888889 91 48.35164835 92 47.82608696 93 47.31182796 94 47.87234043 95 47.36842105 96 47.91666667 97 47.42268041 98 47.95918367 99 47.47474747 100 47.00000000 101 47.52475248 102 47.05882353 103 46.60194175 104 47.11538462 105 47.61904762 106 47.16981132 107 46.72897196 108 47.22222222 109 46.78899083 110 46.36363636 111 46.84684685 112 47.32142857 113 46.90265487 114 47.36842105 115 47.82608696 116 48.27586207 117 47.86324786 118 47.45762712 119 47.05882353 120 46.66666667 121 47.10743802 122 46.72131148 123 46.34146341 124 46.77419355 125 47.20000000 126 46.82539683 127 46.45669291 128 46.87500000 129 46.51162791 130 46.92307692 131 47.32824427 132 47.72727273 133 47.36842105 134 47.01492537 135 46.66666667 136 47.05882353 137 47.44525547 138 47.10144928 139 47.48201439 140 47.85714286 141 48.22695035 142 48.59154930 143 48.95104895 144 48.61111111 145 48.96551724 146 49.31506849 147 48.97959184 148 49.32432432 149 49.66442953 150 49.33333333 151 49.00662252 152 49.34210526 153 49.01960784 154 48.70129870 155 48.38709677 156 48.07692308 157 48.40764331 158 48.73417722 159 48.42767296 160 48.75000000 161 48.44720497 162 48.14814815 163 47.85276074 164 47.56097561 165 47.27272727 166 46.98795181 167 47.30538922 168 47.02380952 169 47.33727811 170 47.05882353 171 47.36842105 172 47.09302326 173 46.82080925 174 46.55172414 175 46.28571429 176 46.59090909 177 46.32768362 178 46.62921348 179 46.36871508 180 46.11111111 181 45.85635359 182 45.60439560 183 45.35519126 184 45.10869565 185 45.40540541 186 45.69892473 187 45.45454545 188 45.74468085 189 46.03174603 190 45.78947368 191 46.07329843 192 45.83333333 193 45.59585492 194 45.87628866 195 46.15384615 196 46.42857143 197 46.19289340 198 46.46464646 199 46.23115578 200 46.00000000 201 45.77114428 202 45.54455446 203 45.32019704 204 45.09803922 205 45.36585366 206 45.14563107 207 45.41062802 208 45.19230769 209 44.97607656 210 44.76190476 211 44.54976303 212 44.33962264 213 44.13145540 214 43.92523364 215 44.18604651 216 43.98148148 217 43.77880184 218 44.03669725 219 44.29223744 220 44.09090909 221 44.34389140 222 44.14414414 223 44.39461883 224 44.64285714 225 44.88888889 226 45.13274336 227 44.93392070 228 45.17543860 229 44.97816594 230 44.78260870 231 44.58874459 232 44.82758621 233 44.63519313 234 44.87179487 235 44.68085106 236 44.49152542 237 44.30379747 238 44.11764706 239 43.93305439 240 44.16666667 241 43.98340249 242 44.21487603 243 44.44444444 244 44.26229508 245 44.48979592 246 44.30894309 247 44.12955466 248 44.35483871 249 44.17670683 250 44.00000000 251 43.82470120 252 43.65079365 253 43.87351779 254 43.70078740 255 43.52941176 256 43.75000000 257 43.96887160 258 43.79844961 259 43.62934363 260 43.84615385 261 44.06130268 262 43.89312977 263 43.72623574 264 43.93939394 265 43.77358491 266 43.98496241 267 43.82022472 268 43.65671642 269 43.86617100 270 44.07407407 271 43.91143911 272 44.11764706 273 43.95604396 274 43.79562044 275 43.63636364 276 43.84057971 277 44.04332130 278 44.24460432 279 44.08602151 280 43.92857143 281 43.77224199 282 43.61702128 283 43.46289753 284 43.30985915 285 43.15789474 286 43.00699301 287 43.20557491 288 43.05555556 289 42.90657439 290 43.10344828 291 43.29896907 292 43.49315068 293 43.68600683 294 43.53741497 295 43.38983051 296 43.24324324 297 43.09764310 298 43.28859060 299 43.47826087 300 43.66666667 301 43.52159468 302 43.70860927 303 43.56435644 304 43.42105263 305 43.60655738 306 43.46405229 307 43.64820847 308 43.83116883 309 44.01294498 310 43.87096774 311 44.05144695 312 44.23076923 313 44.40894569 314 44.58598726 315 44.44444444 316 44.30379747 317 44.16403785 318 44.02515723 319 44.20062696 320 44.06250000 321 43.92523364 322 44.09937888 323 43.96284830 324 43.82716049 325 44.00000000 326 43.86503067 327 44.03669725 328 44.20731707 329 44.37689970 330 44.24242424 331 44.10876133 332 44.27710843 333 44.44444444 334 44.31137725 335 44.47761194 336 44.64285714 337 44.51038576 338 44.37869822 339 44.24778761 340 44.41176471 341 44.57478006 342 44.73684211 343 44.60641399 344 44.76744186 345 44.63768116 346 44.79768786 347 44.66858790 348 44.54022989 349 44.41260745 350 44.57142857 351 44.44444444 352 44.31818182 353 44.47592068 354 44.35028249 355 44.22535211 356 44.38202247 357 44.53781513 358 44.41340782 359 44.28969359 360 44.44444444 361 44.32132964 362 44.47513812 363 44.35261708 364 44.50549451 365 44.38356164 366 44.53551913 367 44.41416894 368 44.29347826 369 44.17344173 370 44.32432432 371 44.47439353 372 44.35483871 373 44.23592493 374 44.38502674 375 44.53333333 376 44.41489362 377 44.29708223 378 44.44444444 379 44.32717678 380 44.21052632 381 44.35695538 382 44.50261780 383 44.64751958 384 44.53125000 385 44.67532468 386 44.55958549 387 44.44444444 388 44.32989691 389 44.47300771 390 44.35897436 391 44.50127877 392 44.64285714 393 44.52926209 394 44.41624365 395 44.55696203 396 44.69696970 397 44.83627204 398 44.72361809 399 44.86215539 400 44.75000000 401 44.88778055 402 44.77611940 403 44.66501241 404 44.80198020 405 44.69135802 406 44.58128079 407 44.47174447 408 44.36274510 409 44.49877751 410 44.39024390 411 44.28223844 412 44.17475728 413 44.06779661 414 44.20289855 415 44.09638554 416 43.99038462 417 43.88489209 418 43.77990431 419 43.91408115 420 43.80952381 421 43.70546318 422 43.60189573 423 43.49881797 424 43.63207547 425 43.52941176 426 43.66197183 427 43.55971897 428 43.45794393 429 43.58974359 430 43.48837209 431 43.61948956 432 43.51851852 433 43.41801386 434 43.31797235 435 43.21839080 436 43.11926606 437 43.02059497 438 43.15068493 439 43.28018223 440 43.18181818 441 43.08390023 442 42.98642534 443 42.88939052 444 42.79279279 445 42.69662921 446 42.60089686 447 42.72930649 448 42.63392857 449 42.53897550 450 42.66666667 451 42.79379157 452 42.92035398 453 43.04635762 454 42.95154185 455 42.85714286 456 42.98245614 457 43.10722101 458 43.01310044 459 42.91938998 460 43.04347826 461 42.95010846 462 43.07359307 463 43.19654428 464 43.31896552 465 43.22580645 466 43.13304721 467 43.25481799 468 43.37606838 469 43.28358209 470 43.19148936 471 43.09978769 472 43.00847458 473 42.91754757 474 42.82700422 475 42.73684211 476 42.85714286 477 42.76729560 478 42.88702929 479 43.00626305 480 42.91666667 481 43.03534304 482 42.94605809 483 42.85714286 484 42.76859504 485 42.68041237 486 42.79835391 487 42.91581109 488 42.82786885 489 42.74028630 490 42.85714286 491 42.76985743 492 42.68292683 493 42.79918864 494 42.91497976 495 43.03030303 496 43.14516129 497 43.05835010 498 42.97188755 499 43.08617234 500 43.00000000 501 42.91417166 502 42.82868526 503 42.74353877 504 42.85714286 505 42.77227723 506 42.88537549 507 42.80078895 508 42.71653543 509 42.63261297 510 42.54901961 511 42.46575342 512 42.38281250 513 42.30019493 514 42.21789883 515 42.13592233 516 42.24806202 517 42.35976789 518 42.47104247 519 42.38921002 520 42.50000000 521 42.41842610 522 42.52873563 523 42.44741874 524 42.55725191 525 42.47619048 526 42.39543726 527 42.50474383 528 42.61363636 529 42.72211720 530 42.83018868 531 42.74952919 532 42.66917293 533 42.58911820 534 42.69662921 535 42.80373832 536 42.72388060 537 42.83054004 538 42.93680297 539 42.85714286 540 42.77777778 541 42.88354898 542 42.98892989 543 42.90976059 544 43.01470588 545 42.93577982 546 42.85714286 547 42.96160878 548 43.06569343 549 42.98724954 550 42.90909091 551 43.01270417 552 43.11594203 553 43.03797468 554 43.14079422 555 43.06306306 556 43.16546763 557 43.08797127 558 43.18996416 559 43.29159213 560 43.21428571 561 43.13725490 562 43.23843416 563 43.33925400 564 43.26241135 565 43.18584071 566 43.10954064 567 43.03350970 568 42.95774648 569 42.88224956 570 42.98245614 571 42.90718039 572 43.00699301 573 42.93193717 574 42.85714286 575 42.78260870 576 42.88194444 577 42.98093588 578 43.07958478 579 43.00518135 580 43.10344828 581 43.20137694 582 43.12714777 583 43.22469983 584 43.32191781 585 43.41880342 586 43.51535836 587 43.44122658 588 43.36734694 589 43.29371817 590 43.22033898 591 43.14720812 592 43.24324324 593 43.33895447 594 43.43434343 595 43.36134454 596 43.45637584 597 43.38358459 598 43.47826087 599 43.57262104 600 43.50000000 601 43.59400998 602 43.52159468 603 43.61525705 604 43.54304636 605 43.47107438 606 43.39933993 607 43.49258649 608 43.42105263 609 43.34975369 610 43.27868852 611 43.37152209 612 43.30065359 613 43.23001631 614 43.32247557 615 43.25203252 616 43.34415584 617 43.27390600 618 43.36569579 619 43.29563813 620 43.22580645 621 43.15619968 622 43.08681672 623 43.01765650 624 42.94871795 625 42.88000000 626 42.97124601 627 42.90271132 628 42.83439490 629 42.92527822 630 42.85714286 631 42.78922345 632 42.87974684 633 42.81200632 634 42.74447950 635 42.83464567 636 42.76729560 637 42.70015699 638 42.78996865 639 42.87949922 640 42.96875000 641 42.90171607 642 42.99065421 643 42.92379471 644 43.01242236 645 42.94573643 646 42.87925697 647 42.81298300 648 42.74691358 649 42.68104777 650 42.76923077 651 42.70353303 652 42.79141104 653 42.72588055 654 42.81345566 655 42.74809160 656 42.83536585 657 42.92237443 658 43.00911854 659 42.94385432 660 42.87878788 661 42.81391831 662 42.74924471 663 42.68476621 664 42.62048193 665 42.70676692 666 42.64264264 667 42.72863568 668 42.81437126 669 42.89985052 670 42.98507463 671 42.92101341 672 42.85714286 673 42.79346211 674 42.72997033 675 42.66666667 676 42.60355030 677 42.54062038 678 42.62536873 679 42.56259205 680 42.64705882 681 42.73127753 682 42.66862170 683 42.60614934 684 42.69005848 685 42.62773723 686 42.56559767 687 42.50363901 688 42.44186047 689 42.38026125 690 42.31884058 691 42.40231548 692 42.34104046 693 42.27994228 694 42.21902017 695 42.15827338 696 42.24137931 697 42.18077475 698 42.12034384 699 42.20314735 700 42.28571429 701 42.22539230 702 42.30769231 703 42.38975818 704 42.32954545 705 42.26950355 706 42.35127479 707 42.29137199 708 42.23163842 709 42.31311707 710 42.39436620 711 42.47538678 712 42.55617978 713 42.63674614 714 42.71708683 715 42.65734266 716 42.59776536 717 42.53835425 718 42.47910864 719 42.55910987 720 42.50000000 721 42.44105409 722 42.52077562 723 42.60027663 724 42.54143646 725 42.62068966 726 42.69972452 727 42.64099037 728 42.71978022 729 42.66117970 730 42.60273973 731 42.54445964 732 42.48633880 733 42.56480218 734 42.64305177 735 42.58503401 736 42.52717391 737 42.46947083 738 42.54742547 739 42.62516915 740 42.70270270 741 42.64507422 742 42.72237197 743 42.79946164 744 42.87634409 745 42.81879195 746 42.76139410 747 42.70414993 748 42.64705882 749 42.59012016 750 42.66666667 Final result: 42.6667 +/- 1.8072 Random chance: 25.0000 +/- 1.5822