common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 0.00000000 2 0.00000000 3 0.00000000 4 25.00000000 5 20.00000000 6 16.66666667 7 28.57142857 8 37.50000000 9 33.33333333 10 40.00000000 11 36.36363636 12 33.33333333 13 30.76923077 14 28.57142857 15 26.66666667 16 31.25000000 17 35.29411765 18 33.33333333 19 36.84210526 20 40.00000000 21 38.09523810 22 36.36363636 23 39.13043478 24 41.66666667 25 44.00000000 26 42.30769231 27 44.44444444 28 46.42857143 29 44.82758621 30 43.33333333 31 45.16129032 32 46.87500000 33 45.45454545 34 47.05882353 35 48.57142857 36 47.22222222 37 45.94594595 38 47.36842105 39 46.15384615 40 45.00000000 41 43.90243902 42 42.85714286 43 44.18604651 44 43.18181818 45 44.44444444 46 43.47826087 47 42.55319149 48 43.75000000 49 42.85714286 50 44.00000000 51 43.13725490 52 42.30769231 53 41.50943396 54 40.74074074 55 41.81818182 56 42.85714286 57 43.85964912 58 43.10344828 59 44.06779661 60 43.33333333 61 44.26229508 62 43.54838710 63 42.85714286 64 43.75000000 65 43.07692308 66 43.93939394 67 44.77611940 68 44.11764706 69 43.47826087 70 42.85714286 71 42.25352113 72 43.05555556 73 42.46575342 74 43.24324324 75 44.00000000 76 44.73684211 77 45.45454545 78 46.15384615 79 45.56962025 80 45.00000000 81 45.67901235 82 45.12195122 83 45.78313253 84 45.23809524 85 44.70588235 86 45.34883721 87 45.97701149 88 46.59090909 89 46.06741573 90 45.55555556 91 45.05494505 92 44.56521739 93 44.08602151 94 44.68085106 95 44.21052632 96 44.79166667 97 44.32989691 98 44.89795918 99 44.44444444 100 44.00000000 101 44.55445545 102 44.11764706 103 43.68932039 104 44.23076923 105 44.76190476 106 44.33962264 107 43.92523364 108 43.51851852 109 43.11926606 110 43.63636364 111 44.14414414 112 44.64285714 113 44.24778761 114 44.73684211 115 45.21739130 116 45.68965517 117 45.29914530 118 44.91525424 119 44.53781513 120 44.16666667 121 44.62809917 122 44.26229508 123 43.90243902 124 44.35483871 125 44.80000000 126 44.44444444 127 44.09448819 128 44.53125000 129 44.18604651 130 44.61538462 131 45.03816794 132 45.45454545 133 45.11278195 134 44.77611940 135 44.44444444 136 44.85294118 137 45.25547445 138 44.92753623 139 45.32374101 140 45.71428571 141 46.09929078 142 46.47887324 143 46.85314685 144 46.52777778 145 46.89655172 146 47.26027397 147 46.93877551 148 47.29729730 149 47.65100671 150 47.33333333 151 47.01986755 152 47.36842105 153 47.05882353 154 46.75324675 155 46.45161290 156 46.15384615 157 46.49681529 158 46.83544304 159 46.54088050 160 46.87500000 161 46.58385093 162 46.29629630 163 46.01226994 164 46.34146341 165 46.06060606 166 45.78313253 167 46.10778443 168 45.83333333 169 46.15384615 170 45.88235294 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 45.45454545 177 45.19774011 178 45.50561798 179 45.25139665 180 45.00000000 181 44.75138122 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 45.45454545 188 45.21276596 189 45.50264550 190 45.26315789 191 45.54973822 192 45.31250000 193 45.07772021 194 44.84536082 195 45.12820513 196 45.40816327 197 45.17766497 198 45.45454545 199 45.22613065 200 45.00000000 201 44.77611940 202 44.55445545 203 44.33497537 204 44.11764706 205 44.39024390 206 44.17475728 207 44.44444444 208 44.23076923 209 44.01913876 210 43.80952381 211 43.60189573 212 43.39622642 213 43.19248826 214 42.99065421 215 43.25581395 216 43.05555556 217 42.85714286 218 43.11926606 219 43.37899543 220 43.18181818 221 43.43891403 222 43.24324324 223 43.49775785 224 43.75000000 225 44.00000000 226 44.24778761 227 44.05286344 228 44.29824561 229 44.10480349 230 43.91304348 231 43.72294372 232 43.96551724 233 43.77682403 234 44.01709402 235 43.82978723 236 43.64406780 237 43.45991561 238 43.27731092 239 43.09623431 240 43.33333333 241 43.15352697 242 43.38842975 243 43.62139918 244 43.85245902 245 44.08163265 246 43.90243902 247 43.72469636 248 43.95161290 249 43.77510040 250 43.60000000 251 43.42629482 252 43.25396825 253 43.47826087 254 43.30708661 255 43.13725490 256 43.35937500 257 43.57976654 258 43.41085271 259 43.24324324 260 43.46153846 261 43.67816092 262 43.51145038 263 43.34600760 264 43.56060606 265 43.39622642 266 43.60902256 267 43.82022472 268 43.65671642 269 43.86617100 270 44.07407407 271 43.91143911 272 44.11764706 273 43.95604396 274 43.79562044 275 43.63636364 276 43.84057971 277 44.04332130 278 44.24460432 279 44.08602151 280 44.28571429 281 44.12811388 282 43.97163121 283 43.81625442 284 43.66197183 285 43.85964912 286 43.70629371 287 43.90243902 288 43.75000000 289 43.59861592 290 43.79310345 291 43.98625430 292 44.17808219 293 44.36860068 294 44.21768707 295 44.06779661 296 43.91891892 297 43.77104377 298 43.95973154 299 44.14715719 300 44.33333333 301 44.51827243 302 44.70198675 303 44.55445545 304 44.73684211 305 44.91803279 306 45.09803922 307 45.27687296 308 45.45454545 309 45.63106796 310 45.48387097 311 45.65916399 312 45.83333333 313 46.00638978 314 46.17834395 315 46.03174603 316 45.88607595 317 45.74132492 318 45.59748428 319 45.76802508 320 45.62500000 321 45.48286604 322 45.65217391 323 45.51083591 324 45.37037037 325 45.53846154 326 45.39877301 327 45.56574924 328 45.73170732 329 45.89665653 330 45.75757576 331 45.61933535 332 45.48192771 333 45.64564565 334 45.50898204 335 45.67164179 336 45.83333333 337 45.69732938 338 45.56213018 339 45.42772861 340 45.58823529 341 45.74780059 342 45.90643275 343 45.77259475 344 45.93023256 345 45.79710145 346 45.95375723 347 46.10951009 348 45.97701149 349 45.84527221 350 46.00000000 351 45.86894587 352 45.73863636 353 45.89235127 354 45.76271186 355 45.63380282 356 45.78651685 357 45.65826331 358 45.53072626 359 45.40389972 360 45.55555556 361 45.70637119 362 45.85635359 363 45.73002755 364 45.87912088 365 45.75342466 366 45.62841530 367 45.50408719 368 45.38043478 369 45.25745257 370 45.40540541 371 45.55256065 372 45.43010753 373 45.30831099 374 45.45454545 375 45.60000000 376 45.47872340 377 45.35809019 378 45.50264550 379 45.38258575 380 45.26315789 381 45.40682415 382 45.54973822 383 45.69190601 384 45.57291667 385 45.71428571 386 45.59585492 387 45.47803618 388 45.36082474 389 45.50128535 390 45.38461538 391 45.52429668 392 45.66326531 393 45.54707379 394 45.43147208 395 45.56962025 396 45.70707071 397 45.84382872 398 45.72864322 399 45.86466165 400 45.75000000 401 45.88528678 402 45.77114428 403 45.65756824 404 45.79207921 405 45.92592593 406 45.81280788 407 45.70024570 408 45.58823529 409 45.72127139 410 45.60975610 411 45.49878345 412 45.38834951 413 45.27845036 414 45.41062802 415 45.54216867 416 45.43269231 417 45.32374101 418 45.21531100 419 45.34606205 420 45.47619048 421 45.60570071 422 45.49763033 423 45.39007092 424 45.51886792 425 45.41176471 426 45.30516432 427 45.19906323 428 45.09345794 429 45.22144522 430 45.11627907 431 45.24361949 432 45.37037037 433 45.26558891 434 45.16129032 435 45.05747126 436 44.95412844 437 44.85125858 438 44.97716895 439 45.10250569 440 45.00000000 441 44.89795918 442 44.79638009 443 44.69525959 444 44.59459459 445 44.49438202 446 44.39461883 447 44.51901566 448 44.41964286 449 44.32071269 450 44.44444444 451 44.56762749 452 44.46902655 453 44.59161148 454 44.49339207 455 44.39560440 456 44.51754386 457 44.42013129 458 44.32314410 459 44.22657952 460 44.34782609 461 44.46854664 462 44.58874459 463 44.70842333 464 44.82758621 465 44.73118280 466 44.63519313 467 44.75374732 468 44.87179487 469 44.98933902 470 44.89361702 471 44.79830149 472 44.70338983 473 44.60887949 474 44.51476793 475 44.42105263 476 44.53781513 477 44.44444444 478 44.56066946 479 44.46764092 480 44.58333333 481 44.49064449 482 44.39834025 483 44.30641822 484 44.21487603 485 44.12371134 486 44.03292181 487 44.14784394 488 44.05737705 489 43.96728016 490 44.08163265 491 43.99185336 492 43.90243902 493 44.01622718 494 44.12955466 495 44.24242424 496 44.35483871 497 44.26559356 498 44.17670683 499 44.28857715 500 44.20000000 501 44.11177645 502 44.02390438 503 44.13518887 504 44.24603175 505 44.15841584 506 44.26877470 507 44.18145957 508 44.09448819 509 44.00785855 510 43.92156863 511 43.83561644 512 43.75000000 513 43.66471735 514 43.57976654 515 43.68932039 516 43.79844961 517 43.90715667 518 44.01544402 519 43.93063584 520 44.03846154 521 43.95393474 522 44.06130268 523 43.97705545 524 44.08396947 525 44.00000000 526 43.91634981 527 44.02277040 528 44.12878788 529 44.04536862 530 44.15094340 531 44.06779661 532 43.98496241 533 43.90243902 534 44.00749064 535 44.11214953 536 44.02985075 537 44.13407821 538 44.23791822 539 44.15584416 540 44.07407407 541 44.17744917 542 44.28044280 543 44.19889503 544 44.30147059 545 44.22018349 546 44.13919414 547 44.24131627 548 44.34306569 549 44.26229508 550 44.18181818 551 44.28312160 552 44.38405797 553 44.30379747 554 44.40433213 555 44.32432432 556 44.42446043 557 44.34470377 558 44.44444444 559 44.54382826 560 44.46428571 561 44.38502674 562 44.48398577 563 44.58259325 564 44.50354610 565 44.60176991 566 44.52296820 567 44.44444444 568 44.36619718 569 44.28822496 570 44.38596491 571 44.30823117 572 44.40559441 573 44.32809773 574 44.25087108 575 44.17391304 576 44.27083333 577 44.36741768 578 44.46366782 579 44.38687392 580 44.48275862 581 44.57831325 582 44.50171821 583 44.59691252 584 44.69178082 585 44.78632479 586 44.88054608 587 44.80408859 588 44.72789116 589 44.65195246 590 44.57627119 591 44.50084602 592 44.59459459 593 44.68802698 594 44.78114478 595 44.70588235 596 44.79865772 597 44.72361809 598 44.64882943 599 44.74123539 600 44.66666667 601 44.75873544 602 44.68438538 603 44.77611940 604 44.70198675 605 44.62809917 606 44.55445545 607 44.64579901 608 44.57236842 609 44.49917898 610 44.42622951 611 44.51718494 612 44.44444444 613 44.53507341 614 44.62540717 615 44.55284553 616 44.64285714 617 44.57050243 618 44.66019417 619 44.58804523 620 44.51612903 621 44.44444444 622 44.37299035 623 44.30176565 624 44.23076923 625 44.16000000 626 44.24920128 627 44.33811802 628 44.26751592 629 44.35612083 630 44.28571429 631 44.21553090 632 44.30379747 633 44.23380727 634 44.16403785 635 44.25196850 636 44.18238994 637 44.11302983 638 44.20062696 639 44.28794992 640 44.37500000 641 44.30577223 642 44.39252336 643 44.32348367 644 44.40993789 645 44.34108527 646 44.27244582 647 44.20401855 648 44.13580247 649 44.06779661 650 44.15384615 651 44.08602151 652 44.17177914 653 44.10413476 654 44.18960245 655 44.12213740 656 44.20731707 657 44.29223744 658 44.37689970 659 44.30955994 660 44.24242424 661 44.17549168 662 44.10876133 663 44.04223228 664 43.97590361 665 43.90977444 666 43.84384384 667 43.92803598 668 44.01197605 669 44.09566517 670 44.02985075 671 43.96423249 672 43.89880952 673 43.83358098 674 43.76854599 675 43.70370370 676 43.63905325 677 43.57459380 678 43.65781711 679 43.59351988 680 43.67647059 681 43.75917768 682 43.69501466 683 43.63103953 684 43.71345029 685 43.64963504 686 43.58600583 687 43.52256186 688 43.45930233 689 43.39622642 690 43.33333333 691 43.41534009 692 43.35260116 693 43.29004329 694 43.22766571 695 43.16546763 696 43.24712644 697 43.18507891 698 43.12320917 699 43.20457797 700 43.28571429 701 43.36661912 702 43.44729345 703 43.52773826 704 43.46590909 705 43.40425532 706 43.48441926 707 43.42291372 708 43.36158192 709 43.44146685 710 43.52112676 711 43.60056259 712 43.67977528 713 43.75876578 714 43.83753501 715 43.77622378 716 43.71508380 717 43.65411437 718 43.59331476 719 43.67176634 720 43.61111111 721 43.55062413 722 43.62880886 723 43.70677732 724 43.64640884 725 43.72413793 726 43.80165289 727 43.74140303 728 43.81868132 729 43.75857339 730 43.69863014 731 43.63885089 732 43.57923497 733 43.65620737 734 43.73297003 735 43.67346939 736 43.61413043 737 43.55495251 738 43.63143631 739 43.70771313 740 43.78378378 741 43.72469636 742 43.80053908 743 43.87617766 744 43.95161290 745 43.89261745 746 43.83378016 747 43.77510040 748 43.71657754 749 43.65821095 750 43.73333333 Final result: 43.7333 +/- 1.8126 Random chance: 25.0000 +/- 1.5822