common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 0.00000000 2 0.00000000 3 0.00000000 4 25.00000000 5 20.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 43.47826087 24 45.83333333 25 48.00000000 26 46.15384615 27 48.14814815 28 50.00000000 29 48.27586207 30 46.66666667 31 48.38709677 32 50.00000000 33 48.48484848 34 50.00000000 35 51.42857143 36 50.00000000 37 48.64864865 38 50.00000000 39 48.71794872 40 47.50000000 41 46.34146341 42 45.23809524 43 46.51162791 44 45.45454545 45 46.66666667 46 45.65217391 47 44.68085106 48 45.83333333 49 46.93877551 50 48.00000000 51 47.05882353 52 46.15384615 53 45.28301887 54 44.44444444 55 45.45454545 56 46.42857143 57 47.36842105 58 46.55172414 59 47.45762712 60 46.66666667 61 47.54098361 62 46.77419355 63 46.03174603 64 46.87500000 65 46.15384615 66 46.96969697 67 47.76119403 68 47.05882353 69 46.37681159 70 45.71428571 71 46.47887324 72 47.22222222 73 46.57534247 74 47.29729730 75 48.00000000 76 48.68421053 77 49.35064935 78 50.00000000 79 49.36708861 80 48.75000000 81 49.38271605 82 48.78048780 83 49.39759036 84 48.80952381 85 48.23529412 86 48.83720930 87 49.42528736 88 50.00000000 89 49.43820225 90 48.88888889 91 48.35164835 92 47.82608696 93 47.31182796 94 47.87234043 95 47.36842105 96 47.91666667 97 47.42268041 98 47.95918367 99 47.47474747 100 47.00000000 101 47.52475248 102 47.05882353 103 46.60194175 104 47.11538462 105 47.61904762 106 47.16981132 107 46.72897196 108 46.29629630 109 45.87155963 110 46.36363636 111 46.84684685 112 47.32142857 113 46.90265487 114 47.36842105 115 47.82608696 116 48.27586207 117 47.86324786 118 47.45762712 119 47.05882353 120 46.66666667 121 47.10743802 122 46.72131148 123 46.34146341 124 46.77419355 125 47.20000000 126 46.82539683 127 46.45669291 128 46.87500000 129 46.51162791 130 46.92307692 131 47.32824427 132 47.72727273 133 47.36842105 134 47.01492537 135 46.66666667 136 47.05882353 137 47.44525547 138 47.10144928 139 47.48201439 140 47.85714286 141 48.22695035 142 48.59154930 143 48.95104895 144 48.61111111 145 48.27586207 146 48.63013699 147 48.29931973 148 48.64864865 149 48.99328859 150 48.66666667 151 48.34437086 152 48.68421053 153 48.36601307 154 48.05194805 155 47.74193548 156 47.43589744 157 47.77070064 158 48.10126582 159 47.79874214 160 48.12500000 161 47.82608696 162 47.53086420 163 47.23926380 164 46.95121951 165 46.66666667 166 46.38554217 167 46.70658683 168 46.42857143 169 46.74556213 170 46.47058824 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 45.45454545 177 45.19774011 178 45.50561798 179 45.25139665 180 45.00000000 181 44.75138122 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 45.45454545 188 45.74468085 189 46.03174603 190 45.78947368 191 46.07329843 192 45.83333333 193 45.59585492 194 45.36082474 195 45.64102564 196 45.91836735 197 45.68527919 198 45.95959596 199 45.72864322 200 45.50000000 201 45.27363184 202 45.04950495 203 44.82758621 204 44.60784314 205 44.87804878 206 44.66019417 207 44.92753623 208 44.71153846 209 44.49760766 210 44.28571429 211 44.07582938 212 43.86792453 213 43.66197183 214 43.45794393 215 43.72093023 216 43.51851852 217 43.31797235 218 43.57798165 219 43.83561644 220 43.63636364 221 43.89140271 222 43.69369369 223 43.94618834 224 44.19642857 225 44.44444444 226 44.69026549 227 44.49339207 228 44.73684211 229 44.54148472 230 44.34782609 231 44.15584416 232 44.39655172 233 44.20600858 234 44.44444444 235 44.25531915 236 44.06779661 237 43.88185654 238 43.69747899 239 43.51464435 240 43.75000000 241 43.56846473 242 43.80165289 243 44.03292181 244 44.26229508 245 44.48979592 246 44.30894309 247 44.12955466 248 44.35483871 249 44.17670683 250 44.00000000 251 43.82470120 252 43.65079365 253 43.87351779 254 43.70078740 255 43.52941176 256 43.75000000 257 43.96887160 258 43.79844961 259 43.62934363 260 43.84615385 261 44.06130268 262 43.89312977 263 43.72623574 264 43.93939394 265 43.77358491 266 43.98496241 267 44.19475655 268 44.02985075 269 44.23791822 270 44.44444444 271 44.28044280 272 44.48529412 273 44.32234432 274 44.16058394 275 44.00000000 276 44.20289855 277 44.40433213 278 44.60431655 279 44.44444444 280 44.64285714 281 44.48398577 282 44.32624113 283 44.16961131 284 44.01408451 285 44.21052632 286 44.05594406 287 44.25087108 288 44.09722222 289 43.94463668 290 44.13793103 291 44.32989691 292 44.52054795 293 44.70989761 294 44.55782313 295 44.40677966 296 44.25675676 297 44.10774411 298 44.29530201 299 44.48160535 300 44.66666667 301 44.85049834 302 45.03311258 303 44.88448845 304 45.06578947 305 45.24590164 306 45.42483660 307 45.60260586 308 45.45454545 309 45.63106796 310 45.48387097 311 45.65916399 312 45.83333333 313 46.00638978 314 46.17834395 315 46.03174603 316 45.88607595 317 45.74132492 318 45.59748428 319 45.76802508 320 45.62500000 321 45.48286604 322 45.65217391 323 45.51083591 324 45.37037037 325 45.53846154 326 45.39877301 327 45.56574924 328 45.73170732 329 45.89665653 330 45.75757576 331 45.61933535 332 45.48192771 333 45.64564565 334 45.50898204 335 45.67164179 336 45.83333333 337 45.69732938 338 45.56213018 339 45.42772861 340 45.58823529 341 45.74780059 342 45.90643275 343 45.77259475 344 45.93023256 345 45.79710145 346 45.95375723 347 46.10951009 348 45.97701149 349 45.84527221 350 46.00000000 351 45.86894587 352 45.73863636 353 45.89235127 354 45.76271186 355 45.63380282 356 45.78651685 357 45.65826331 358 45.53072626 359 45.40389972 360 45.55555556 361 45.70637119 362 45.85635359 363 45.73002755 364 45.87912088 365 45.75342466 366 45.90163934 367 45.77656676 368 45.65217391 369 45.52845528 370 45.67567568 371 45.82210243 372 45.69892473 373 45.57640751 374 45.72192513 375 45.86666667 376 45.74468085 377 45.62334218 378 45.76719577 379 45.64643799 380 45.52631579 381 45.66929134 382 45.81151832 383 45.95300261 384 45.83333333 385 45.97402597 386 45.85492228 387 45.73643411 388 45.61855670 389 45.75835476 390 45.64102564 391 45.78005115 392 45.91836735 393 45.80152672 394 45.68527919 395 45.82278481 396 45.95959596 397 46.09571788 398 45.97989950 399 46.11528822 400 46.00000000 401 46.13466334 402 46.01990050 403 45.90570720 404 46.03960396 405 46.17283951 406 46.05911330 407 45.94594595 408 45.83333333 409 45.96577017 410 45.85365854 411 45.74209246 412 45.63106796 413 45.52058111 414 45.65217391 415 45.78313253 416 45.67307692 417 45.56354916 418 45.45454545 419 45.58472554 420 45.71428571 421 45.84323040 422 45.73459716 423 45.62647754 424 45.75471698 425 45.64705882 426 45.53990610 427 45.43325527 428 45.32710280 429 45.45454545 430 45.34883721 431 45.47563805 432 45.60185185 433 45.49653580 434 45.39170507 435 45.28735632 436 45.18348624 437 45.08009153 438 45.20547945 439 45.33029613 440 45.22727273 441 45.12471655 442 45.02262443 443 44.92099323 444 44.81981982 445 44.71910112 446 44.61883408 447 44.74272931 448 44.64285714 449 44.54342984 450 44.66666667 451 44.78935698 452 44.69026549 453 44.59161148 454 44.49339207 455 44.39560440 456 44.51754386 457 44.42013129 458 44.32314410 459 44.22657952 460 44.34782609 461 44.46854664 462 44.58874459 463 44.70842333 464 44.82758621 465 44.73118280 466 44.63519313 467 44.75374732 468 44.87179487 469 44.98933902 470 44.89361702 471 44.79830149 472 44.70338983 473 44.60887949 474 44.51476793 475 44.42105263 476 44.53781513 477 44.44444444 478 44.56066946 479 44.46764092 480 44.58333333 481 44.49064449 482 44.39834025 483 44.30641822 484 44.21487603 485 44.12371134 486 44.03292181 487 44.14784394 488 44.05737705 489 43.96728016 490 44.08163265 491 43.99185336 492 43.90243902 493 44.01622718 494 44.12955466 495 44.24242424 496 44.35483871 497 44.26559356 498 44.17670683 499 44.28857715 500 44.20000000 501 44.11177645 502 44.02390438 503 44.13518887 504 44.24603175 505 44.15841584 506 44.26877470 507 44.18145957 508 44.09448819 509 44.00785855 510 43.92156863 511 43.83561644 512 43.75000000 513 43.66471735 514 43.57976654 515 43.68932039 516 43.79844961 517 43.90715667 518 44.01544402 519 43.93063584 520 44.03846154 521 43.95393474 522 44.06130268 523 43.97705545 524 44.08396947 525 44.00000000 526 43.91634981 527 43.83301708 528 43.93939394 529 43.85633270 530 43.96226415 531 43.87947269 532 43.79699248 533 43.71482176 534 43.82022472 535 43.92523364 536 43.84328358 537 43.94785847 538 44.05204461 539 43.97031540 540 43.88888889 541 43.99260628 542 44.09594096 543 44.01473297 544 44.11764706 545 44.03669725 546 43.95604396 547 44.05850091 548 44.16058394 549 44.08014572 550 44.00000000 551 44.10163339 552 44.20289855 553 44.12296564 554 44.22382671 555 44.14414414 556 44.24460432 557 44.16517056 558 44.26523297 559 44.36493739 560 44.28571429 561 44.20677362 562 44.30604982 563 44.40497336 564 44.32624113 565 44.42477876 566 44.52296820 567 44.44444444 568 44.36619718 569 44.28822496 570 44.38596491 571 44.30823117 572 44.40559441 573 44.32809773 574 44.25087108 575 44.17391304 576 44.27083333 577 44.36741768 578 44.46366782 579 44.38687392 580 44.48275862 581 44.57831325 582 44.50171821 583 44.59691252 584 44.69178082 585 44.78632479 586 44.88054608 587 44.80408859 588 44.72789116 589 44.65195246 590 44.57627119 591 44.50084602 592 44.59459459 593 44.68802698 594 44.78114478 595 44.70588235 596 44.79865772 597 44.72361809 598 44.64882943 599 44.74123539 600 44.66666667 601 44.75873544 602 44.68438538 603 44.77611940 604 44.70198675 605 44.62809917 606 44.55445545 607 44.64579901 608 44.57236842 609 44.49917898 610 44.42622951 611 44.51718494 612 44.44444444 613 44.53507341 614 44.62540717 615 44.55284553 616 44.64285714 617 44.57050243 618 44.66019417 619 44.58804523 620 44.51612903 621 44.44444444 622 44.37299035 623 44.30176565 624 44.23076923 625 44.16000000 626 44.24920128 627 44.33811802 628 44.26751592 629 44.35612083 630 44.28571429 631 44.21553090 632 44.30379747 633 44.23380727 634 44.16403785 635 44.25196850 636 44.18238994 637 44.11302983 638 44.20062696 639 44.28794992 640 44.37500000 641 44.30577223 642 44.39252336 643 44.32348367 644 44.40993789 645 44.34108527 646 44.27244582 647 44.20401855 648 44.13580247 649 44.06779661 650 44.15384615 651 44.08602151 652 44.17177914 653 44.10413476 654 44.18960245 655 44.12213740 656 44.20731707 657 44.29223744 658 44.37689970 659 44.30955994 660 44.24242424 661 44.17549168 662 44.10876133 663 44.04223228 664 43.97590361 665 43.90977444 666 43.84384384 667 43.92803598 668 44.01197605 669 44.09566517 670 44.02985075 671 43.96423249 672 43.89880952 673 43.83358098 674 43.76854599 675 43.70370370 676 43.63905325 677 43.57459380 678 43.65781711 679 43.59351988 680 43.67647059 681 43.75917768 682 43.69501466 683 43.63103953 684 43.71345029 685 43.64963504 686 43.58600583 687 43.66812227 688 43.60465116 689 43.54136430 690 43.47826087 691 43.56005789 692 43.49710983 693 43.43434343 694 43.37175793 695 43.30935252 696 43.39080460 697 43.32855093 698 43.26647564 699 43.34763948 700 43.42857143 701 43.50927247 702 43.58974359 703 43.66998578 704 43.60795455 705 43.54609929 706 43.62606232 707 43.56435644 708 43.50282486 709 43.58251058 710 43.66197183 711 43.74120956 712 43.82022472 713 43.89901823 714 43.97759104 715 43.91608392 716 43.85474860 717 43.79358438 718 43.73259053 719 43.81084840 720 43.75000000 721 43.68932039 722 43.76731302 723 43.84508990 724 43.78453039 725 43.86206897 726 43.93939394 727 43.87895461 728 43.95604396 729 43.89574760 730 43.83561644 731 43.77564979 732 43.71584699 733 43.79263302 734 43.86920981 735 43.80952381 736 43.75000000 737 43.69063772 738 43.76693767 739 43.84303112 740 43.78378378 741 43.72469636 742 43.80053908 743 43.87617766 744 43.95161290 745 43.89261745 746 43.83378016 747 43.77510040 748 43.71657754 749 43.65821095 750 43.73333333 Final result: 43.7333 +/- 1.8126 Random chance: 25.0000 +/- 1.5822