common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 39.13043478 24 41.66666667 25 44.00000000 26 42.30769231 27 44.44444444 28 42.85714286 29 41.37931034 30 40.00000000 31 38.70967742 32 40.62500000 33 39.39393939 34 41.17647059 35 42.85714286 36 41.66666667 37 40.54054054 38 42.10526316 39 41.02564103 40 40.00000000 41 39.02439024 42 38.09523810 43 39.53488372 44 38.63636364 45 40.00000000 46 39.13043478 47 38.29787234 48 39.58333333 49 40.81632653 50 42.00000000 51 41.17647059 52 42.30769231 53 41.50943396 54 40.74074074 55 41.81818182 56 41.07142857 57 42.10526316 58 43.10344828 59 44.06779661 60 43.33333333 61 44.26229508 62 43.54838710 63 42.85714286 64 43.75000000 65 43.07692308 66 42.42424242 67 43.28358209 68 42.64705882 69 42.02898551 70 42.85714286 71 43.66197183 72 44.44444444 73 43.83561644 74 44.59459459 75 45.33333333 76 46.05263158 77 46.75324675 78 47.43589744 79 46.83544304 80 46.25000000 81 46.91358025 82 46.34146341 83 46.98795181 84 46.42857143 85 47.05882353 86 46.51162791 87 47.12643678 88 47.72727273 89 47.19101124 90 46.66666667 91 46.15384615 92 45.65217391 93 45.16129032 94 45.74468085 95 45.26315789 96 45.83333333 97 45.36082474 98 45.91836735 99 46.46464646 100 46.00000000 101 46.53465347 102 46.07843137 103 45.63106796 104 46.15384615 105 46.66666667 106 46.22641509 107 45.79439252 108 45.37037037 109 44.95412844 110 45.45454545 111 45.94594595 112 45.53571429 113 45.13274336 114 45.61403509 115 46.08695652 116 46.55172414 117 46.15384615 118 45.76271186 119 45.37815126 120 45.00000000 121 45.45454545 122 45.08196721 123 44.71544715 124 45.16129032 125 45.60000000 126 45.23809524 127 44.88188976 128 45.31250000 129 44.96124031 130 45.38461538 131 45.80152672 132 46.21212121 133 45.86466165 134 45.52238806 135 45.18518519 136 45.58823529 137 45.98540146 138 45.65217391 139 46.04316547 140 46.42857143 141 46.80851064 142 47.18309859 143 47.55244755 144 47.22222222 145 47.58620690 146 47.94520548 147 47.61904762 148 47.97297297 149 48.32214765 150 48.00000000 151 47.68211921 152 48.02631579 153 47.71241830 154 47.40259740 155 47.09677419 156 46.79487179 157 47.13375796 158 47.46835443 159 47.16981132 160 47.50000000 161 47.20496894 162 46.91358025 163 46.62576687 164 46.34146341 165 46.06060606 166 45.78313253 167 46.10778443 168 45.83333333 169 46.15384615 170 45.88235294 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 45.45454545 177 45.19774011 178 45.50561798 179 45.81005587 180 45.55555556 181 45.30386740 182 45.05494505 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 45.21276596 189 45.50264550 190 45.26315789 191 45.54973822 192 45.31250000 193 45.07772021 194 44.84536082 195 45.12820513 196 45.40816327 197 45.17766497 198 45.45454545 199 45.22613065 200 45.00000000 201 44.77611940 202 44.55445545 203 44.33497537 204 44.11764706 205 44.39024390 206 44.17475728 207 44.44444444 208 44.23076923 209 44.01913876 210 43.80952381 211 43.60189573 212 43.39622642 213 43.19248826 214 42.99065421 215 43.25581395 216 43.05555556 217 42.85714286 218 43.11926606 219 43.37899543 220 43.18181818 221 42.98642534 222 42.79279279 223 43.04932735 224 43.30357143 225 43.55555556 226 43.80530973 227 43.61233480 228 43.85964912 229 43.66812227 230 43.47826087 231 43.29004329 232 43.53448276 233 43.34763948 234 43.58974359 235 43.40425532 236 43.22033898 237 43.03797468 238 42.85714286 239 42.67782427 240 42.91666667 241 42.73858921 242 42.97520661 243 43.20987654 244 43.03278689 245 43.26530612 246 43.08943089 247 42.91497976 248 43.14516129 249 42.97188755 250 42.80000000 251 42.62948207 252 42.46031746 253 42.68774704 254 42.51968504 255 42.35294118 256 42.57812500 257 42.80155642 258 42.63565891 259 42.47104247 260 42.69230769 261 42.91187739 262 42.74809160 263 42.58555133 264 42.80303030 265 42.64150943 266 42.85714286 267 43.07116105 268 42.91044776 269 43.12267658 270 43.33333333 271 43.17343173 272 43.38235294 273 43.22344322 274 43.06569343 275 42.90909091 276 43.11594203 277 43.32129964 278 43.52517986 279 43.36917563 280 43.21428571 281 43.06049822 282 42.90780142 283 43.10954064 284 42.95774648 285 43.15789474 286 43.00699301 287 43.20557491 288 43.05555556 289 42.90657439 290 43.10344828 291 43.29896907 292 43.49315068 293 43.68600683 294 43.53741497 295 43.38983051 296 43.24324324 297 43.09764310 298 43.28859060 299 43.47826087 300 43.66666667 301 43.52159468 302 43.70860927 303 43.56435644 304 43.42105263 305 43.60655738 306 43.79084967 307 43.97394137 308 44.15584416 309 44.33656958 310 44.19354839 311 44.37299035 312 44.55128205 313 44.72843450 314 44.90445860 315 44.76190476 316 44.62025316 317 44.47949527 318 44.33962264 319 44.51410658 320 44.37500000 321 44.23676012 322 44.40993789 323 44.27244582 324 44.13580247 325 44.30769231 326 44.17177914 327 44.34250765 328 44.51219512 329 44.68085106 330 44.54545455 331 44.41087613 332 44.57831325 333 44.74474474 334 44.61077844 335 44.77611940 336 44.94047619 337 44.80712166 338 44.67455621 339 44.54277286 340 44.70588235 341 44.86803519 342 45.02923977 343 44.89795918 344 45.05813953 345 44.92753623 346 45.08670520 347 45.24495677 348 45.11494253 349 45.27220630 350 45.42857143 351 45.29914530 352 45.17045455 353 45.32577904 354 45.19774011 355 45.07042254 356 45.22471910 357 45.37815126 358 45.25139665 359 45.12534819 360 45.27777778 361 45.42936288 362 45.58011050 363 45.45454545 364 45.60439560 365 45.47945205 366 45.62841530 367 45.50408719 368 45.38043478 369 45.25745257 370 45.40540541 371 45.55256065 372 45.43010753 373 45.30831099 374 45.45454545 375 45.60000000 376 45.47872340 377 45.35809019 378 45.50264550 379 45.38258575 380 45.26315789 381 45.40682415 382 45.54973822 383 45.69190601 384 45.57291667 385 45.71428571 386 45.59585492 387 45.47803618 388 45.36082474 389 45.50128535 390 45.38461538 391 45.52429668 392 45.66326531 393 45.54707379 394 45.43147208 395 45.56962025 396 45.70707071 397 45.59193955 398 45.47738693 399 45.61403509 400 45.50000000 401 45.63591022 402 45.52238806 403 45.40942928 404 45.54455446 405 45.43209877 406 45.32019704 407 45.20884521 408 45.09803922 409 45.23227384 410 45.12195122 411 45.01216545 412 44.90291262 413 44.79418886 414 44.92753623 415 44.81927711 416 44.71153846 417 44.60431655 418 44.49760766 419 44.63007160 420 44.52380952 421 44.41805226 422 44.31279621 423 44.20803783 424 44.33962264 425 44.23529412 426 44.36619718 427 44.26229508 428 44.15887850 429 44.28904429 430 44.18604651 431 44.31554524 432 44.21296296 433 44.11085450 434 44.00921659 435 43.90804598 436 43.80733945 437 43.70709382 438 43.83561644 439 43.73576310 440 43.63636364 441 43.53741497 442 43.43891403 443 43.34085779 444 43.24324324 445 43.14606742 446 43.04932735 447 43.17673378 448 43.08035714 449 42.98440980 450 43.11111111 451 43.23725055 452 43.36283186 453 43.48785872 454 43.39207048 455 43.29670330 456 43.42105263 457 43.32603939 458 43.23144105 459 43.13725490 460 43.26086957 461 43.38394794 462 43.50649351 463 43.62850972 464 43.75000000 465 43.65591398 466 43.56223176 467 43.68308351 468 43.80341880 469 43.92324094 470 43.82978723 471 43.73673036 472 43.64406780 473 43.55179704 474 43.45991561 475 43.36842105 476 43.48739496 477 43.39622642 478 43.51464435 479 43.63256785 480 43.54166667 481 43.45114345 482 43.36099585 483 43.27122153 484 43.18181818 485 43.09278351 486 43.00411523 487 43.12114990 488 43.03278689 489 42.94478528 490 43.06122449 491 42.97352342 492 42.88617886 493 42.79918864 494 42.91497976 495 43.03030303 496 43.14516129 497 43.05835010 498 42.97188755 499 43.08617234 500 43.00000000 501 42.91417166 502 42.82868526 503 42.94234592 504 43.05555556 505 42.97029703 506 43.08300395 507 42.99802761 508 42.91338583 509 42.82907662 510 42.74509804 511 42.66144814 512 42.57812500 513 42.49512671 514 42.41245136 515 42.52427184 516 42.63565891 517 42.74661509 518 42.85714286 519 42.77456647 520 42.88461538 521 42.80230326 522 42.91187739 523 42.82982792 524 42.93893130 525 42.85714286 526 42.77566540 527 42.88425047 528 42.99242424 529 43.10018904 530 43.20754717 531 43.12617702 532 43.04511278 533 42.96435272 534 43.07116105 535 43.17757009 536 43.09701493 537 43.20297952 538 43.30855019 539 43.22820037 540 43.14814815 541 43.25323475 542 43.35793358 543 43.27808471 544 43.38235294 545 43.30275229 546 43.22344322 547 43.32723949 548 43.43065693 549 43.35154827 550 43.27272727 551 43.37568058 552 43.47826087 553 43.39963834 554 43.50180505 555 43.42342342 556 43.52517986 557 43.44703770 558 43.54838710 559 43.64937388 560 43.57142857 561 43.49376114 562 43.59430605 563 43.69449378 564 43.61702128 565 43.71681416 566 43.63957597 567 43.56261023 568 43.48591549 569 43.40949033 570 43.50877193 571 43.43257443 572 43.53146853 573 43.45549738 574 43.37979094 575 43.30434783 576 43.40277778 577 43.50086655 578 43.59861592 579 43.52331606 580 43.62068966 581 43.71772806 582 43.64261168 583 43.73927959 584 43.83561644 585 43.93162393 586 44.02730375 587 43.95229983 588 43.87755102 589 43.80305603 590 43.72881356 591 43.65482234 592 43.75000000 593 43.84485666 594 43.93939394 595 43.86554622 596 43.95973154 597 43.88609715 598 43.97993311 599 44.07345576 600 44.00000000 601 44.09317804 602 44.01993355 603 44.11276949 604 44.03973510 605 43.96694215 606 43.89438944 607 43.98682043 608 43.91447368 609 43.84236453 610 43.77049180 611 43.86252046 612 43.79084967 613 43.71941272 614 43.81107492 615 43.73983740 616 43.83116883 617 43.76012966 618 43.85113269 619 43.78029079 620 43.70967742 621 43.63929147 622 43.56913183 623 43.49919743 624 43.42948718 625 43.36000000 626 43.45047923 627 43.38118022 628 43.31210191 629 43.40222576 630 43.33333333 631 43.26465927 632 43.35443038 633 43.28593997 634 43.21766562 635 43.30708661 636 43.23899371 637 43.17111460 638 43.26018809 639 43.34898279 640 43.43750000 641 43.36973479 642 43.45794393 643 43.39035770 644 43.47826087 645 43.41085271 646 43.34365325 647 43.27666151 648 43.20987654 649 43.14329738 650 43.23076923 651 43.16436252 652 43.25153374 653 43.18529862 654 43.27217125 655 43.20610687 656 43.29268293 657 43.37899543 658 43.46504559 659 43.39908953 660 43.33333333 661 43.26777610 662 43.20241692 663 43.13725490 664 43.07228916 665 43.00751880 666 42.94294294 667 43.02848576 668 43.11377246 669 43.19880419 670 43.28358209 671 43.21907601 672 43.15476190 673 43.09063893 674 43.02670623 675 42.96296296 676 42.89940828 677 42.83604136 678 42.92035398 679 42.85714286 680 42.94117647 681 43.02496329 682 42.96187683 683 43.04538799 684 43.12865497 685 43.06569343 686 43.00291545 687 42.94032023 688 42.87790698 689 42.81567489 690 42.75362319 691 42.83646889 692 42.77456647 693 42.71284271 694 42.65129683 695 42.58992806 696 42.67241379 697 42.61119082 698 42.55014327 699 42.63233190 700 42.71428571 701 42.65335235 702 42.73504274 703 42.81650071 704 42.75568182 705 42.69503546 706 42.77620397 707 42.71570014 708 42.65536723 709 42.73624824 710 42.81690141 711 42.89732771 712 42.97752809 713 43.05750351 714 43.13725490 715 43.07692308 716 43.01675978 717 42.95676430 718 43.03621170 719 43.11543811 720 43.05555556 721 42.99583911 722 43.07479224 723 43.15352697 724 43.09392265 725 43.17241379 726 43.25068871 727 43.32874828 728 43.40659341 729 43.34705075 730 43.28767123 731 43.22845417 732 43.16939891 733 43.24693042 734 43.32425068 735 43.40136054 736 43.34239130 737 43.28358209 738 43.36043360 739 43.43707713 740 43.51351351 741 43.45479082 742 43.53099730 743 43.60699865 744 43.68279570 745 43.62416107 746 43.56568365 747 43.50736278 748 43.44919786 749 43.39118825 750 43.46666667 Final result: 43.4667 +/- 1.8113 Random chance: 25.0000 +/- 1.5822