common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 71.42857143 8 75.00000000 9 66.66666667 10 60.00000000 11 54.54545455 12 58.33333333 13 53.84615385 14 57.14285714 15 60.00000000 16 62.50000000 17 64.70588235 18 66.66666667 19 68.42105263 20 65.00000000 21 66.66666667 22 68.18181818 23 69.56521739 24 70.83333333 25 68.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 65.51724138 30 66.66666667 31 64.51612903 32 65.62500000 33 66.66666667 34 64.70588235 35 65.71428571 36 63.88888889 37 64.86486486 38 65.78947368 39 66.66666667 40 67.50000000 41 65.85365854 42 66.66666667 43 65.11627907 44 63.63636364 45 64.44444444 46 65.21739130 47 65.95744681 48 66.66666667 49 67.34693878 50 66.00000000 51 64.70588235 52 65.38461538 53 66.03773585 54 64.81481481 55 63.63636364 56 64.28571429 57 64.91228070 58 65.51724138 59 66.10169492 60 65.00000000 61 65.57377049 62 66.12903226 63 66.66666667 64 67.18750000 65 66.15384615 66 66.66666667 67 67.16417910 68 67.64705882 69 68.11594203 70 68.57142857 71 69.01408451 72 68.05555556 73 68.49315068 74 68.91891892 75 69.33333333 76 69.73684211 77 70.12987013 78 70.51282051 79 70.88607595 80 71.25000000 81 71.60493827 82 71.95121951 83 71.08433735 84 70.23809524 85 70.58823529 86 70.93023256 87 71.26436782 88 71.59090909 89 71.91011236 90 72.22222222 91 72.52747253 92 72.82608696 93 72.04301075 94 71.27659574 95 71.57894737 96 71.87500000 97 71.13402062 98 71.42857143 99 71.71717172 100 72.00000000 101 71.28712871 102 71.56862745 103 70.87378641 104 70.19230769 105 70.47619048 106 69.81132075 107 70.09345794 108 69.44444444 109 68.80733945 110 68.18181818 111 68.46846847 112 68.75000000 113 68.14159292 114 67.54385965 115 67.82608696 116 67.24137931 117 67.52136752 118 67.79661017 119 67.22689076 120 67.50000000 121 66.94214876 122 67.21311475 123 67.47967480 124 66.93548387 125 67.20000000 126 67.46031746 127 67.71653543 128 67.96875000 129 67.44186047 130 67.69230769 131 67.93893130 132 67.42424242 133 67.66917293 134 67.16417910 135 66.66666667 136 66.91176471 137 67.15328467 138 66.66666667 139 66.90647482 140 66.42857143 141 65.95744681 142 66.19718310 143 66.43356643 144 66.66666667 145 66.89655172 146 66.43835616 147 66.66666667 148 66.21621622 149 66.44295302 150 66.66666667 151 66.88741722 152 67.10526316 153 67.32026144 154 66.88311688 155 67.09677419 156 67.30769231 157 67.51592357 158 67.72151899 159 67.92452830 160 68.12500000 161 68.32298137 162 68.51851852 163 68.09815951 164 68.29268293 165 68.48484848 166 68.67469880 167 68.86227545 168 69.04761905 169 68.63905325 170 68.23529412 171 68.42105263 172 68.60465116 173 68.78612717 174 68.96551724 175 68.57142857 176 68.75000000 177 68.92655367 178 69.10112360 179 69.27374302 180 69.44444444 181 69.06077348 182 69.23076923 183 68.85245902 184 69.02173913 185 69.18918919 186 69.35483871 187 69.51871658 188 69.14893617 189 69.31216931 190 68.94736842 191 69.10994764 192 68.75000000 193 68.39378238 194 68.55670103 195 68.71794872 196 68.87755102 197 69.03553299 198 69.19191919 199 68.84422111 200 69.00000000 201 69.15422886 202 69.30693069 203 69.45812808 204 69.60784314 205 69.75609756 206 69.41747573 207 69.56521739 208 69.71153846 209 69.85645933 210 70.00000000 211 70.14218009 212 69.81132075 213 69.95305164 214 70.09345794 215 70.23255814 216 69.90740741 217 70.04608295 218 70.18348624 219 70.31963470 220 70.00000000 221 70.13574661 222 70.27027027 223 69.95515695 224 70.08928571 225 69.77777778 226 69.46902655 227 69.60352423 228 69.73684211 229 69.86899563 230 70.00000000 231 69.69696970 232 69.39655172 233 69.09871245 234 68.80341880 235 68.51063830 236 68.64406780 237 68.35443038 238 68.06722689 239 68.20083682 240 68.33333333 241 68.46473029 242 68.18181818 243 68.31275720 244 68.03278689 245 68.16326531 246 68.29268293 247 68.42105263 248 68.54838710 249 68.27309237 250 68.40000000 251 68.52589641 252 68.65079365 253 68.77470356 254 68.89763780 255 69.01960784 256 69.14062500 257 68.87159533 258 68.99224806 259 69.11196911 260 68.84615385 261 68.96551724 262 69.08396947 263 69.20152091 264 69.31818182 265 69.43396226 266 69.17293233 267 69.28838951 268 69.40298507 269 69.51672862 270 69.62962963 271 69.74169742 272 69.85294118 273 69.96336996 274 69.70802920 275 69.45454545 276 69.56521739 277 69.67509025 278 69.78417266 279 69.89247312 280 70.00000000 281 70.10676157 282 70.21276596 283 69.96466431 284 70.07042254 285 70.17543860 286 70.27972028 287 70.38327526 288 70.48611111 289 70.24221453 290 70.34482759 291 70.44673540 292 70.54794521 293 70.64846416 294 70.40816327 295 70.50847458 296 70.60810811 297 70.70707071 298 70.80536913 299 70.90301003 300 71.00000000 301 70.76411960 302 70.86092715 303 70.95709571 304 71.05263158 305 71.14754098 306 70.91503268 307 71.00977199 308 71.10389610 309 71.19741100 310 70.96774194 311 71.06109325 312 70.83333333 313 70.92651757 314 71.01910828 315 71.11111111 316 71.20253165 317 71.29337539 318 71.38364780 319 71.15987461 320 70.93750000 321 71.02803738 322 71.11801242 323 71.20743034 324 71.29629630 325 71.38461538 326 71.47239264 327 71.25382263 328 71.34146341 329 71.42857143 330 71.21212121 331 71.29909366 332 71.38554217 333 71.47147147 334 71.55688623 335 71.34328358 336 71.13095238 337 71.21661721 338 71.00591716 339 71.09144543 340 71.17647059 341 71.26099707 342 71.34502924 343 71.42857143 344 71.51162791 345 71.30434783 346 71.38728324 347 71.46974063 348 71.26436782 349 71.34670487 350 71.42857143 351 71.50997151 352 71.59090909 353 71.67138810 354 71.46892655 355 71.26760563 356 71.34831461 357 71.42857143 358 71.50837989 359 71.58774373 360 71.66666667 361 71.74515235 362 71.82320442 363 71.90082645 364 71.97802198 365 72.05479452 366 71.85792350 367 71.93460490 368 72.01086957 369 72.08672087 370 72.16216216 371 72.23719677 372 72.31182796 373 72.38605898 374 72.45989305 375 72.26666667 376 72.34042553 377 72.41379310 378 72.48677249 379 72.29551451 380 72.36842105 381 72.17847769 382 71.98952880 383 71.80156658 384 71.87500000 385 71.94805195 386 71.76165803 387 71.83462532 388 71.90721649 389 71.72236504 390 71.53846154 391 71.35549872 392 71.42857143 393 71.50127226 394 71.31979695 395 71.39240506 396 71.21212121 397 71.28463476 398 71.10552764 399 71.17794486 400 71.00000000 401 71.07231920 402 71.14427861 403 71.21588089 404 71.28712871 405 71.35802469 406 71.42857143 407 71.49877150 408 71.56862745 409 71.39364303 410 71.46341463 411 71.53284672 412 71.60194175 413 71.67070218 414 71.73913043 415 71.80722892 416 71.87500000 417 71.70263789 418 71.77033493 419 71.59904535 420 71.66666667 421 71.73396675 422 71.80094787 423 71.86761229 424 71.93396226 425 71.76470588 426 71.83098592 427 71.66276347 428 71.49532710 429 71.56177156 430 71.39534884 431 71.46171694 432 71.52777778 433 71.36258661 434 71.42857143 435 71.49425287 436 71.55963303 437 71.62471396 438 71.46118721 439 71.52619590 440 71.59090909 441 71.65532880 442 71.49321267 443 71.55756208 444 71.62162162 445 71.68539326 446 71.74887892 447 71.81208054 448 71.87500000 449 71.93763920 450 71.77777778 451 71.84035477 452 71.90265487 453 71.96467991 454 71.80616740 455 71.86813187 456 71.92982456 457 71.99124726 458 72.05240175 459 72.11328976 460 71.95652174 461 72.01735358 462 72.07792208 463 71.92224622 464 71.76724138 465 71.82795699 466 71.88841202 467 71.94860814 468 72.00854701 469 71.85501066 470 71.91489362 471 71.97452229 472 72.03389831 473 71.88160677 474 71.94092827 475 71.78947368 476 71.84873950 477 71.90775681 478 71.96652720 479 72.02505219 480 71.87500000 481 71.93347193 482 71.99170124 483 72.04968944 484 72.10743802 485 72.16494845 486 72.22222222 487 72.07392197 488 72.13114754 489 72.18813906 490 72.24489796 491 72.30142566 492 72.35772358 493 72.21095335 494 72.26720648 495 72.32323232 496 72.17741935 497 72.03219316 498 72.08835341 499 72.14428858 500 72.20000000 501 72.25548902 502 72.31075697 503 72.36580517 504 72.42063492 505 72.47524752 506 72.52964427 507 72.58382643 508 72.63779528 509 72.69155206 510 72.54901961 511 72.40704501 512 72.46093750 513 72.51461988 514 72.56809339 515 72.62135922 516 72.48062016 517 72.53384913 518 72.58687259 519 72.63969171 520 72.50000000 521 72.55278311 522 72.60536398 523 72.46653920 524 72.32824427 525 72.19047619 526 72.05323194 527 71.91650854 528 71.78030303 529 71.64461248 530 71.69811321 531 71.75141243 532 71.80451128 533 71.85741088 534 71.72284644 535 71.58878505 536 71.64179104 537 71.69459963 538 71.56133829 539 71.61410019 540 71.48148148 541 71.53419593 542 71.58671587 543 71.45488029 544 71.50735294 545 71.55963303 546 71.61172161 547 71.48080439 548 71.53284672 549 71.58469945 550 71.63636364 551 71.50635209 552 71.37681159 553 71.42857143 554 71.48014440 555 71.53153153 556 71.40287770 557 71.45421903 558 71.32616487 559 71.37745975 560 71.42857143 561 71.47950089 562 71.53024911 563 71.58081705 564 71.63120567 565 71.50442478 566 71.55477032 567 71.60493827 568 71.65492958 569 71.52899824 570 71.40350877 571 71.45359019 572 71.32867133 573 71.37870855 574 71.42857143 575 71.47826087 576 71.52777778 577 71.57712305 578 71.45328720 579 71.50259067 580 71.55172414 581 71.42857143 582 71.47766323 583 71.35506003 584 71.40410959 585 71.45299145 586 71.50170648 587 71.55025554 588 71.59863946 589 71.64685908 590 71.69491525 591 71.74280880 592 71.62162162 593 71.66947723 594 71.71717172 595 71.76470588 596 71.64429530 597 71.69179229 598 71.73913043 599 71.61936561 600 71.66666667 601 71.71381032 602 71.59468439 603 71.47595357 604 71.52317881 605 71.57024793 606 71.61716172 607 71.66392092 608 71.71052632 609 71.59277504 610 71.63934426 611 71.52209493 612 71.56862745 613 71.45187602 614 71.49837134 615 71.54471545 616 71.42857143 617 71.47487844 618 71.52103560 619 71.40549273 620 71.45161290 621 71.49758454 622 71.38263666 623 71.42857143 624 71.31410256 625 71.36000000 626 71.40575080 627 71.45135566 628 71.33757962 629 71.38314785 630 71.42857143 631 71.47385103 632 71.36075949 633 71.24802528 634 71.29337539 635 71.33858268 636 71.38364780 637 71.27158556 638 71.31661442 639 71.36150235 640 71.40625000 641 71.45085803 642 71.33956386 643 71.22861586 644 71.27329193 645 71.16279070 646 71.20743034 647 71.09737249 648 71.14197531 649 71.03235747 650 71.07692308 651 71.12135177 652 71.01226994 653 71.05666156 654 70.94801223 655 70.83969466 656 70.73170732 657 70.77625571 658 70.82066869 659 70.71320182 660 70.75757576 661 70.65052950 662 70.69486405 663 70.73906486 664 70.78313253 665 70.82706767 666 70.87087087 667 70.91454273 668 70.80838323 669 70.85201794 670 70.89552239 671 70.93889717 672 70.98214286 673 71.02526003 674 71.06824926 675 71.11111111 676 71.15384615 677 71.19645495 678 71.23893805 679 71.28129602 680 71.32352941 681 71.36563877 682 71.40762463 683 71.30307467 684 71.34502924 685 71.38686131 686 71.28279883 687 71.17903930 688 71.22093023 689 71.11756168 690 71.15942029 691 71.20115774 692 71.09826590 693 71.13997114 694 71.18155620 695 71.07913669 696 70.97701149 697 71.01865136 698 70.91690544 699 70.95851216 700 70.85714286 701 70.75606277 702 70.79772080 703 70.83926031 704 70.88068182 705 70.92198582 706 70.96317280 707 71.00424328 708 71.04519774 709 71.08603667 710 71.12676056 711 71.16736990 712 71.20786517 713 71.24824684 714 71.14845938 715 71.04895105 716 70.94972067 717 70.99023710 718 70.89136490 719 70.79276773 720 70.83333333 721 70.73509015 722 70.77562327 723 70.81604426 724 70.71823204 725 70.75862069 726 70.66115702 727 70.70151307 728 70.74175824 729 70.78189300 730 70.82191781 731 70.86183311 732 70.90163934 733 70.94133697 734 70.98092643 735 71.02040816 736 71.05978261 737 70.96336499 738 71.00271003 739 70.90663058 740 70.81081081 741 70.71524966 742 70.75471698 743 70.79407806 744 70.69892473 745 70.73825503 746 70.77747989 747 70.68273092 748 70.58823529 749 70.49399199 750 70.53333333 Final result: 70.5333 +/- 1.6658 Random chance: 25.0083 +/- 1.5824