common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 31.81818182 45 33.33333333 46 32.60869565 47 34.04255319 48 33.33333333 49 32.65306122 50 34.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 34.37500000 65 33.84615385 66 34.84848485 67 34.32835821 68 33.82352941 69 33.33333333 70 34.28571429 71 33.80281690 72 34.72222222 73 34.24657534 74 33.78378378 75 33.33333333 76 34.21052632 77 33.76623377 78 33.33333333 79 32.91139241 80 32.50000000 81 32.09876543 82 32.92682927 83 32.53012048 84 32.14285714 85 31.76470588 86 31.39534884 87 31.03448276 88 30.68181818 89 31.46067416 90 31.11111111 91 30.76923077 92 30.43478261 93 30.10752688 94 29.78723404 95 29.47368421 96 30.20833333 97 30.92783505 98 30.61224490 99 31.31313131 100 32.00000000 101 32.67326733 102 32.35294118 103 33.00970874 104 32.69230769 105 32.38095238 106 33.01886792 107 33.64485981 108 34.25925926 109 34.86238532 110 35.45454545 111 35.13513514 112 35.71428571 113 35.39823009 114 35.96491228 115 35.65217391 116 36.20689655 117 36.75213675 118 36.44067797 119 36.13445378 120 35.83333333 121 36.36363636 122 36.06557377 123 36.58536585 124 36.29032258 125 36.80000000 126 36.50793651 127 36.22047244 128 35.93750000 129 36.43410853 130 36.92307692 131 36.64122137 132 36.36363636 133 36.09022556 134 35.82089552 135 35.55555556 136 35.29411765 137 35.76642336 138 36.23188406 139 36.69064748 140 36.42857143 141 36.87943262 142 36.61971831 143 36.36363636 144 36.80555556 145 37.24137931 146 36.98630137 147 36.73469388 148 36.48648649 149 36.24161074 150 36.00000000 151 36.42384106 152 36.18421053 153 35.94771242 154 35.71428571 155 35.48387097 156 35.89743590 157 36.30573248 158 36.70886076 159 37.10691824 160 36.87500000 161 36.64596273 162 36.41975309 163 36.80981595 164 36.58536585 165 36.96969697 166 37.34939759 167 37.12574850 168 36.90476190 169 36.68639053 170 36.47058824 171 36.25730994 172 36.04651163 173 35.83815029 174 36.20689655 175 36.00000000 176 35.79545455 177 35.59322034 178 35.39325843 179 35.19553073 180 35.00000000 181 34.80662983 182 34.61538462 183 34.42622951 184 34.23913043 185 34.59459459 186 34.94623656 187 34.75935829 188 34.57446809 189 34.39153439 190 34.21052632 191 34.03141361 192 33.85416667 193 33.67875648 194 34.02061856 195 33.84615385 196 34.18367347 197 34.51776650 198 34.84848485 199 34.67336683 200 35.00000000 201 34.82587065 202 35.14851485 203 34.97536946 204 34.80392157 205 35.12195122 206 34.95145631 207 35.26570048 208 35.09615385 209 34.92822967 210 34.76190476 211 34.59715640 212 34.43396226 213 34.27230047 214 34.11214953 215 33.95348837 216 34.25925926 217 34.10138249 218 33.94495413 219 33.78995434 220 33.63636364 221 33.93665158 222 33.78378378 223 33.63228700 224 33.92857143 225 34.22222222 226 34.51327434 227 34.80176211 228 34.64912281 229 34.49781659 230 34.34782609 231 34.19913420 232 34.48275862 233 34.33476395 234 34.61538462 235 34.46808511 236 34.74576271 237 34.59915612 238 34.87394958 239 34.72803347 240 34.58333333 241 34.43983402 242 34.29752066 243 34.15637860 244 34.01639344 245 33.87755102 246 33.73983740 247 34.00809717 248 34.27419355 249 34.53815261 250 34.40000000 251 34.66135458 252 34.52380952 253 34.38735178 254 34.64566929 255 34.90196078 256 34.76562500 257 34.63035019 258 34.49612403 259 34.36293436 260 34.23076923 261 34.48275862 262 34.35114504 263 34.22053232 264 34.46969697 265 34.71698113 266 34.58646617 267 34.45692884 268 34.32835821 269 34.20074349 270 34.07407407 271 33.94833948 272 33.82352941 273 33.69963370 274 33.94160584 275 33.81818182 276 33.69565217 277 33.57400722 278 33.45323741 279 33.33333333 280 33.57142857 281 33.80782918 282 33.68794326 283 33.56890459 284 33.80281690 285 33.68421053 286 33.91608392 287 34.14634146 288 34.37500000 289 34.25605536 290 34.13793103 291 34.36426117 292 34.24657534 293 34.47098976 294 34.35374150 295 34.57627119 296 34.79729730 297 34.68013468 298 34.56375839 299 34.44816054 300 34.33333333 301 34.21926910 302 34.10596026 303 34.32343234 304 34.21052632 305 34.09836066 306 33.98692810 307 33.87622150 308 34.09090909 309 33.98058252 310 33.87096774 311 33.76205788 312 33.97435897 313 33.86581470 314 33.75796178 315 33.65079365 316 33.54430380 317 33.43848580 318 33.33333333 319 33.22884013 320 33.12500000 321 33.02180685 322 33.22981366 323 33.12693498 324 33.02469136 325 32.92307692 326 32.82208589 327 33.02752294 328 33.23170732 329 33.13069909 330 33.33333333 331 33.23262840 332 33.43373494 333 33.33333333 334 33.23353293 335 33.43283582 336 33.63095238 337 33.53115727 338 33.43195266 339 33.62831858 340 33.52941176 341 33.72434018 342 33.91812865 343 33.81924198 344 34.01162791 345 34.20289855 346 34.10404624 347 34.29394813 348 34.19540230 349 34.09742120 350 34.00000000 351 33.90313390 352 33.80681818 353 33.71104816 354 33.61581921 355 33.80281690 356 33.70786517 357 33.89355742 358 34.07821229 359 33.98328691 360 33.88888889 361 34.07202216 362 33.97790055 363 33.88429752 364 33.79120879 365 33.69863014 366 33.60655738 367 33.78746594 368 33.69565217 369 33.60433604 370 33.51351351 371 33.42318059 372 33.60215054 373 33.51206434 374 33.68983957 375 33.60000000 376 33.51063830 377 33.68700265 378 33.59788360 379 33.50923483 380 33.42105263 381 33.33333333 382 33.50785340 383 33.42036554 384 33.33333333 385 33.24675325 386 33.16062176 387 33.07493540 388 33.24742268 389 33.16195373 390 33.07692308 391 32.99232737 392 32.90816327 393 33.07888041 394 33.24873096 395 33.16455696 396 33.08080808 397 33.24937028 398 33.16582915 399 33.08270677 400 33.00000000 401 33.16708229 402 33.33333333 403 33.49875931 404 33.41584158 405 33.33333333 406 33.25123153 407 33.16953317 408 33.08823529 409 33.00733496 410 32.92682927 411 32.84671533 412 33.00970874 413 33.17191283 414 33.33333333 415 33.49397590 416 33.41346154 417 33.33333333 418 33.25358852 419 33.17422434 420 33.33333333 421 33.49168646 422 33.41232227 423 33.33333333 424 33.25471698 425 33.17647059 426 33.09859155 427 33.02107728 428 32.94392523 429 32.86713287 430 33.02325581 431 32.94663573 432 33.10185185 433 33.25635104 434 33.41013825 435 33.33333333 436 33.25688073 437 33.18077803 438 33.33333333 439 33.48519362 440 33.40909091 441 33.33333333 442 33.25791855 443 33.18284424 444 33.33333333 445 33.25842697 446 33.18385650 447 33.10961969 448 33.03571429 449 33.18485523 450 33.11111111 451 33.03769401 452 32.96460177 453 33.11258278 454 33.03964758 455 32.96703297 456 32.89473684 457 32.82275711 458 32.75109170 459 32.67973856 460 32.60869565 461 32.53796095 462 32.46753247 463 32.61339093 464 32.75862069 465 32.90322581 466 32.83261803 467 32.76231263 468 32.69230769 469 32.62260128 470 32.55319149 471 32.69639066 472 32.83898305 473 32.76955603 474 32.70042194 475 32.84210526 476 32.77310924 477 32.91404612 478 32.84518828 479 32.77661795 480 32.70833333 481 32.64033264 482 32.78008299 483 32.71221532 484 32.85123967 485 32.98969072 486 32.92181070 487 33.05954825 488 32.99180328 489 32.92433538 490 33.06122449 491 33.19755601 492 33.33333333 493 33.26572008 494 33.40080972 495 33.53535354 496 33.46774194 497 33.40040241 498 33.33333333 499 33.46693387 500 33.40000000 501 33.33333333 502 33.26693227 503 33.20079523 504 33.33333333 505 33.26732673 506 33.20158103 507 33.33333333 508 33.46456693 509 33.59528487 510 33.52941176 511 33.46379648 512 33.39843750 513 33.33333333 514 33.26848249 515 33.39805825 516 33.33333333 517 33.46228240 518 33.39768340 519 33.33333333 520 33.46153846 521 33.39731286 522 33.33333333 523 33.26959847 524 33.20610687 525 33.33333333 526 33.26996198 527 33.39658444 528 33.33333333 529 33.45935728 530 33.39622642 531 33.33333333 532 33.27067669 533 33.39587242 534 33.33333333 535 33.27102804 536 33.20895522 537 33.33333333 538 33.27137546 539 33.20964750 540 33.14814815 541 33.27171904 542 33.21033210 543 33.14917127 544 33.27205882 545 33.39449541 546 33.33333333 547 33.45521024 548 33.57664234 549 33.51548270 550 33.45454545 551 33.39382940 552 33.51449275 553 33.63471971 554 33.57400722 555 33.69369369 556 33.63309353 557 33.57271095 558 33.51254480 559 33.45259392 560 33.57142857 561 33.68983957 562 33.62989324 563 33.57015986 564 33.68794326 565 33.80530973 566 33.92226148 567 33.86243386 568 33.97887324 569 34.09490334 570 34.03508772 571 34.15061296 572 34.09090909 573 34.03141361 574 34.14634146 575 34.26086957 576 34.20138889 577 34.31542461 578 34.42906574 579 34.54231434 580 34.65517241 581 34.59552496 582 34.70790378 583 34.64837050 584 34.76027397 585 34.70085470 586 34.81228669 587 34.75298126 588 34.86394558 589 34.80475382 590 34.74576271 591 34.68697124 592 34.62837838 593 34.73861720 594 34.84848485 595 34.78991597 596 34.73154362 597 34.84087102 598 34.78260870 599 34.72454090 600 34.66666667 601 34.77537438 602 34.88372093 603 34.99170813 604 34.93377483 605 35.04132231 606 35.14851485 607 35.09060956 608 35.03289474 609 35.13957307 610 35.08196721 611 35.02454992 612 35.13071895 613 35.07340946 614 35.01628664 615 34.95934959 616 35.06493506 617 35.17017828 618 35.11326861 619 35.21809370 620 35.16129032 621 35.10466989 622 35.20900322 623 35.15248796 624 35.09615385 625 35.04000000 626 34.98402556 627 35.08771930 628 35.19108280 629 35.29411765 630 35.23809524 631 35.34072900 632 35.28481013 633 35.38704581 634 35.48895899 635 35.43307087 636 35.37735849 637 35.32182104 638 35.26645768 639 35.21126761 640 35.31250000 641 35.25741030 642 35.35825545 643 35.30326594 644 35.24844720 645 35.34883721 646 35.29411765 647 35.39412674 648 35.33950617 649 35.28505393 650 35.38461538 651 35.48387097 652 35.58282209 653 35.68147014 654 35.62691131 655 35.57251908 656 35.51829268 657 35.61643836 658 35.56231003 659 35.50834598 660 35.45454545 661 35.40090772 662 35.34743202 663 35.44494721 664 35.39156627 665 35.48872180 666 35.58558559 667 35.53223388 668 35.47904192 669 35.42600897 670 35.37313433 671 35.32041729 672 35.26785714 673 35.21545319 674 35.16320475 675 35.25925926 676 35.20710059 677 35.15509601 678 35.10324484 679 35.05154639 680 35.14705882 681 35.24229075 682 35.19061584 683 35.13909224 684 35.23391813 685 35.18248175 686 35.13119534 687 35.08005822 688 35.17441860 689 35.26850508 690 35.21739130 691 35.31114327 692 35.40462428 693 35.49783550 694 35.44668588 695 35.53956835 696 35.63218391 697 35.58106169 698 35.53008596 699 35.47925608 700 35.57142857 701 35.52068474 702 35.47008547 703 35.41963016 704 35.36931818 705 35.31914894 706 35.41076487 707 35.50212164 708 35.45197740 709 35.40197461 710 35.49295775 711 35.58368495 712 35.53370787 713 35.48387097 714 35.57422969 715 35.66433566 716 35.61452514 717 35.70432357 718 35.65459610 719 35.74408901 720 35.69444444 721 35.64493759 722 35.59556787 723 35.54633472 724 35.49723757 725 35.44827586 726 35.53719008 727 35.48830812 728 35.43956044 729 35.52812071 730 35.61643836 731 35.56771546 732 35.65573770 733 35.60709413 734 35.55858311 735 35.51020408 736 35.46195652 737 35.54952510 738 35.50135501 739 35.58863329 740 35.54054054 741 35.62753036 742 35.57951482 743 35.53162853 744 35.61827957 745 35.57046980 746 35.52278820 747 35.60910308 748 35.56149733 749 35.64753004 750 35.73333333 Final result: 35.7333 +/- 1.7510 Random chance: 19.8992 +/- 1.4588