common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 27.90697674 44 29.54545455 45 31.11111111 46 30.43478261 47 31.91489362 48 33.33333333 49 32.65306122 50 34.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 32.81250000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 31.42857143 71 30.98591549 72 31.94444444 73 31.50684932 74 31.08108108 75 30.66666667 76 31.57894737 77 31.16883117 78 30.76923077 79 30.37974684 80 30.00000000 81 29.62962963 82 30.48780488 83 30.12048193 84 29.76190476 85 29.41176471 86 29.06976744 87 28.73563218 88 28.40909091 89 29.21348315 90 28.88888889 91 28.57142857 92 28.26086957 93 27.95698925 94 27.65957447 95 27.36842105 96 28.12500000 97 28.86597938 98 28.57142857 99 29.29292929 100 30.00000000 101 30.69306931 102 30.39215686 103 31.06796117 104 30.76923077 105 30.47619048 106 30.18867925 107 30.84112150 108 31.48148148 109 32.11009174 110 32.72727273 111 32.43243243 112 33.03571429 113 32.74336283 114 33.33333333 115 33.04347826 116 33.62068966 117 34.18803419 118 33.89830508 119 33.61344538 120 33.33333333 121 33.88429752 122 33.60655738 123 34.14634146 124 33.87096774 125 34.40000000 126 34.12698413 127 33.85826772 128 33.59375000 129 34.10852713 130 34.61538462 131 34.35114504 132 34.09090909 133 33.83458647 134 33.58208955 135 33.33333333 136 33.08823529 137 33.57664234 138 34.05797101 139 34.53237410 140 34.28571429 141 34.75177305 142 34.50704225 143 34.26573427 144 34.72222222 145 35.17241379 146 34.93150685 147 34.69387755 148 34.45945946 149 34.22818792 150 34.00000000 151 34.43708609 152 34.21052632 153 33.98692810 154 33.76623377 155 33.54838710 156 33.97435897 157 34.39490446 158 34.81012658 159 34.59119497 160 34.37500000 161 34.16149068 162 33.95061728 163 34.35582822 164 34.14634146 165 34.54545455 166 34.93975904 167 34.73053892 168 34.52380952 169 34.31952663 170 34.11764706 171 33.91812865 172 33.72093023 173 33.52601156 174 33.90804598 175 33.71428571 176 33.52272727 177 33.33333333 178 33.14606742 179 32.96089385 180 32.77777778 181 32.59668508 182 32.41758242 183 32.24043716 184 32.06521739 185 32.43243243 186 32.79569892 187 32.62032086 188 32.44680851 189 32.27513228 190 32.10526316 191 31.93717277 192 31.77083333 193 31.60621762 194 31.95876289 195 31.79487179 196 32.14285714 197 32.48730964 198 32.82828283 199 32.66331658 200 33.00000000 201 32.83582090 202 33.16831683 203 33.00492611 204 32.84313725 205 33.17073171 206 33.00970874 207 33.33333333 208 33.17307692 209 33.01435407 210 32.85714286 211 32.70142180 212 32.54716981 213 32.39436620 214 32.24299065 215 32.09302326 216 32.40740741 217 32.25806452 218 32.11009174 219 31.96347032 220 31.81818182 221 32.12669683 222 31.98198198 223 32.28699552 224 32.58928571 225 32.44444444 226 32.30088496 227 32.59911894 228 32.45614035 229 32.31441048 230 32.17391304 231 32.03463203 232 32.32758621 233 32.18884120 234 32.47863248 235 32.34042553 236 32.62711864 237 32.48945148 238 32.77310924 239 32.63598326 240 32.50000000 241 32.36514523 242 32.23140496 243 32.09876543 244 31.96721311 245 31.83673469 246 31.70731707 247 31.98380567 248 32.25806452 249 32.53012048 250 32.40000000 251 32.66932271 252 32.53968254 253 32.41106719 254 32.67716535 255 32.94117647 256 32.81250000 257 32.68482490 258 32.55813953 259 32.43243243 260 32.30769231 261 32.56704981 262 32.44274809 263 32.31939163 264 32.57575758 265 32.83018868 266 32.70676692 267 32.58426966 268 32.46268657 269 32.34200743 270 32.22222222 271 32.10332103 272 31.98529412 273 31.86813187 274 32.11678832 275 32.00000000 276 31.88405797 277 31.76895307 278 31.65467626 279 31.54121864 280 31.78571429 281 32.02846975 282 31.91489362 283 31.80212014 284 32.04225352 285 31.92982456 286 32.16783217 287 32.40418118 288 32.63888889 289 32.52595156 290 32.41379310 291 32.64604811 292 32.53424658 293 32.76450512 294 32.65306122 295 32.88135593 296 33.10810811 297 32.99663300 298 32.88590604 299 32.77591973 300 32.66666667 301 32.55813953 302 32.45033113 303 32.34323432 304 32.23684211 305 32.13114754 306 32.02614379 307 31.92182410 308 32.14285714 309 32.03883495 310 31.93548387 311 31.83279743 312 32.05128205 313 31.94888179 314 31.84713376 315 31.74603175 316 31.64556962 317 31.54574132 318 31.44654088 319 31.34796238 320 31.25000000 321 31.15264798 322 31.36645963 323 31.26934985 324 31.17283951 325 31.07692308 326 30.98159509 327 31.19266055 328 31.40243902 329 31.30699088 330 31.51515152 331 31.41993958 332 31.62650602 333 31.53153153 334 31.43712575 335 31.64179104 336 31.84523810 337 31.75074184 338 31.65680473 339 31.85840708 340 31.76470588 341 31.96480938 342 32.16374269 343 32.06997085 344 32.26744186 345 32.46376812 346 32.36994220 347 32.56484150 348 32.47126437 349 32.37822350 350 32.28571429 351 32.19373219 352 32.10227273 353 32.01133144 354 31.92090395 355 32.11267606 356 32.02247191 357 32.21288515 358 32.40223464 359 32.31197772 360 32.22222222 361 32.40997230 362 32.32044199 363 32.23140496 364 32.14285714 365 32.05479452 366 31.96721311 367 32.15258856 368 32.06521739 369 31.97831978 370 31.89189189 371 31.80592992 372 31.98924731 373 31.90348525 374 32.08556150 375 32.00000000 376 31.91489362 377 31.83023873 378 31.74603175 379 31.66226913 380 31.84210526 381 31.75853018 382 31.93717277 383 31.85378590 384 31.77083333 385 31.68831169 386 31.60621762 387 31.52454780 388 31.70103093 389 31.61953728 390 31.53846154 391 31.45780051 392 31.37755102 393 31.29770992 394 31.21827411 395 31.13924051 396 31.06060606 397 31.23425693 398 31.15577889 399 31.07769424 400 31.00000000 401 31.17206983 402 31.34328358 403 31.51364764 404 31.43564356 405 31.35802469 406 31.28078818 407 31.20393120 408 31.12745098 409 31.05134474 410 30.97560976 411 30.90024331 412 31.06796117 413 31.23486683 414 31.40096618 415 31.56626506 416 31.49038462 417 31.41486811 418 31.33971292 419 31.26491647 420 31.42857143 421 31.59144893 422 31.51658768 423 31.44208038 424 31.36792453 425 31.29411765 426 31.22065728 427 31.14754098 428 31.07476636 429 31.23543124 430 31.39534884 431 31.32250580 432 31.48148148 433 31.63972286 434 31.79723502 435 31.72413793 436 31.65137615 437 31.57894737 438 31.73515982 439 31.89066059 440 31.81818182 441 31.74603175 442 31.67420814 443 31.60270880 444 31.75675676 445 31.68539326 446 31.83856502 447 31.76733781 448 31.69642857 449 31.84855234 450 31.77777778 451 31.70731707 452 31.63716814 453 31.78807947 454 31.71806167 455 31.64835165 456 31.57894737 457 31.50984683 458 31.44104803 459 31.37254902 460 31.30434783 461 31.23644252 462 31.16883117 463 31.31749460 464 31.46551724 465 31.61290323 466 31.54506438 467 31.47751606 468 31.41025641 469 31.34328358 470 31.27659574 471 31.42250531 472 31.56779661 473 31.50105708 474 31.64556962 475 31.78947368 476 31.72268908 477 31.86582809 478 31.79916318 479 31.73277662 480 31.66666667 481 31.60083160 482 31.74273859 483 31.67701863 484 31.81818182 485 31.95876289 486 31.89300412 487 31.82751540 488 31.76229508 489 31.90184049 490 32.04081633 491 32.17922607 492 32.31707317 493 32.25152130 494 32.38866397 495 32.32323232 496 32.25806452 497 32.19315895 498 32.32931727 499 32.46492986 500 32.40000000 501 32.33532934 502 32.27091633 503 32.20675944 504 32.34126984 505 32.27722772 506 32.21343874 507 32.34714004 508 32.48031496 509 32.61296660 510 32.54901961 511 32.48532290 512 32.42187500 513 32.35867446 514 32.29571984 515 32.42718447 516 32.36434109 517 32.49516441 518 32.43243243 519 32.36994220 520 32.50000000 521 32.43761996 522 32.37547893 523 32.31357553 524 32.25190840 525 32.38095238 526 32.31939163 527 32.44781784 528 32.38636364 529 32.51417769 530 32.45283019 531 32.39171375 532 32.33082707 533 32.45778612 534 32.39700375 535 32.33644860 536 32.27611940 537 32.40223464 538 32.34200743 539 32.28200371 540 32.22222222 541 32.34750462 542 32.28782288 543 32.22836096 544 32.16911765 545 32.29357798 546 32.41758242 547 32.54113346 548 32.66423358 549 32.60473588 550 32.54545455 551 32.48638838 552 32.60869565 553 32.73056058 554 32.67148014 555 32.79279279 556 32.73381295 557 32.67504488 558 32.61648746 559 32.55813953 560 32.67857143 561 32.79857398 562 32.74021352 563 32.68206039 564 32.80141844 565 32.92035398 566 33.03886926 567 32.98059965 568 33.09859155 569 33.21616872 570 33.15789474 571 33.27495622 572 33.21678322 573 33.15881326 574 33.27526132 575 33.39130435 576 33.50694444 577 33.62218371 578 33.73702422 579 33.85146805 580 33.96551724 581 33.90705680 582 34.02061856 583 33.96226415 584 34.07534247 585 34.01709402 586 34.12969283 587 34.07155026 588 34.01360544 589 33.95585739 590 33.89830508 591 33.84094755 592 33.78378378 593 33.89544688 594 34.00673401 595 33.94957983 596 33.89261745 597 34.00335008 598 33.94648829 599 33.88981636 600 33.83333333 601 33.94342762 602 34.05315615 603 34.16252073 604 34.10596026 605 34.21487603 606 34.32343234 607 34.26688633 608 34.21052632 609 34.31855501 610 34.26229508 611 34.20621931 612 34.31372549 613 34.25774878 614 34.20195440 615 34.14634146 616 34.25324675 617 34.35980551 618 34.30420712 619 34.41033926 620 34.35483871 621 34.46054750 622 34.56591640 623 34.51043339 624 34.45512821 625 34.40000000 626 34.34504792 627 34.44976077 628 34.55414013 629 34.65818760 630 34.60317460 631 34.70681458 632 34.65189873 633 34.75513428 634 34.85804416 635 34.80314961 636 34.74842767 637 34.69387755 638 34.63949843 639 34.58528951 640 34.68750000 641 34.63338534 642 34.73520249 643 34.68118196 644 34.62732919 645 34.72868217 646 34.67492260 647 34.77588872 648 34.72222222 649 34.66872111 650 34.76923077 651 34.86943164 652 34.96932515 653 35.06891271 654 35.01529052 655 34.96183206 656 34.90853659 657 35.00761035 658 34.95440729 659 34.90136571 660 34.84848485 661 34.79576399 662 34.74320242 663 34.84162896 664 34.78915663 665 34.88721805 666 34.98498498 667 34.93253373 668 34.88023952 669 34.82810164 670 34.77611940 671 34.72429210 672 34.67261905 673 34.62109955 674 34.56973294 675 34.66666667 676 34.76331361 677 34.85967504 678 34.80825959 679 34.75699558 680 34.85294118 681 34.94860499 682 34.89736070 683 34.84626647 684 34.94152047 685 34.89051095 686 34.83965015 687 34.78893741 688 34.88372093 689 34.97822932 690 34.92753623 691 35.02170767 692 35.11560694 693 35.20923521 694 35.15850144 695 35.25179856 696 35.34482759 697 35.29411765 698 35.24355301 699 35.19313305 700 35.28571429 701 35.23537803 702 35.18518519 703 35.13513514 704 35.08522727 705 35.03546099 706 35.12747875 707 35.21923621 708 35.16949153 709 35.11988717 710 35.21126761 711 35.30239100 712 35.25280899 713 35.20336606 714 35.29411765 715 35.38461538 716 35.33519553 717 35.42538354 718 35.37604457 719 35.46592490 720 35.41666667 721 35.36754508 722 35.31855956 723 35.26970954 724 35.22099448 725 35.17241379 726 35.26170799 727 35.21320495 728 35.16483516 729 35.25377229 730 35.34246575 731 35.29411765 732 35.38251366 733 35.33424284 734 35.28610354 735 35.23809524 736 35.19021739 737 35.27815468 738 35.23035230 739 35.31799729 740 35.27027027 741 35.35762483 742 35.30997305 743 35.26244953 744 35.34946237 745 35.30201342 746 35.25469169 747 35.34136546 748 35.29411765 749 35.38050734 750 35.46666667 Final result: 35.4667 +/- 1.7481 Random chance: 19.8992 +/- 1.4588