common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 28.57142857 22 27.27272727 23 30.43478261 24 33.33333333 25 32.00000000 26 30.76923077 27 29.62962963 28 28.57142857 29 31.03448276 30 30.00000000 31 29.03225806 32 31.25000000 33 33.33333333 34 32.35294118 35 31.42857143 36 30.55555556 37 32.43243243 38 31.57894737 39 33.33333333 40 32.50000000 41 31.70731707 42 30.95238095 43 30.23255814 44 31.81818182 45 33.33333333 46 32.60869565 47 34.04255319 48 35.41666667 49 34.69387755 50 36.00000000 51 37.25490196 52 36.53846154 53 35.84905660 54 35.18518519 55 36.36363636 56 35.71428571 57 35.08771930 58 34.48275862 59 33.89830508 60 35.00000000 61 34.42622951 62 35.48387097 63 34.92063492 64 35.93750000 65 35.38461538 66 36.36363636 67 35.82089552 68 35.29411765 69 36.23188406 70 35.71428571 71 35.21126761 72 36.11111111 73 35.61643836 74 35.13513514 75 34.66666667 76 35.52631579 77 35.06493506 78 34.61538462 79 34.17721519 80 35.00000000 81 34.56790123 82 35.36585366 83 34.93975904 84 34.52380952 85 34.11764706 86 33.72093023 87 33.33333333 88 32.95454545 89 33.70786517 90 33.33333333 91 32.96703297 92 32.60869565 93 32.25806452 94 31.91489362 95 31.57894737 96 32.29166667 97 32.98969072 98 32.65306122 99 33.33333333 100 34.00000000 101 34.65346535 102 34.31372549 103 34.95145631 104 34.61538462 105 34.28571429 106 34.90566038 107 35.51401869 108 36.11111111 109 36.69724771 110 37.27272727 111 36.93693694 112 37.50000000 113 37.16814159 114 37.71929825 115 37.39130435 116 37.93103448 117 38.46153846 118 38.13559322 119 37.81512605 120 37.50000000 121 38.01652893 122 37.70491803 123 38.21138211 124 37.90322581 125 38.40000000 126 38.09523810 127 37.79527559 128 37.50000000 129 37.98449612 130 38.46153846 131 38.16793893 132 37.87878788 133 37.59398496 134 37.31343284 135 37.03703704 136 36.76470588 137 37.22627737 138 37.68115942 139 38.12949640 140 37.85714286 141 38.29787234 142 38.02816901 143 37.76223776 144 38.19444444 145 38.62068966 146 38.35616438 147 38.09523810 148 37.83783784 149 37.58389262 150 37.33333333 151 37.74834437 152 37.50000000 153 37.25490196 154 37.01298701 155 36.77419355 156 37.17948718 157 37.57961783 158 37.97468354 159 37.73584906 160 37.50000000 161 37.88819876 162 37.65432099 163 38.03680982 164 37.80487805 165 38.18181818 166 38.55421687 167 38.32335329 168 38.09523810 169 37.86982249 170 37.64705882 171 37.42690058 172 37.20930233 173 36.99421965 174 37.35632184 175 37.14285714 176 36.93181818 177 36.72316384 178 36.51685393 179 36.31284916 180 36.11111111 181 35.91160221 182 35.71428571 183 35.51912568 184 35.32608696 185 35.67567568 186 36.02150538 187 35.82887701 188 35.63829787 189 35.44973545 190 35.26315789 191 35.07853403 192 34.89583333 193 34.71502591 194 35.05154639 195 34.87179487 196 34.69387755 197 35.02538071 198 35.35353535 199 35.17587940 200 35.50000000 201 35.32338308 202 35.64356436 203 35.46798030 204 35.29411765 205 35.60975610 206 35.43689320 207 35.74879227 208 35.57692308 209 35.40669856 210 35.23809524 211 35.07109005 212 34.90566038 213 34.74178404 214 34.57943925 215 34.41860465 216 34.72222222 217 34.56221198 218 34.40366972 219 34.24657534 220 34.09090909 221 34.38914027 222 34.23423423 223 34.08071749 224 34.37500000 225 34.66666667 226 34.51327434 227 34.80176211 228 34.64912281 229 34.49781659 230 34.34782609 231 34.19913420 232 34.48275862 233 34.33476395 234 34.18803419 235 34.04255319 236 34.32203390 237 34.17721519 238 34.45378151 239 34.30962343 240 34.16666667 241 34.02489627 242 33.88429752 243 33.74485597 244 33.60655738 245 33.46938776 246 33.33333333 247 33.60323887 248 33.87096774 249 34.13654618 250 34.00000000 251 34.26294821 252 34.12698413 253 33.99209486 254 34.25196850 255 34.50980392 256 34.37500000 257 34.24124514 258 34.10852713 259 33.97683398 260 33.84615385 261 34.09961686 262 33.96946565 263 33.84030418 264 34.09090909 265 34.33962264 266 34.21052632 267 34.45692884 268 34.32835821 269 34.20074349 270 34.07407407 271 33.94833948 272 33.82352941 273 33.69963370 274 33.94160584 275 33.81818182 276 33.69565217 277 33.57400722 278 33.45323741 279 33.69175627 280 33.92857143 281 34.16370107 282 34.04255319 283 33.92226148 284 34.15492958 285 34.03508772 286 34.26573427 287 34.49477352 288 34.72222222 289 34.60207612 290 34.48275862 291 34.70790378 292 34.58904110 293 34.81228669 294 34.69387755 295 34.91525424 296 35.13513514 297 35.01683502 298 34.89932886 299 34.78260870 300 34.66666667 301 34.55149502 302 34.43708609 303 34.65346535 304 34.53947368 305 34.42622951 306 34.31372549 307 34.20195440 308 34.41558442 309 34.30420712 310 34.19354839 311 34.08360129 312 34.29487179 313 34.18530351 314 34.07643312 315 34.28571429 316 34.17721519 317 34.06940063 318 33.96226415 319 33.85579937 320 33.75000000 321 33.64485981 322 33.54037267 323 33.74613003 324 33.64197531 325 33.53846154 326 33.43558282 327 33.63914373 328 33.84146341 329 33.73860182 330 33.93939394 331 33.83685801 332 34.03614458 333 33.93393393 334 33.83233533 335 34.02985075 336 34.22619048 337 34.12462908 338 34.02366864 339 34.21828909 340 34.11764706 341 34.31085044 342 34.50292398 343 34.40233236 344 34.59302326 345 34.78260870 346 34.68208092 347 34.87031700 348 34.77011494 349 34.67048711 350 34.57142857 351 34.47293447 352 34.37500000 353 34.27762040 354 34.18079096 355 34.36619718 356 34.26966292 357 34.45378151 358 34.63687151 359 34.54038997 360 34.44444444 361 34.62603878 362 34.53038674 363 34.43526171 364 34.34065934 365 34.24657534 366 34.15300546 367 34.33242507 368 34.51086957 369 34.41734417 370 34.32432432 371 34.23180593 372 34.40860215 373 34.31635389 374 34.49197861 375 34.40000000 376 34.30851064 377 34.48275862 378 34.39153439 379 34.30079156 380 34.47368421 381 34.38320210 382 34.55497382 383 34.46475196 384 34.37500000 385 34.28571429 386 34.19689119 387 34.10852713 388 34.27835052 389 34.19023136 390 34.10256410 391 34.01534527 392 33.92857143 393 33.84223919 394 34.01015228 395 33.92405063 396 33.83838384 397 34.00503778 398 33.91959799 399 33.83458647 400 33.75000000 401 33.91521197 402 34.07960199 403 34.24317618 404 34.15841584 405 34.07407407 406 33.99014778 407 33.90663391 408 33.82352941 409 33.74083130 410 33.65853659 411 33.57664234 412 33.73786408 413 33.89830508 414 34.05797101 415 34.21686747 416 34.13461538 417 34.05275779 418 33.97129187 419 33.89021480 420 34.04761905 421 34.20427553 422 34.12322275 423 34.04255319 424 33.96226415 425 33.88235294 426 33.80281690 427 33.72365340 428 33.64485981 429 33.56643357 430 33.72093023 431 33.64269142 432 33.79629630 433 33.94919169 434 34.10138249 435 34.02298851 436 33.94495413 437 33.86727689 438 33.78995434 439 33.94077449 440 33.86363636 441 33.78684807 442 33.71040724 443 33.63431151 444 33.78378378 445 33.70786517 446 33.63228700 447 33.55704698 448 33.48214286 449 33.40757238 450 33.33333333 451 33.25942350 452 33.18584071 453 33.33333333 454 33.25991189 455 33.18681319 456 33.11403509 457 33.04157549 458 32.96943231 459 32.89760349 460 32.82608696 461 32.75488069 462 32.68398268 463 32.82937365 464 32.97413793 465 33.11827957 466 33.04721030 467 32.97644540 468 32.90598291 469 32.83582090 470 32.76595745 471 32.90870488 472 32.83898305 473 32.76955603 474 32.91139241 475 33.05263158 476 32.98319328 477 33.12368973 478 33.05439331 479 32.98538622 480 32.91666667 481 32.84823285 482 32.98755187 483 32.91925466 484 33.05785124 485 33.19587629 486 33.12757202 487 33.05954825 488 32.99180328 489 33.12883436 490 33.26530612 491 33.40122200 492 33.53658537 493 33.46855984 494 33.60323887 495 33.53535354 496 33.46774194 497 33.40040241 498 33.33333333 499 33.46693387 500 33.40000000 501 33.33333333 502 33.26693227 503 33.20079523 504 33.33333333 505 33.26732673 506 33.20158103 507 33.33333333 508 33.46456693 509 33.59528487 510 33.52941176 511 33.46379648 512 33.39843750 513 33.33333333 514 33.26848249 515 33.39805825 516 33.33333333 517 33.46228240 518 33.39768340 519 33.33333333 520 33.26923077 521 33.20537428 522 33.33333333 523 33.26959847 524 33.20610687 525 33.33333333 526 33.26996198 527 33.39658444 528 33.33333333 529 33.27032136 530 33.20754717 531 33.14500942 532 33.08270677 533 33.20825516 534 33.14606742 535 33.08411215 536 33.02238806 537 33.14711359 538 33.08550186 539 33.02411874 540 32.96296296 541 33.08687616 542 33.02583026 543 32.96500921 544 33.08823529 545 33.21100917 546 33.15018315 547 33.27239488 548 33.39416058 549 33.33333333 550 33.27272727 551 33.21234120 552 33.33333333 553 33.45388788 554 33.39350181 555 33.51351351 556 33.45323741 557 33.39317774 558 33.33333333 559 33.27370304 560 33.39285714 561 33.51158645 562 33.45195730 563 33.39253996 564 33.51063830 565 33.62831858 566 33.74558304 567 33.68606702 568 33.80281690 569 33.91915641 570 33.85964912 571 33.97548161 572 33.91608392 573 33.85689354 574 33.97212544 575 34.08695652 576 34.02777778 577 34.14211438 578 34.25605536 579 34.36960276 580 34.48275862 581 34.42340792 582 34.53608247 583 34.47684391 584 34.58904110 585 34.52991453 586 34.64163823 587 34.58262351 588 34.52380952 589 34.46519525 590 34.40677966 591 34.34856176 592 34.29054054 593 34.40134907 594 34.51178451 595 34.45378151 596 34.39597315 597 34.50586265 598 34.44816054 599 34.39065109 600 34.33333333 601 34.44259567 602 34.55149502 603 34.66003317 604 34.60264901 605 34.54545455 606 34.65346535 607 34.59637562 608 34.53947368 609 34.64696223 610 34.59016393 611 34.53355155 612 34.64052288 613 34.58401305 614 34.52768730 615 34.47154472 616 34.57792208 617 34.68395462 618 34.62783172 619 34.73344103 620 34.67741935 621 34.62157810 622 34.72668810 623 34.67094703 624 34.61538462 625 34.56000000 626 34.50479233 627 34.60925040 628 34.71337580 629 34.81717011 630 34.76190476 631 34.86529319 632 34.81012658 633 34.91311216 634 35.01577287 635 34.96062992 636 34.90566038 637 34.85086342 638 34.79623824 639 34.74178404 640 34.84375000 641 34.78939158 642 34.89096573 643 34.83670295 644 34.78260870 645 34.88372093 646 34.82972136 647 34.93044822 648 34.87654321 649 34.82280431 650 34.92307692 651 35.02304147 652 35.12269939 653 35.22205207 654 35.16819572 655 35.11450382 656 35.21341463 657 35.31202435 658 35.25835866 659 35.20485584 660 35.15151515 661 35.09833585 662 35.04531722 663 35.14328808 664 35.09036145 665 35.03759398 666 34.98498498 667 34.93253373 668 34.88023952 669 34.82810164 670 34.92537313 671 34.87332340 672 34.82142857 673 34.76968796 674 34.71810089 675 34.81481481 676 34.91124260 677 34.85967504 678 34.80825959 679 34.75699558 680 34.85294118 681 34.94860499 682 34.89736070 683 34.84626647 684 34.94152047 685 34.89051095 686 34.83965015 687 34.78893741 688 34.88372093 689 34.97822932 690 34.92753623 691 35.02170767 692 35.11560694 693 35.06493506 694 35.01440922 695 35.10791367 696 35.20114943 697 35.15064562 698 35.10028653 699 35.05007153 700 35.14285714 701 35.09272468 702 35.04273504 703 34.99288762 704 34.94318182 705 34.89361702 706 34.84419263 707 34.93635078 708 34.88700565 709 34.83779972 710 34.78873239 711 34.88045007 712 34.83146067 713 34.78260870 714 34.87394958 715 34.82517483 716 34.77653631 717 34.86750349 718 34.81894150 719 34.90959666 720 34.86111111 721 34.81276006 722 34.76454294 723 34.71645920 724 34.66850829 725 34.62068966 726 34.71074380 727 34.66299862 728 34.61538462 729 34.70507545 730 34.79452055 731 34.74692202 732 34.83606557 733 34.78854025 734 34.74114441 735 34.69387755 736 34.64673913 737 34.73541384 738 34.68834688 739 34.77672530 740 34.72972973 741 34.81781377 742 34.77088949 743 34.72409152 744 34.81182796 745 34.76510067 746 34.71849866 747 34.80589023 748 34.75935829 749 34.84646195 750 34.93333333 Final result: 34.9300 +/- 1.6774 Random chance: 19.5384 +/- 1.7940