common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 77.77777778 19 78.94736842 20 75.00000000 21 76.19047619 22 77.27272727 23 78.26086957 24 79.16666667 25 76.00000000 26 73.07692308 27 74.07407407 28 71.42857143 29 72.41379310 30 73.33333333 31 74.19354839 32 75.00000000 33 75.75757576 34 76.47058824 35 77.14285714 36 75.00000000 37 75.67567568 38 76.31578947 39 76.92307692 40 77.50000000 41 75.60975610 42 76.19047619 43 76.74418605 44 75.00000000 45 75.55555556 46 76.08695652 47 76.59574468 48 77.08333333 49 75.51020408 50 74.00000000 51 72.54901961 52 73.07692308 53 73.58490566 54 72.22222222 55 70.90909091 56 71.42857143 57 71.92982456 58 72.41379310 59 72.88135593 60 73.33333333 61 73.77049180 62 74.19354839 63 74.60317460 64 75.00000000 65 73.84615385 66 74.24242424 67 74.62686567 68 75.00000000 69 75.36231884 70 75.71428571 71 76.05633803 72 75.00000000 73 75.34246575 74 75.67567568 75 76.00000000 76 76.31578947 77 76.62337662 78 76.92307692 79 77.21518987 80 77.50000000 81 77.77777778 82 78.04878049 83 78.31325301 84 78.57142857 85 78.82352941 86 79.06976744 87 79.31034483 88 79.54545455 89 79.77528090 90 80.00000000 91 80.21978022 92 80.43478261 93 79.56989247 94 78.72340426 95 78.94736842 96 79.16666667 97 78.35051546 98 78.57142857 99 78.78787879 100 79.00000000 101 78.21782178 102 78.43137255 103 77.66990291 104 76.92307692 105 77.14285714 106 76.41509434 107 76.63551402 108 75.92592593 109 75.22935780 110 75.45454545 111 75.67567568 112 75.89285714 113 75.22123894 114 75.43859649 115 75.65217391 116 75.86206897 117 76.06837607 118 76.27118644 119 75.63025210 120 75.83333333 121 75.20661157 122 75.40983607 123 75.60975610 124 75.80645161 125 76.00000000 126 76.19047619 127 76.37795276 128 76.56250000 129 75.96899225 130 76.15384615 131 76.33587786 132 76.51515152 133 76.69172932 134 76.11940299 135 75.55555556 136 75.73529412 137 75.91240876 138 75.36231884 139 75.53956835 140 75.00000000 141 74.46808511 142 74.64788732 143 74.82517483 144 75.00000000 145 75.17241379 146 75.34246575 147 75.51020408 148 75.00000000 149 75.16778523 150 75.33333333 151 75.49668874 152 75.65789474 153 75.81699346 154 75.97402597 155 75.48387097 156 75.64102564 157 75.79617834 158 75.94936709 159 76.10062893 160 76.25000000 161 76.39751553 162 76.54320988 163 76.07361963 164 76.21951220 165 76.36363636 166 76.50602410 167 76.64670659 168 76.78571429 169 76.33136095 170 76.47058824 171 76.60818713 172 76.74418605 173 76.87861272 174 77.01149425 175 76.57142857 176 76.70454545 177 76.83615819 178 76.96629213 179 77.09497207 180 77.22222222 181 77.34806630 182 77.47252747 183 77.04918033 184 77.17391304 185 77.29729730 186 77.41935484 187 77.54010695 188 77.12765957 189 77.24867725 190 76.84210526 191 76.96335079 192 76.56250000 193 76.16580311 194 76.28865979 195 76.41025641 196 76.53061224 197 76.64974619 198 76.76767677 199 76.38190955 200 76.50000000 201 76.61691542 202 76.73267327 203 76.84729064 204 76.96078431 205 77.07317073 206 76.69902913 207 76.81159420 208 76.92307692 209 77.03349282 210 77.14285714 211 77.25118483 212 76.88679245 213 76.99530516 214 77.10280374 215 77.20930233 216 76.85185185 217 76.95852535 218 77.06422018 219 77.16894977 220 76.81818182 221 76.92307692 222 77.02702703 223 76.68161435 224 76.78571429 225 76.44444444 226 76.10619469 227 76.21145374 228 76.31578947 229 76.41921397 230 76.52173913 231 76.19047619 232 75.86206897 233 75.53648069 234 75.21367521 235 74.89361702 236 75.00000000 237 74.68354430 238 74.36974790 239 74.47698745 240 74.58333333 241 74.68879668 242 74.79338843 243 74.89711934 244 74.59016393 245 74.69387755 246 74.79674797 247 74.89878543 248 75.00000000 249 75.10040161 250 75.20000000 251 75.29880478 252 75.39682540 253 75.49407115 254 75.59055118 255 75.68627451 256 75.78125000 257 75.48638132 258 75.58139535 259 75.67567568 260 75.76923077 261 75.86206897 262 75.95419847 263 76.04562738 264 76.13636364 265 76.22641509 266 75.93984962 267 76.02996255 268 76.11940299 269 76.20817844 270 76.29629630 271 76.38376384 272 76.47058824 273 76.55677656 274 76.27737226 275 76.00000000 276 76.08695652 277 76.17328520 278 76.25899281 279 76.34408602 280 76.42857143 281 76.51245552 282 76.59574468 283 76.32508834 284 76.40845070 285 76.49122807 286 76.57342657 287 76.65505226 288 76.73611111 289 76.47058824 290 76.55172414 291 76.63230241 292 76.71232877 293 76.79180887 294 76.53061224 295 76.61016949 296 76.68918919 297 76.76767677 298 76.84563758 299 76.92307692 300 77.00000000 301 76.74418605 302 76.82119205 303 76.89768977 304 76.97368421 305 77.04918033 306 76.79738562 307 76.87296417 308 76.94805195 309 77.02265372 310 76.77419355 311 76.84887460 312 76.92307692 313 76.99680511 314 77.07006369 315 77.14285714 316 77.21518987 317 77.28706625 318 77.35849057 319 77.42946708 320 77.18750000 321 77.25856698 322 77.32919255 323 77.39938080 324 77.46913580 325 77.53846154 326 77.60736196 327 77.37003058 328 77.43902439 329 77.50759878 330 77.27272727 331 77.34138973 332 77.40963855 333 77.47747748 334 77.54491018 335 77.31343284 336 77.08333333 337 76.85459941 338 76.62721893 339 76.69616519 340 76.76470588 341 76.83284457 342 76.90058480 343 76.96793003 344 77.03488372 345 76.81159420 346 76.87861272 347 76.94524496 348 76.72413793 349 76.79083095 350 76.85714286 351 76.92307692 352 76.98863636 353 77.05382436 354 76.83615819 355 76.61971831 356 76.68539326 357 76.75070028 358 76.81564246 359 76.60167131 360 76.66666667 361 76.73130194 362 76.79558011 363 76.85950413 364 76.92307692 365 76.98630137 366 76.77595628 367 76.83923706 368 76.90217391 369 76.96476965 370 77.02702703 371 76.81940701 372 76.88172043 373 76.94369973 374 77.00534759 375 76.80000000 376 76.86170213 377 76.65782493 378 76.71957672 379 76.51715040 380 76.57894737 381 76.37795276 382 76.43979058 383 76.24020888 384 76.30208333 385 76.36363636 386 76.16580311 387 76.22739018 388 76.28865979 389 76.09254499 390 75.89743590 391 75.70332481 392 75.76530612 393 75.82697201 394 75.63451777 395 75.69620253 396 75.50505051 397 75.56675063 398 75.37688442 399 75.43859649 400 75.50000000 401 75.56109726 402 75.62189055 403 75.68238213 404 75.74257426 405 75.80246914 406 75.86206897 407 75.92137592 408 75.98039216 409 75.79462103 410 75.85365854 411 75.91240876 412 75.97087379 413 76.02905569 414 76.08695652 415 76.14457831 416 76.20192308 417 76.01918465 418 76.07655502 419 75.89498807 420 75.95238095 421 76.00950119 422 76.06635071 423 76.12293144 424 76.17924528 425 76.00000000 426 75.82159624 427 75.64402810 428 75.46728972 429 75.52447552 430 75.34883721 431 75.40603248 432 75.46296296 433 75.28868360 434 75.34562212 435 75.40229885 436 75.45871560 437 75.51487414 438 75.34246575 439 75.39863326 440 75.45454545 441 75.51020408 442 75.33936652 443 75.39503386 444 75.45045045 445 75.50561798 446 75.56053812 447 75.61521253 448 75.66964286 449 75.72383073 450 75.55555556 451 75.60975610 452 75.66371681 453 75.71743929 454 75.55066079 455 75.60439560 456 75.65789474 457 75.71115974 458 75.76419214 459 75.81699346 460 75.65217391 461 75.70498915 462 75.75757576 463 75.59395248 464 75.64655172 465 75.69892473 466 75.75107296 467 75.80299786 468 75.85470085 469 75.69296375 470 75.74468085 471 75.79617834 472 75.84745763 473 75.68710359 474 75.73839662 475 75.57894737 476 75.63025210 477 75.68134172 478 75.73221757 479 75.78288100 480 75.83333333 481 75.88357588 482 75.93360996 483 75.98343685 484 76.03305785 485 76.08247423 486 76.13168724 487 75.97535934 488 76.02459016 489 76.07361963 490 76.12244898 491 76.17107943 492 76.21951220 493 76.06490872 494 76.11336032 495 76.16161616 496 76.00806452 497 75.85513078 498 75.90361446 499 75.95190381 500 76.00000000 501 76.04790419 502 76.09561753 503 76.14314115 504 76.19047619 505 76.23762376 506 76.08695652 507 76.13412229 508 76.18110236 509 76.22789784 510 76.27450980 511 76.12524462 512 76.17187500 513 76.21832359 514 76.26459144 515 76.31067961 516 76.16279070 517 76.20889749 518 76.25482625 519 76.30057803 520 76.34615385 521 76.39155470 522 76.43678161 523 76.29063098 524 76.14503817 525 76.00000000 526 75.85551331 527 75.71157495 528 75.56818182 529 75.42533081 530 75.47169811 531 75.51789077 532 75.56390977 533 75.60975610 534 75.46816479 535 75.32710280 536 75.37313433 537 75.41899441 538 75.27881041 539 75.32467532 540 75.18518519 541 75.23105360 542 75.27675277 543 75.13812155 544 75.18382353 545 75.22935780 546 75.27472527 547 75.13711152 548 75.18248175 549 75.22768670 550 75.27272727 551 75.13611615 552 75.00000000 553 75.04520796 554 75.09025271 555 75.13513514 556 75.00000000 557 75.04488330 558 74.91039427 559 74.95527728 560 75.00000000 561 75.04456328 562 75.08896797 563 75.13321492 564 75.17730496 565 75.04424779 566 75.08833922 567 75.13227513 568 75.17605634 569 75.04393673 570 74.91228070 571 74.95621716 572 74.82517483 573 74.86910995 574 74.91289199 575 74.95652174 576 75.00000000 577 75.04332756 578 74.91349481 579 74.95682211 580 75.00000000 581 74.87091222 582 74.91408935 583 74.78559177 584 74.82876712 585 74.87179487 586 74.91467577 587 74.95741056 588 75.00000000 589 75.04244482 590 75.08474576 591 74.95769882 592 74.83108108 593 74.87352445 594 74.91582492 595 74.95798319 596 74.83221477 597 74.87437186 598 74.91638796 599 74.79131886 600 74.83333333 601 74.87520799 602 74.75083056 603 74.62686567 604 74.66887417 605 74.71074380 606 74.75247525 607 74.79406919 608 74.83552632 609 74.71264368 610 74.75409836 611 74.63175123 612 74.67320261 613 74.55138662 614 74.59283388 615 74.47154472 616 74.35064935 617 74.39222042 618 74.43365696 619 74.31340872 620 74.35483871 621 74.39613527 622 74.43729904 623 74.47833066 624 74.35897436 625 74.40000000 626 74.44089457 627 74.48165869 628 74.36305732 629 74.40381558 630 74.44444444 631 74.48494453 632 74.36708861 633 74.24960506 634 74.29022082 635 74.33070866 636 74.37106918 637 74.25431711 638 74.29467085 639 74.33489828 640 74.37500000 641 74.41497660 642 74.29906542 643 74.18351477 644 74.22360248 645 74.10852713 646 74.14860681 647 74.03400309 648 74.07407407 649 73.95993837 650 74.00000000 651 74.03993856 652 73.92638037 653 73.96630934 654 73.85321101 655 73.74045802 656 73.62804878 657 73.66818874 658 73.70820669 659 73.59635812 660 73.63636364 661 73.52496218 662 73.56495468 663 73.60482655 664 73.64457831 665 73.68421053 666 73.72372372 667 73.76311844 668 73.65269461 669 73.69207773 670 73.58208955 671 73.62146051 672 73.66071429 673 73.69985141 674 73.73887240 675 73.77777778 676 73.81656805 677 73.85524372 678 73.89380531 679 73.78497791 680 73.82352941 681 73.86196769 682 73.90029326 683 73.79209370 684 73.83040936 685 73.86861314 686 73.90670554 687 73.94468705 688 73.98255814 689 73.87518142 690 73.91304348 691 73.95079595 692 73.84393064 693 73.88167388 694 73.91930836 695 73.81294964 696 73.85057471 697 73.74461980 698 73.63896848 699 73.67668097 700 73.57142857 701 73.46647646 702 73.50427350 703 73.54196302 704 73.57954545 705 73.61702128 706 73.65439093 707 73.69165488 708 73.72881356 709 73.76586742 710 73.80281690 711 73.83966245 712 73.87640449 713 73.91304348 714 73.80952381 715 73.70629371 716 73.74301676 717 73.77963738 718 73.67688022 719 73.57440890 720 73.61111111 721 73.50901526 722 73.54570637 723 73.58229599 724 73.48066298 725 73.51724138 726 73.41597796 727 73.45254470 728 73.48901099 729 73.52537723 730 73.56164384 731 73.59781122 732 73.63387978 733 73.66984993 734 73.70572207 735 73.74149660 736 73.77717391 737 73.67706920 738 73.71273713 739 73.61299053 740 73.51351351 741 73.54925776 742 73.58490566 743 73.62045760 744 73.52150538 745 73.55704698 746 73.59249330 747 73.49397590 748 73.52941176 749 73.43124166 750 73.46666667 Final result: 73.4667 +/- 1.6132 Random chance: 25.0083 +/- 1.5824