common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 77.77777778 19 78.94736842 20 75.00000000 21 76.19047619 22 77.27272727 23 73.91304348 24 75.00000000 25 72.00000000 26 69.23076923 27 70.37037037 28 67.85714286 29 68.96551724 30 70.00000000 31 67.74193548 32 68.75000000 33 69.69696970 34 70.58823529 35 71.42857143 36 69.44444444 37 70.27027027 38 71.05263158 39 71.79487179 40 72.50000000 41 70.73170732 42 71.42857143 43 69.76744186 44 68.18181818 45 68.88888889 46 69.56521739 47 70.21276596 48 70.83333333 49 71.42857143 50 70.00000000 51 68.62745098 52 69.23076923 53 69.81132075 54 68.51851852 55 67.27272727 56 67.85714286 57 68.42105263 58 68.96551724 59 69.49152542 60 68.33333333 61 68.85245902 62 69.35483871 63 69.84126984 64 70.31250000 65 69.23076923 66 69.69696970 67 70.14925373 68 70.58823529 69 71.01449275 70 71.42857143 71 71.83098592 72 70.83333333 73 71.23287671 74 71.62162162 75 72.00000000 76 72.36842105 77 72.72727273 78 73.07692308 79 73.41772152 80 73.75000000 81 74.07407407 82 74.39024390 83 74.69879518 84 75.00000000 85 74.11764706 86 74.41860465 87 74.71264368 88 75.00000000 89 75.28089888 90 75.55555556 91 75.82417582 92 76.08695652 93 75.26881720 94 74.46808511 95 74.73684211 96 75.00000000 97 74.22680412 98 74.48979592 99 74.74747475 100 75.00000000 101 75.24752475 102 75.49019608 103 74.75728155 104 74.03846154 105 74.28571429 106 73.58490566 107 73.83177570 108 73.14814815 109 72.47706422 110 72.72727273 111 72.97297297 112 73.21428571 113 72.56637168 114 72.80701754 115 73.04347826 116 73.27586207 117 73.50427350 118 73.72881356 119 73.10924370 120 73.33333333 121 72.72727273 122 72.95081967 123 73.17073171 124 73.38709677 125 73.60000000 126 73.80952381 127 74.01574803 128 74.21875000 129 73.64341085 130 73.84615385 131 74.04580153 132 73.48484848 133 73.68421053 134 73.88059701 135 73.33333333 136 73.52941176 137 73.72262774 138 73.18840580 139 73.38129496 140 72.85714286 141 72.34042553 142 72.53521127 143 72.72727273 144 72.91666667 145 73.10344828 146 73.28767123 147 73.46938776 148 72.97297297 149 73.15436242 150 73.33333333 151 73.50993377 152 73.68421053 153 73.85620915 154 74.02597403 155 73.54838710 156 73.71794872 157 73.88535032 158 74.05063291 159 74.21383648 160 74.37500000 161 74.53416149 162 74.69135802 163 74.23312883 164 74.39024390 165 74.54545455 166 74.69879518 167 74.85029940 168 75.00000000 169 74.55621302 170 74.70588235 171 74.85380117 172 74.41860465 173 74.56647399 174 74.71264368 175 74.28571429 176 74.43181818 177 74.57627119 178 74.71910112 179 74.86033520 180 75.00000000 181 75.13812155 182 75.27472527 183 74.86338798 184 75.00000000 185 75.13513514 186 75.26881720 187 75.40106952 188 75.00000000 189 75.13227513 190 74.73684211 191 74.86910995 192 74.47916667 193 74.09326425 194 74.22680412 195 74.35897436 196 74.48979592 197 74.61928934 198 74.74747475 199 74.87437186 200 75.00000000 201 75.12437811 202 75.24752475 203 75.36945813 204 75.49019608 205 75.60975610 206 75.24271845 207 75.36231884 208 75.48076923 209 75.59808612 210 75.71428571 211 75.82938389 212 75.47169811 213 75.58685446 214 75.70093458 215 75.81395349 216 75.46296296 217 75.57603687 218 75.68807339 219 75.79908676 220 75.45454545 221 75.56561086 222 75.67567568 223 75.33632287 224 75.44642857 225 75.11111111 226 74.77876106 227 74.88986784 228 75.00000000 229 75.10917031 230 75.21739130 231 74.89177489 232 74.56896552 233 74.24892704 234 74.35897436 235 74.04255319 236 74.15254237 237 73.83966245 238 73.52941176 239 73.64016736 240 73.75000000 241 73.85892116 242 73.96694215 243 74.07407407 244 73.77049180 245 73.87755102 246 73.98373984 247 74.08906883 248 74.19354839 249 74.29718876 250 74.40000000 251 74.50199203 252 74.60317460 253 74.70355731 254 74.80314961 255 74.90196078 256 75.00000000 257 74.70817121 258 74.80620155 259 74.90347490 260 75.00000000 261 75.09578544 262 75.19083969 263 75.28517110 264 75.37878788 265 75.47169811 266 75.18796992 267 75.28089888 268 75.37313433 269 75.46468401 270 75.55555556 271 75.64575646 272 75.73529412 273 75.45787546 274 75.18248175 275 74.90909091 276 75.00000000 277 75.09025271 278 75.17985612 279 75.26881720 280 75.35714286 281 75.44483986 282 75.53191489 283 75.61837456 284 75.70422535 285 75.78947368 286 75.87412587 287 75.60975610 288 75.69444444 289 75.43252595 290 75.51724138 291 75.60137457 292 75.68493151 293 75.76791809 294 75.51020408 295 75.59322034 296 75.67567568 297 75.75757576 298 75.83892617 299 75.91973244 300 76.00000000 301 75.74750831 302 75.82781457 303 75.90759076 304 75.98684211 305 76.06557377 306 75.81699346 307 75.89576547 308 75.97402597 309 76.05177994 310 75.80645161 311 75.88424437 312 75.64102564 313 75.71884984 314 75.79617834 315 75.87301587 316 75.94936709 317 76.02523659 318 76.10062893 319 75.86206897 320 75.62500000 321 75.70093458 322 75.77639752 323 75.85139319 324 75.92592593 325 76.00000000 326 76.07361963 327 75.84097859 328 75.91463415 329 75.98784195 330 75.75757576 331 75.52870091 332 75.60240964 333 75.67567568 334 75.74850299 335 75.52238806 336 75.29761905 337 75.37091988 338 75.14792899 339 75.22123894 340 75.29411765 341 75.36656891 342 75.43859649 343 75.51020408 344 75.58139535 345 75.36231884 346 75.43352601 347 75.50432277 348 75.28735632 349 75.35816619 350 75.42857143 351 75.49857550 352 75.56818182 353 75.63739377 354 75.42372881 355 75.21126761 356 75.28089888 357 75.35014006 358 75.41899441 359 75.20891365 360 75.27777778 361 75.34626039 362 75.41436464 363 75.48209366 364 75.54945055 365 75.61643836 366 75.40983607 367 75.47683924 368 75.54347826 369 75.60975610 370 75.67567568 371 75.47169811 372 75.26881720 373 75.33512064 374 75.40106952 375 75.20000000 376 75.26595745 377 75.06631300 378 75.13227513 379 74.93403694 380 75.00000000 381 74.80314961 382 74.86910995 383 74.67362924 384 74.73958333 385 74.80519481 386 74.61139896 387 74.41860465 388 74.48453608 389 74.29305913 390 74.10256410 391 73.91304348 392 73.97959184 393 74.04580153 394 73.85786802 395 73.92405063 396 73.73737374 397 73.80352645 398 73.61809045 399 73.68421053 400 73.75000000 401 73.81546135 402 73.88059701 403 73.94540943 404 74.00990099 405 74.07407407 406 74.13793103 407 74.20147420 408 74.26470588 409 74.08312958 410 74.14634146 411 74.20924574 412 74.27184466 413 74.33414044 414 74.39613527 415 74.45783133 416 74.51923077 417 74.34052758 418 74.40191388 419 74.22434368 420 74.28571429 421 74.34679335 422 74.40758294 423 74.46808511 424 74.52830189 425 74.35294118 426 74.17840376 427 74.00468384 428 73.83177570 429 73.89277389 430 73.72093023 431 73.78190255 432 73.84259259 433 73.67205543 434 73.73271889 435 73.79310345 436 73.85321101 437 73.91304348 438 73.74429224 439 73.80410023 440 73.86363636 441 73.92290249 442 73.75565611 443 73.81489842 444 73.87387387 445 73.93258427 446 73.99103139 447 74.04921700 448 74.10714286 449 74.16481069 450 74.00000000 451 74.05764967 452 74.11504425 453 74.17218543 454 74.00881057 455 74.06593407 456 74.12280702 457 74.17943107 458 74.23580786 459 74.29193900 460 74.13043478 461 74.18655098 462 74.24242424 463 74.08207343 464 73.92241379 465 73.97849462 466 74.03433476 467 74.08993576 468 74.14529915 469 73.98720682 470 74.04255319 471 74.09766454 472 74.15254237 473 73.99577167 474 74.05063291 475 73.89473684 476 73.94957983 477 74.00419287 478 74.05857741 479 74.11273486 480 73.95833333 481 74.01247401 482 74.06639004 483 74.12008282 484 74.17355372 485 74.22680412 486 74.27983539 487 74.12731006 488 74.18032787 489 74.23312883 490 74.28571429 491 74.33808554 492 74.39024390 493 74.23935091 494 74.29149798 495 74.34343434 496 74.19354839 497 74.04426559 498 74.09638554 499 74.14829659 500 74.20000000 501 74.25149701 502 74.30278884 503 74.35387674 504 74.40476190 505 74.45544554 506 74.30830040 507 74.35897436 508 74.40944882 509 74.26326130 510 74.31372549 511 74.16829746 512 74.21875000 513 74.07407407 514 74.12451362 515 74.17475728 516 74.03100775 517 74.08123791 518 74.13127413 519 74.18111753 520 74.03846154 521 74.08829175 522 74.13793103 523 73.99617591 524 73.85496183 525 73.71428571 526 73.57414449 527 73.43453510 528 73.29545455 529 73.15689981 530 73.20754717 531 73.25800377 532 73.30827068 533 73.35834897 534 73.22097378 535 73.08411215 536 73.13432836 537 73.18435754 538 73.04832714 539 73.09833024 540 72.96296296 541 73.01293900 542 73.06273063 543 73.11233886 544 73.16176471 545 73.21100917 546 73.07692308 547 73.12614260 548 73.17518248 549 73.22404372 550 73.27272727 551 73.13974592 552 73.00724638 553 73.05605787 554 73.10469314 555 73.15315315 556 73.02158273 557 73.07001795 558 72.93906810 559 72.98747764 560 73.03571429 561 73.08377897 562 73.13167260 563 73.17939609 564 73.22695035 565 73.09734513 566 73.14487633 567 73.19223986 568 73.23943662 569 73.11072056 570 72.98245614 571 73.02977233 572 72.90209790 573 72.94938918 574 72.99651568 575 73.04347826 576 73.09027778 577 73.13691508 578 73.18339100 579 73.22970639 580 73.27586207 581 73.14974182 582 73.19587629 583 73.24185249 584 73.28767123 585 73.33333333 586 73.37883959 587 73.42419080 588 73.46938776 589 73.51443124 590 73.55932203 591 73.43485618 592 73.47972973 593 73.52445194 594 73.56902357 595 73.61344538 596 73.48993289 597 73.53433836 598 73.57859532 599 73.45575960 600 73.50000000 601 73.54409318 602 73.42192691 603 73.30016584 604 73.34437086 605 73.38842975 606 73.43234323 607 73.47611203 608 73.51973684 609 73.39901478 610 73.44262295 611 73.32242226 612 73.36601307 613 73.24632953 614 73.28990228 615 73.17073171 616 73.05194805 617 73.09562399 618 73.13915858 619 73.02100162 620 73.06451613 621 73.10789050 622 72.99035370 623 73.03370787 624 73.07692308 625 73.12000000 626 73.16293930 627 73.20574163 628 73.08917197 629 73.13195548 630 73.17460317 631 73.21711569 632 73.10126582 633 72.98578199 634 73.02839117 635 73.07086614 636 73.11320755 637 73.15541601 638 73.19749216 639 73.23943662 640 73.28125000 641 73.32293292 642 73.20872274 643 73.09486781 644 73.13664596 645 73.02325581 646 73.06501548 647 72.95208655 648 72.99382716 649 72.88135593 650 72.92307692 651 72.96466974 652 72.85276074 653 72.89433384 654 72.78287462 655 72.67175573 656 72.71341463 657 72.75494673 658 72.79635258 659 72.68588771 660 72.72727273 661 72.61724660 662 72.65861027 663 72.69984917 664 72.74096386 665 72.78195489 666 72.82282282 667 72.86356822 668 72.75449102 669 72.79521674 670 72.68656716 671 72.72727273 672 72.76785714 673 72.80832095 674 72.84866469 675 72.88888889 676 72.92899408 677 72.96898080 678 73.00884956 679 73.04860088 680 73.08823529 681 72.98091043 682 73.02052786 683 72.91361640 684 72.95321637 685 72.99270073 686 73.03206997 687 73.07132460 688 73.11046512 689 73.00435414 690 73.04347826 691 72.93777135 692 72.83236994 693 72.87157287 694 72.91066282 695 72.80575540 696 72.84482759 697 72.88378766 698 72.77936963 699 72.67525036 700 72.57142857 701 72.46790300 702 72.50712251 703 72.54623044 704 72.58522727 705 72.62411348 706 72.66288952 707 72.70155587 708 72.74011299 709 72.77856135 710 72.81690141 711 72.85513361 712 72.89325843 713 72.93127630 714 72.82913165 715 72.72727273 716 72.76536313 717 72.80334728 718 72.70194986 719 72.60083449 720 72.63888889 721 72.53814147 722 72.57617729 723 72.61410788 724 72.51381215 725 72.55172414 726 72.45179063 727 72.48968363 728 72.52747253 729 72.56515775 730 72.60273973 731 72.64021888 732 72.67759563 733 72.71487040 734 72.75204360 735 72.78911565 736 72.69021739 737 72.59158752 738 72.62872629 739 72.53044655 740 72.43243243 741 72.46963563 742 72.50673854 743 72.54374159 744 72.44623656 745 72.48322148 746 72.52010724 747 72.42302544 748 72.32620321 749 72.22963952 750 72.26666667 Final result: 72.2667 +/- 1.6358 Random chance: 25.0083 +/- 1.5824