common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 43.47826087 24 45.83333333 25 48.00000000 26 46.15384615 27 48.14814815 28 50.00000000 29 48.27586207 30 46.66666667 31 48.38709677 32 50.00000000 33 48.48484848 34 50.00000000 35 48.57142857 36 47.22222222 37 45.94594595 38 47.36842105 39 46.15384615 40 45.00000000 41 43.90243902 42 42.85714286 43 41.86046512 44 40.90909091 45 42.22222222 46 41.30434783 47 40.42553191 48 41.66666667 49 42.85714286 50 42.00000000 51 41.17647059 52 40.38461538 53 39.62264151 54 38.88888889 55 40.00000000 56 39.28571429 57 40.35087719 58 41.37931034 59 42.37288136 60 41.66666667 61 42.62295082 62 41.93548387 63 41.26984127 64 42.18750000 65 41.53846154 66 40.90909091 67 41.79104478 68 41.17647059 69 40.57971014 70 40.00000000 71 39.43661972 72 40.27777778 73 39.72602740 74 39.18918919 75 40.00000000 76 40.78947368 77 41.55844156 78 42.30769231 79 41.77215190 80 42.50000000 81 43.20987654 82 42.68292683 83 43.37349398 84 42.85714286 85 42.35294118 86 43.02325581 87 43.67816092 88 44.31818182 89 43.82022472 90 43.33333333 91 42.85714286 92 42.39130435 93 41.93548387 94 42.55319149 95 42.10526316 96 42.70833333 97 42.26804124 98 42.85714286 99 43.43434343 100 43.00000000 101 43.56435644 102 43.13725490 103 42.71844660 104 43.26923077 105 43.80952381 106 43.39622642 107 42.99065421 108 42.59259259 109 42.20183486 110 42.72727273 111 42.34234234 112 41.96428571 113 41.59292035 114 42.10526316 115 41.73913043 116 41.37931034 117 41.02564103 118 40.67796610 119 40.33613445 120 40.00000000 121 40.49586777 122 40.16393443 123 39.83739837 124 40.32258065 125 40.80000000 126 40.47619048 127 40.15748031 128 40.62500000 129 40.31007752 130 40.76923077 131 41.22137405 132 41.66666667 133 41.35338346 134 41.04477612 135 40.74074074 136 41.17647059 137 41.60583942 138 41.30434783 139 41.72661871 140 42.14285714 141 42.55319149 142 42.95774648 143 43.35664336 144 43.05555556 145 42.75862069 146 43.15068493 147 42.85714286 148 43.24324324 149 43.62416107 150 43.33333333 151 43.04635762 152 43.42105263 153 43.79084967 154 43.50649351 155 43.22580645 156 42.94871795 157 43.31210191 158 43.67088608 159 43.39622642 160 43.75000000 161 43.47826087 162 43.20987654 163 42.94478528 164 42.68292683 165 42.42424242 166 42.16867470 167 42.51497006 168 42.26190476 169 42.60355030 170 42.35294118 171 42.69005848 172 42.44186047 173 42.19653179 174 41.95402299 175 41.71428571 176 42.04545455 177 41.80790960 178 41.57303371 179 41.34078212 180 41.11111111 181 40.88397790 182 40.65934066 183 40.43715847 184 40.21739130 185 40.54054054 186 40.86021505 187 41.17647059 188 40.95744681 189 41.26984127 190 41.05263158 191 41.36125654 192 41.14583333 193 40.93264249 194 40.72164948 195 41.02564103 196 41.32653061 197 41.11675127 198 41.41414141 199 41.20603015 200 41.00000000 201 40.79601990 202 40.59405941 203 40.39408867 204 40.19607843 205 40.48780488 206 40.29126214 207 40.57971014 208 40.38461538 209 40.19138756 210 40.00000000 211 39.81042654 212 39.62264151 213 39.43661972 214 39.25233645 215 39.53488372 216 39.35185185 217 39.17050691 218 39.44954128 219 39.72602740 220 39.54545455 221 39.81900452 222 39.63963964 223 39.91031390 224 39.73214286 225 40.00000000 226 40.26548673 227 40.08810573 228 40.35087719 229 40.17467249 230 40.00000000 231 39.82683983 232 40.08620690 233 39.91416309 234 40.17094017 235 40.00000000 236 39.83050847 237 39.66244726 238 39.49579832 239 39.33054393 240 39.58333333 241 39.41908714 242 39.66942149 243 39.91769547 244 39.75409836 245 40.00000000 246 39.83739837 247 39.67611336 248 39.91935484 249 39.75903614 250 39.60000000 251 39.44223108 252 39.28571429 253 39.52569170 254 39.37007874 255 39.21568627 256 39.45312500 257 39.68871595 258 39.53488372 259 39.38223938 260 39.61538462 261 39.84674330 262 39.69465649 263 39.54372624 264 39.77272727 265 40.00000000 266 40.22556391 267 40.44943820 268 40.29850746 269 40.52044610 270 40.74074074 271 40.59040590 272 40.80882353 273 40.65934066 274 40.51094891 275 40.36363636 276 40.57971014 277 40.79422383 278 41.00719424 279 40.86021505 280 41.07142857 281 40.92526690 282 40.78014184 283 40.63604240 284 40.49295775 285 40.35087719 286 40.20979021 287 40.41811847 288 40.27777778 289 40.13840830 290 40.34482759 291 40.54982818 292 40.75342466 293 40.95563140 294 40.81632653 295 40.67796610 296 40.54054054 297 40.40404040 298 40.60402685 299 40.80267559 300 41.00000000 301 41.19601329 302 41.39072848 303 41.25412541 304 41.44736842 305 41.63934426 306 41.83006536 307 42.01954397 308 42.20779221 309 42.07119741 310 41.93548387 311 42.12218650 312 42.30769231 313 42.49201278 314 42.67515924 315 42.53968254 316 42.40506329 317 42.27129338 318 42.13836478 319 42.31974922 320 42.18750000 321 42.05607477 322 42.23602484 323 42.10526316 324 41.97530864 325 42.15384615 326 42.02453988 327 42.20183486 328 42.37804878 329 42.55319149 330 42.42424242 331 42.29607251 332 42.16867470 333 42.34234234 334 42.21556886 335 42.38805970 336 42.55952381 337 42.43323442 338 42.30769231 339 42.18289086 340 42.35294118 341 42.52199413 342 42.69005848 343 42.56559767 344 42.73255814 345 42.60869565 346 42.77456647 347 42.93948127 348 42.81609195 349 42.97994269 350 43.14285714 351 43.01994302 352 42.89772727 353 43.05949008 354 42.93785311 355 42.81690141 356 42.97752809 357 43.13725490 358 43.01675978 359 42.89693593 360 43.05555556 361 42.93628809 362 43.09392265 363 42.97520661 364 43.13186813 365 43.01369863 366 43.16939891 367 43.05177112 368 42.93478261 369 42.81842818 370 42.97297297 371 43.12668464 372 43.01075269 373 42.89544236 374 43.04812834 375 43.20000000 376 43.08510638 377 42.97082228 378 43.12169312 379 43.00791557 380 42.89473684 381 43.04461942 382 43.19371728 383 43.34203655 384 43.22916667 385 43.37662338 386 43.26424870 387 43.15245478 388 43.04123711 389 43.18766067 390 43.07692308 391 43.22250639 392 43.36734694 393 43.25699746 394 43.14720812 395 43.29113924 396 43.43434343 397 43.32493703 398 43.21608040 399 43.35839599 400 43.25000000 401 43.39152120 402 43.28358209 403 43.17617866 404 43.31683168 405 43.45679012 406 43.34975369 407 43.24324324 408 43.13725490 409 43.27628362 410 43.17073171 411 43.06569343 412 42.96116505 413 42.85714286 414 42.99516908 415 43.13253012 416 43.02884615 417 42.92565947 418 42.82296651 419 42.95942721 420 43.09523810 421 42.99287411 422 42.89099526 423 42.78959811 424 42.92452830 425 42.82352941 426 42.95774648 427 42.85714286 428 42.75700935 429 42.89044289 430 42.79069767 431 42.92343387 432 42.82407407 433 42.72517321 434 42.62672811 435 42.52873563 436 42.43119266 437 42.33409611 438 42.46575342 439 42.59681093 440 42.50000000 441 42.40362812 442 42.30769231 443 42.21218962 444 42.11711712 445 42.02247191 446 41.92825112 447 42.05816555 448 41.96428571 449 41.87082405 450 42.00000000 451 42.12860310 452 42.25663717 453 42.16335541 454 42.07048458 455 41.97802198 456 42.10526316 457 42.23194748 458 42.13973799 459 42.04793028 460 42.17391304 461 42.29934924 462 42.42424242 463 42.54859611 464 42.67241379 465 42.58064516 466 42.70386266 467 42.82655246 468 42.94871795 469 43.07036247 470 42.97872340 471 42.88747346 472 42.79661017 473 42.70613108 474 42.61603376 475 42.52631579 476 42.64705882 477 42.55765199 478 42.67782427 479 42.79749478 480 42.70833333 481 42.61954262 482 42.53112033 483 42.44306418 484 42.35537190 485 42.26804124 486 42.18106996 487 42.29979466 488 42.21311475 489 42.12678937 490 42.24489796 491 42.15885947 492 42.07317073 493 41.98782961 494 42.10526316 495 42.22222222 496 42.33870968 497 42.25352113 498 42.16867470 499 42.28456914 500 42.20000000 501 42.11576846 502 42.03187251 503 42.14711730 504 42.26190476 505 42.17821782 506 42.29249012 507 42.20907298 508 42.12598425 509 42.04322200 510 41.96078431 511 41.87866928 512 41.79687500 513 41.71539961 514 41.63424125 515 41.74757282 516 41.86046512 517 41.97292070 518 42.08494208 519 42.00385356 520 42.11538462 521 42.03454894 522 42.14559387 523 42.06500956 524 42.17557252 525 42.09523810 526 42.01520913 527 42.12523719 528 42.23484848 529 42.34404537 530 42.45283019 531 42.37288136 532 42.29323308 533 42.21388368 534 42.32209738 535 42.42990654 536 42.35074627 537 42.45810056 538 42.56505576 539 42.48608534 540 42.40740741 541 42.32902033 542 42.43542435 543 42.35727440 544 42.46323529 545 42.38532110 546 42.30769231 547 42.41316271 548 42.51824818 549 42.44080146 550 42.36363636 551 42.46823956 552 42.57246377 553 42.49547920 554 42.59927798 555 42.52252252 556 42.62589928 557 42.54937163 558 42.65232975 559 42.75491950 560 42.67857143 561 42.60249554 562 42.70462633 563 42.80639432 564 42.73049645 565 42.83185841 566 42.75618375 567 42.68077601 568 42.60563380 569 42.53075571 570 42.63157895 571 42.55691769 572 42.65734266 573 42.58289703 574 42.50871080 575 42.43478261 576 42.53472222 577 42.63431542 578 42.73356401 579 42.65975820 580 42.75862069 581 42.85714286 582 42.78350515 583 42.88164666 584 42.97945205 585 43.07692308 586 43.17406143 587 43.10051107 588 43.02721088 589 42.95415959 590 42.88135593 591 42.80879865 592 42.90540541 593 43.00168634 594 43.09764310 595 43.02521008 596 43.12080537 597 43.04857621 598 42.97658863 599 43.07178631 600 43.00000000 601 42.92845258 602 42.85714286 603 42.95190713 604 42.88079470 605 42.80991736 606 42.73927393 607 42.83360791 608 42.76315789 609 42.69293924 610 42.62295082 611 42.71685761 612 42.64705882 613 42.74061990 614 42.83387622 615 42.76422764 616 42.85714286 617 42.78768233 618 42.88025890 619 42.81098546 620 42.74193548 621 42.67310789 622 42.60450161 623 42.53611557 624 42.46794872 625 42.40000000 626 42.49201278 627 42.42424242 628 42.35668790 629 42.44833068 630 42.38095238 631 42.31378764 632 42.40506329 633 42.33807267 634 42.27129338 635 42.36220472 636 42.29559748 637 42.22919937 638 42.31974922 639 42.41001565 640 42.50000000 641 42.43369735 642 42.52336449 643 42.45723173 644 42.54658385 645 42.48062016 646 42.41486068 647 42.34930448 648 42.28395062 649 42.21879815 650 42.30769231 651 42.24270353 652 42.33128834 653 42.26646248 654 42.35474006 655 42.29007634 656 42.37804878 657 42.46575342 658 42.55319149 659 42.48861912 660 42.57575758 661 42.51134644 662 42.44712991 663 42.38310709 664 42.31927711 665 42.25563910 666 42.19219219 667 42.27886057 668 42.36526946 669 42.45142003 670 42.53731343 671 42.47391952 672 42.41071429 673 42.34769688 674 42.28486647 675 42.22222222 676 42.15976331 677 42.09748892 678 42.18289086 679 42.12076583 680 42.20588235 681 42.29074890 682 42.22873900 683 42.16691069 684 42.25146199 685 42.18978102 686 42.12827988 687 42.06695779 688 42.00581395 689 41.94484761 690 41.88405797 691 41.96816208 692 41.90751445 693 41.84704185 694 41.78674352 695 41.72661871 696 41.81034483 697 41.75035868 698 41.69054441 699 41.77396280 700 41.85714286 701 41.94008559 702 42.02279202 703 42.10526316 704 42.04545455 705 41.98581560 706 42.06798867 707 42.00848656 708 41.94915254 709 42.03102962 710 41.97183099 711 42.05344585 712 42.13483146 713 42.21598878 714 42.29691877 715 42.23776224 716 42.17877095 717 42.11994421 718 42.06128134 719 42.14186370 720 42.08333333 721 42.02496533 722 42.10526316 723 42.18533887 724 42.12707182 725 42.20689655 726 42.28650138 727 42.36588721 728 42.44505495 729 42.38683128 730 42.32876712 731 42.27086183 732 42.21311475 733 42.29195089 734 42.37057221 735 42.31292517 736 42.25543478 737 42.19810041 738 42.27642276 739 42.21921516 740 42.16216216 741 42.10526316 742 42.18328841 743 42.26110363 744 42.33870968 745 42.28187919 746 42.22520107 747 42.16867470 748 42.11229947 749 42.05607477 750 42.13333333 Final result: 42.1333 +/- 1.8042 Random chance: 25.0000 +/- 1.5822