common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 47.05882353 18 44.44444444 19 47.36842105 20 50.00000000 21 47.61904762 22 45.45454545 23 43.47826087 24 45.83333333 25 48.00000000 26 46.15384615 27 48.14814815 28 46.42857143 29 44.82758621 30 43.33333333 31 45.16129032 32 46.87500000 33 45.45454545 34 47.05882353 35 48.57142857 36 47.22222222 37 45.94594595 38 47.36842105 39 46.15384615 40 45.00000000 41 43.90243902 42 42.85714286 43 44.18604651 44 43.18181818 45 44.44444444 46 43.47826087 47 42.55319149 48 43.75000000 49 44.89795918 50 46.00000000 51 45.09803922 52 44.23076923 53 43.39622642 54 42.59259259 55 43.63636364 56 42.85714286 57 43.85964912 58 44.82758621 59 45.76271186 60 45.00000000 61 45.90163934 62 45.16129032 63 44.44444444 64 45.31250000 65 44.61538462 66 43.93939394 67 44.77611940 68 44.11764706 69 43.47826087 70 44.28571429 71 45.07042254 72 45.83333333 73 45.20547945 74 45.94594595 75 46.66666667 76 47.36842105 77 48.05194805 78 48.71794872 79 48.10126582 80 48.75000000 81 49.38271605 82 48.78048780 83 49.39759036 84 48.80952381 85 49.41176471 86 48.83720930 87 49.42528736 88 50.00000000 89 49.43820225 90 48.88888889 91 48.35164835 92 47.82608696 93 47.31182796 94 47.87234043 95 47.36842105 96 47.91666667 97 47.42268041 98 47.95918367 99 47.47474747 100 47.00000000 101 47.52475248 102 47.05882353 103 46.60194175 104 47.11538462 105 47.61904762 106 47.16981132 107 46.72897196 108 46.29629630 109 46.78899083 110 47.27272727 111 47.74774775 112 47.32142857 113 46.90265487 114 47.36842105 115 47.82608696 116 48.27586207 117 47.86324786 118 47.45762712 119 47.05882353 120 46.66666667 121 47.10743802 122 46.72131148 123 46.34146341 124 46.77419355 125 47.20000000 126 46.82539683 127 46.45669291 128 46.87500000 129 46.51162791 130 46.92307692 131 47.32824427 132 47.72727273 133 47.36842105 134 47.01492537 135 46.66666667 136 47.05882353 137 47.44525547 138 47.10144928 139 47.48201439 140 47.85714286 141 48.22695035 142 48.59154930 143 48.95104895 144 48.61111111 145 48.27586207 146 48.63013699 147 48.29931973 148 48.64864865 149 48.99328859 150 48.66666667 151 48.34437086 152 48.68421053 153 48.36601307 154 48.05194805 155 47.74193548 156 47.43589744 157 47.77070064 158 48.10126582 159 47.79874214 160 48.12500000 161 47.82608696 162 47.53086420 163 47.23926380 164 46.95121951 165 46.66666667 166 46.38554217 167 46.70658683 168 46.42857143 169 46.74556213 170 46.47058824 171 46.78362573 172 46.51162791 173 46.24277457 174 45.97701149 175 45.71428571 176 46.02272727 177 45.76271186 178 46.06741573 179 45.81005587 180 45.55555556 181 45.30386740 182 45.05494505 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 45.21276596 189 45.50264550 190 45.26315789 191 45.54973822 192 45.31250000 193 45.07772021 194 44.84536082 195 45.12820513 196 45.40816327 197 45.17766497 198 45.45454545 199 45.22613065 200 45.00000000 201 44.77611940 202 44.55445545 203 44.33497537 204 44.11764706 205 44.39024390 206 44.17475728 207 44.44444444 208 44.23076923 209 44.01913876 210 43.80952381 211 43.60189573 212 43.39622642 213 43.19248826 214 42.99065421 215 43.25581395 216 43.05555556 217 42.85714286 218 43.11926606 219 43.37899543 220 43.18181818 221 42.98642534 222 42.79279279 223 43.04932735 224 42.85714286 225 43.11111111 226 43.36283186 227 43.17180617 228 43.42105263 229 43.23144105 230 43.04347826 231 42.85714286 232 43.10344828 233 42.91845494 234 43.16239316 235 42.97872340 236 42.79661017 237 42.61603376 238 42.43697479 239 42.25941423 240 42.50000000 241 42.32365145 242 42.56198347 243 42.79835391 244 42.62295082 245 42.85714286 246 42.68292683 247 42.51012146 248 42.74193548 249 42.57028112 250 42.40000000 251 42.23107570 252 42.06349206 253 42.29249012 254 42.12598425 255 41.96078431 256 42.18750000 257 42.41245136 258 42.24806202 259 42.08494208 260 42.30769231 261 42.52873563 262 42.36641221 263 42.20532319 264 42.42424242 265 42.26415094 266 42.48120301 267 42.32209738 268 42.16417910 269 42.37918216 270 42.59259259 271 42.43542435 272 42.64705882 273 42.49084249 274 42.33576642 275 42.18181818 276 42.39130435 277 42.59927798 278 42.80575540 279 42.65232975 280 42.50000000 281 42.34875445 282 42.19858156 283 42.04946996 284 41.90140845 285 42.10526316 286 41.95804196 287 42.16027875 288 42.01388889 289 41.86851211 290 42.06896552 291 42.26804124 292 42.46575342 293 42.66211604 294 42.51700680 295 42.37288136 296 42.22972973 297 42.08754209 298 42.28187919 299 42.47491639 300 42.66666667 301 42.52491694 302 42.71523179 303 42.57425743 304 42.43421053 305 42.62295082 306 42.81045752 307 42.99674267 308 43.18181818 309 43.36569579 310 43.22580645 311 43.40836013 312 43.58974359 313 43.76996805 314 43.94904459 315 43.80952381 316 43.67088608 317 43.53312303 318 43.39622642 319 43.57366771 320 43.43750000 321 43.30218069 322 43.47826087 323 43.34365325 324 43.20987654 325 43.38461538 326 43.25153374 327 43.42507645 328 43.59756098 329 43.76899696 330 43.63636364 331 43.50453172 332 43.67469880 333 43.84384384 334 43.71257485 335 43.88059701 336 44.04761905 337 43.91691395 338 43.78698225 339 43.65781711 340 43.82352941 341 43.98826979 342 44.15204678 343 44.02332362 344 44.18604651 345 44.05797101 346 44.21965318 347 44.38040346 348 44.25287356 349 44.12607450 350 44.28571429 351 44.15954416 352 44.03409091 353 44.19263456 354 44.06779661 355 43.94366197 356 44.10112360 357 43.97759104 358 43.85474860 359 43.73259053 360 43.88888889 361 43.76731302 362 43.92265193 363 43.80165289 364 43.95604396 365 43.83561644 366 43.98907104 367 43.86920981 368 43.75000000 369 43.63143631 370 43.78378378 371 43.93530997 372 43.81720430 373 43.69973190 374 43.85026738 375 44.00000000 376 43.88297872 377 43.76657825 378 43.91534392 379 43.79947230 380 43.68421053 381 43.83202100 382 43.97905759 383 44.12532637 384 44.01041667 385 44.15584416 386 44.04145078 387 43.92764858 388 43.81443299 389 43.95886889 390 43.84615385 391 43.98976982 392 44.13265306 393 44.02035623 394 43.90862944 395 44.05063291 396 44.19191919 397 44.33249370 398 44.22110553 399 44.36090226 400 44.25000000 401 44.38902743 402 44.27860697 403 44.16873449 404 44.30693069 405 44.19753086 406 44.08866995 407 43.98034398 408 43.87254902 409 44.00977995 410 43.90243902 411 43.79562044 412 43.68932039 413 43.58353511 414 43.71980676 415 43.85542169 416 43.75000000 417 43.64508393 418 43.54066986 419 43.67541766 420 43.57142857 421 43.46793349 422 43.36492891 423 43.26241135 424 43.39622642 425 43.29411765 426 43.42723005 427 43.32552693 428 43.22429907 429 43.35664336 430 43.25581395 431 43.38747100 432 43.28703704 433 43.18706697 434 43.08755760 435 42.98850575 436 42.88990826 437 42.79176201 438 42.92237443 439 43.05239180 440 42.95454545 441 42.85714286 442 42.76018100 443 42.66365688 444 42.56756757 445 42.47191011 446 42.37668161 447 42.50559284 448 42.41071429 449 42.31625835 450 42.44444444 451 42.57206208 452 42.69911504 453 42.82560706 454 42.73127753 455 42.63736264 456 42.76315789 457 42.66958425 458 42.57641921 459 42.48366013 460 42.60869565 461 42.73318872 462 42.85714286 463 42.98056156 464 43.10344828 465 43.01075269 466 42.91845494 467 43.04068522 468 43.16239316 469 43.28358209 470 43.19148936 471 43.09978769 472 43.00847458 473 42.91754757 474 42.82700422 475 42.73684211 476 42.85714286 477 42.76729560 478 42.88702929 479 43.00626305 480 42.91666667 481 42.82744283 482 42.73858921 483 42.65010352 484 42.56198347 485 42.47422680 486 42.59259259 487 42.71047228 488 42.62295082 489 42.53578732 490 42.65306122 491 42.56619145 492 42.47967480 493 42.39350913 494 42.51012146 495 42.62626263 496 42.74193548 497 42.65593561 498 42.57028112 499 42.68537074 500 42.60000000 501 42.51497006 502 42.43027888 503 42.54473161 504 42.65873016 505 42.57425743 506 42.68774704 507 42.60355030 508 42.51968504 509 42.43614931 510 42.35294118 511 42.27005871 512 42.18750000 513 42.10526316 514 42.02334630 515 41.94174757 516 42.05426357 517 42.16634429 518 42.27799228 519 42.19653179 520 42.30769231 521 42.22648752 522 42.33716475 523 42.25621415 524 42.36641221 525 42.28571429 526 42.20532319 527 42.31499051 528 42.42424242 529 42.53308129 530 42.64150943 531 42.56120527 532 42.48120301 533 42.40150094 534 42.50936330 535 42.61682243 536 42.53731343 537 42.64432030 538 42.75092937 539 42.67161410 540 42.59259259 541 42.51386322 542 42.61992620 543 42.54143646 544 42.64705882 545 42.56880734 546 42.49084249 547 42.59597806 548 42.70072993 549 42.62295082 550 42.54545455 551 42.64972777 552 42.75362319 553 42.67631103 554 42.77978339 555 42.70270270 556 42.80575540 557 42.72890485 558 42.83154122 559 42.93381038 560 42.85714286 561 42.78074866 562 42.88256228 563 42.98401421 564 42.90780142 565 42.83185841 566 42.75618375 567 42.68077601 568 42.60563380 569 42.53075571 570 42.63157895 571 42.55691769 572 42.65734266 573 42.58289703 574 42.50871080 575 42.43478261 576 42.53472222 577 42.63431542 578 42.73356401 579 42.65975820 580 42.75862069 581 42.85714286 582 42.78350515 583 42.88164666 584 42.97945205 585 43.07692308 586 43.17406143 587 43.10051107 588 43.02721088 589 42.95415959 590 42.88135593 591 42.80879865 592 42.90540541 593 43.00168634 594 43.09764310 595 43.02521008 596 43.12080537 597 43.04857621 598 43.14381271 599 43.23873122 600 43.16666667 601 43.26123128 602 43.18936877 603 43.28358209 604 43.21192053 605 43.14049587 606 43.06930693 607 43.16309720 608 43.09210526 609 43.02134647 610 42.95081967 611 43.04418985 612 42.97385621 613 42.90375204 614 42.99674267 615 42.92682927 616 43.01948052 617 42.94975689 618 43.04207120 619 42.97253635 620 42.90322581 621 42.83413849 622 42.76527331 623 42.69662921 624 42.62820513 625 42.56000000 626 42.65175719 627 42.58373206 628 42.51592357 629 42.60731320 630 42.53968254 631 42.47226624 632 42.56329114 633 42.49605055 634 42.42902208 635 42.51968504 636 42.45283019 637 42.38618524 638 42.47648903 639 42.56651017 640 42.65625000 641 42.58970359 642 42.67912773 643 42.61275272 644 42.70186335 645 42.63565891 646 42.56965944 647 42.50386399 648 42.43827160 649 42.37288136 650 42.46153846 651 42.39631336 652 42.48466258 653 42.41960184 654 42.50764526 655 42.44274809 656 42.53048780 657 42.61796043 658 42.70516717 659 42.64036419 660 42.57575758 661 42.51134644 662 42.44712991 663 42.38310709 664 42.31927711 665 42.40601504 666 42.34234234 667 42.42878561 668 42.51497006 669 42.60089686 670 42.68656716 671 42.62295082 672 42.55952381 673 42.49628529 674 42.43323442 675 42.37037037 676 42.30769231 677 42.24519941 678 42.33038348 679 42.26804124 680 42.35294118 681 42.43759178 682 42.37536657 683 42.31332357 684 42.39766082 685 42.33576642 686 42.27405248 687 42.21251820 688 42.15116279 689 42.08998549 690 42.02898551 691 42.11287988 692 42.05202312 693 41.99134199 694 41.93083573 695 41.87050360 696 41.95402299 697 41.89383070 698 41.83381089 699 41.91702432 700 42.00000000 701 41.94008559 702 42.02279202 703 42.10526316 704 42.04545455 705 41.98581560 706 42.06798867 707 42.00848656 708 41.94915254 709 42.03102962 710 42.11267606 711 42.19409283 712 42.27528090 713 42.35624123 714 42.43697479 715 42.37762238 716 42.31843575 717 42.25941423 718 42.20055710 719 42.28094576 720 42.22222222 721 42.16366158 722 42.24376731 723 42.32365145 724 42.26519337 725 42.34482759 726 42.42424242 727 42.50343879 728 42.58241758 729 42.52400549 730 42.46575342 731 42.40766074 732 42.34972678 733 42.42837653 734 42.50681199 735 42.44897959 736 42.39130435 737 42.33378562 738 42.41192412 739 42.48985115 740 42.56756757 741 42.51012146 742 42.58760108 743 42.66487214 744 42.74193548 745 42.68456376 746 42.62734584 747 42.57028112 748 42.51336898 749 42.45660881 750 42.53333333 Final result: 42.5333 +/- 1.8065 Random chance: 25.0000 +/- 1.5822