common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 39.13043478 24 41.66666667 25 44.00000000 26 42.30769231 27 44.44444444 28 42.85714286 29 41.37931034 30 40.00000000 31 41.93548387 32 43.75000000 33 42.42424242 34 44.11764706 35 45.71428571 36 44.44444444 37 43.24324324 38 44.73684211 39 43.58974359 40 42.50000000 41 41.46341463 42 40.47619048 43 41.86046512 44 40.90909091 45 42.22222222 46 41.30434783 47 40.42553191 48 41.66666667 49 42.85714286 50 44.00000000 51 43.13725490 52 42.30769231 53 41.50943396 54 40.74074074 55 41.81818182 56 41.07142857 57 42.10526316 58 43.10344828 59 44.06779661 60 43.33333333 61 44.26229508 62 43.54838710 63 42.85714286 64 43.75000000 65 43.07692308 66 42.42424242 67 43.28358209 68 42.64705882 69 42.02898551 70 42.85714286 71 43.66197183 72 44.44444444 73 43.83561644 74 44.59459459 75 45.33333333 76 46.05263158 77 46.75324675 78 47.43589744 79 46.83544304 80 47.50000000 81 48.14814815 82 47.56097561 83 48.19277108 84 47.61904762 85 48.23529412 86 47.67441860 87 48.27586207 88 48.86363636 89 48.31460674 90 47.77777778 91 47.25274725 92 46.73913043 93 46.23655914 94 46.80851064 95 46.31578947 96 46.87500000 97 46.39175258 98 46.93877551 99 46.46464646 100 46.00000000 101 46.53465347 102 46.07843137 103 45.63106796 104 46.15384615 105 46.66666667 106 46.22641509 107 45.79439252 108 45.37037037 109 44.95412844 110 45.45454545 111 45.94594595 112 45.53571429 113 45.13274336 114 45.61403509 115 45.21739130 116 45.68965517 117 45.29914530 118 44.91525424 119 44.53781513 120 44.16666667 121 44.62809917 122 44.26229508 123 43.90243902 124 44.35483871 125 44.80000000 126 44.44444444 127 44.09448819 128 44.53125000 129 44.18604651 130 44.61538462 131 45.03816794 132 45.45454545 133 45.11278195 134 44.77611940 135 44.44444444 136 44.85294118 137 45.25547445 138 44.92753623 139 45.32374101 140 45.71428571 141 46.09929078 142 46.47887324 143 46.85314685 144 46.52777778 145 46.89655172 146 47.26027397 147 46.93877551 148 47.29729730 149 47.65100671 150 47.33333333 151 47.01986755 152 47.36842105 153 47.05882353 154 46.75324675 155 46.45161290 156 46.15384615 157 46.49681529 158 46.83544304 159 46.54088050 160 46.87500000 161 46.58385093 162 46.29629630 163 46.01226994 164 45.73170732 165 45.45454545 166 45.18072289 167 45.50898204 168 45.23809524 169 45.56213018 170 45.29411765 171 45.61403509 172 45.34883721 173 45.08670520 174 44.82758621 175 44.57142857 176 44.88636364 177 44.63276836 178 44.38202247 179 44.69273743 180 44.44444444 181 44.19889503 182 43.95604396 183 43.71584699 184 43.47826087 185 43.78378378 186 44.08602151 187 43.85026738 188 44.14893617 189 44.44444444 190 44.21052632 191 44.50261780 192 44.27083333 193 44.04145078 194 44.32989691 195 44.61538462 196 44.89795918 197 44.67005076 198 44.94949495 199 44.72361809 200 44.50000000 201 44.27860697 202 44.05940594 203 43.84236453 204 43.62745098 205 43.90243902 206 43.68932039 207 43.96135266 208 43.75000000 209 43.54066986 210 43.33333333 211 43.12796209 212 42.92452830 213 42.72300469 214 42.52336449 215 42.79069767 216 42.59259259 217 42.39631336 218 42.66055046 219 42.92237443 220 42.72727273 221 42.98642534 222 42.79279279 223 43.04932735 224 43.30357143 225 43.55555556 226 43.80530973 227 43.61233480 228 43.85964912 229 43.66812227 230 43.47826087 231 43.29004329 232 43.53448276 233 43.34763948 234 43.58974359 235 43.40425532 236 43.22033898 237 43.03797468 238 42.85714286 239 42.67782427 240 42.91666667 241 42.73858921 242 42.97520661 243 43.20987654 244 43.03278689 245 43.26530612 246 43.08943089 247 42.91497976 248 43.14516129 249 42.97188755 250 42.80000000 251 42.62948207 252 42.46031746 253 42.68774704 254 42.51968504 255 42.35294118 256 42.57812500 257 42.80155642 258 42.63565891 259 42.47104247 260 42.69230769 261 42.91187739 262 42.74809160 263 42.58555133 264 42.80303030 265 42.64150943 266 42.85714286 267 42.69662921 268 42.53731343 269 42.75092937 270 42.96296296 271 42.80442804 272 43.01470588 273 42.85714286 274 42.70072993 275 42.54545455 276 42.75362319 277 42.96028881 278 43.16546763 279 43.01075269 280 42.85714286 281 42.70462633 282 42.55319149 283 42.40282686 284 42.25352113 285 42.45614035 286 42.30769231 287 42.50871080 288 42.36111111 289 42.21453287 290 42.41379310 291 42.61168385 292 42.80821918 293 43.00341297 294 42.85714286 295 42.71186441 296 42.56756757 297 42.42424242 298 42.61744966 299 42.80936455 300 43.00000000 301 43.18936877 302 43.37748344 303 43.23432343 304 43.09210526 305 43.27868852 306 43.46405229 307 43.64820847 308 43.83116883 309 44.01294498 310 43.87096774 311 44.05144695 312 44.23076923 313 44.40894569 314 44.58598726 315 44.44444444 316 44.30379747 317 44.16403785 318 44.02515723 319 44.20062696 320 44.06250000 321 43.92523364 322 44.09937888 323 43.96284830 324 43.82716049 325 44.00000000 326 43.86503067 327 44.03669725 328 44.20731707 329 44.37689970 330 44.24242424 331 44.10876133 332 44.27710843 333 44.44444444 334 44.31137725 335 44.47761194 336 44.64285714 337 44.51038576 338 44.37869822 339 44.24778761 340 44.41176471 341 44.57478006 342 44.73684211 343 44.60641399 344 44.76744186 345 44.63768116 346 44.79768786 347 44.95677233 348 44.82758621 349 44.69914040 350 44.85714286 351 44.72934473 352 44.60227273 353 44.75920680 354 44.63276836 355 44.50704225 356 44.66292135 357 44.53781513 358 44.41340782 359 44.28969359 360 44.44444444 361 44.32132964 362 44.47513812 363 44.35261708 364 44.50549451 365 44.38356164 366 44.53551913 367 44.41416894 368 44.29347826 369 44.17344173 370 44.32432432 371 44.47439353 372 44.35483871 373 44.23592493 374 44.38502674 375 44.53333333 376 44.41489362 377 44.29708223 378 44.44444444 379 44.32717678 380 44.21052632 381 44.35695538 382 44.50261780 383 44.64751958 384 44.53125000 385 44.67532468 386 44.55958549 387 44.44444444 388 44.32989691 389 44.47300771 390 44.35897436 391 44.50127877 392 44.64285714 393 44.52926209 394 44.41624365 395 44.55696203 396 44.69696970 397 44.58438287 398 44.47236181 399 44.61152882 400 44.50000000 401 44.63840399 402 44.52736318 403 44.41687345 404 44.55445545 405 44.44444444 406 44.33497537 407 44.22604423 408 44.11764706 409 44.25427873 410 44.14634146 411 44.03892944 412 43.93203883 413 43.82566586 414 43.96135266 415 44.09638554 416 43.99038462 417 43.88489209 418 43.77990431 419 43.91408115 420 43.80952381 421 43.70546318 422 43.60189573 423 43.49881797 424 43.63207547 425 43.52941176 426 43.66197183 427 43.55971897 428 43.45794393 429 43.58974359 430 43.48837209 431 43.61948956 432 43.51851852 433 43.41801386 434 43.31797235 435 43.21839080 436 43.11926606 437 43.02059497 438 43.15068493 439 43.28018223 440 43.18181818 441 43.08390023 442 42.98642534 443 42.88939052 444 42.79279279 445 42.69662921 446 42.60089686 447 42.72930649 448 42.63392857 449 42.53897550 450 42.66666667 451 42.79379157 452 42.92035398 453 43.04635762 454 42.95154185 455 42.85714286 456 42.98245614 457 42.88840263 458 42.79475983 459 42.70152505 460 42.82608696 461 42.95010846 462 43.07359307 463 43.19654428 464 43.31896552 465 43.22580645 466 43.13304721 467 43.25481799 468 43.37606838 469 43.49680171 470 43.40425532 471 43.31210191 472 43.22033898 473 43.12896406 474 43.03797468 475 42.94736842 476 43.06722689 477 42.97693920 478 43.09623431 479 43.21503132 480 43.12500000 481 43.03534304 482 42.94605809 483 42.85714286 484 42.76859504 485 42.68041237 486 42.59259259 487 42.71047228 488 42.62295082 489 42.53578732 490 42.65306122 491 42.56619145 492 42.47967480 493 42.39350913 494 42.51012146 495 42.62626263 496 42.74193548 497 42.65593561 498 42.57028112 499 42.68537074 500 42.60000000 501 42.51497006 502 42.43027888 503 42.54473161 504 42.65873016 505 42.57425743 506 42.68774704 507 42.60355030 508 42.51968504 509 42.43614931 510 42.35294118 511 42.27005871 512 42.18750000 513 42.10526316 514 42.02334630 515 41.94174757 516 42.05426357 517 42.16634429 518 42.27799228 519 42.19653179 520 42.30769231 521 42.22648752 522 42.33716475 523 42.25621415 524 42.36641221 525 42.28571429 526 42.20532319 527 42.31499051 528 42.42424242 529 42.53308129 530 42.64150943 531 42.56120527 532 42.48120301 533 42.40150094 534 42.32209738 535 42.42990654 536 42.35074627 537 42.45810056 538 42.56505576 539 42.48608534 540 42.40740741 541 42.32902033 542 42.43542435 543 42.35727440 544 42.46323529 545 42.38532110 546 42.30769231 547 42.41316271 548 42.51824818 549 42.44080146 550 42.36363636 551 42.46823956 552 42.57246377 553 42.49547920 554 42.59927798 555 42.52252252 556 42.62589928 557 42.54937163 558 42.65232975 559 42.75491950 560 42.67857143 561 42.60249554 562 42.70462633 563 42.80639432 564 42.73049645 565 42.83185841 566 42.75618375 567 42.68077601 568 42.60563380 569 42.53075571 570 42.63157895 571 42.55691769 572 42.65734266 573 42.58289703 574 42.50871080 575 42.43478261 576 42.53472222 577 42.63431542 578 42.73356401 579 42.65975820 580 42.75862069 581 42.85714286 582 42.78350515 583 42.88164666 584 42.97945205 585 43.07692308 586 43.17406143 587 43.10051107 588 43.02721088 589 42.95415959 590 42.88135593 591 42.80879865 592 42.90540541 593 43.00168634 594 43.09764310 595 43.02521008 596 43.12080537 597 43.04857621 598 43.14381271 599 43.23873122 600 43.16666667 601 43.26123128 602 43.18936877 603 43.28358209 604 43.21192053 605 43.14049587 606 43.06930693 607 43.16309720 608 43.09210526 609 43.02134647 610 42.95081967 611 43.04418985 612 42.97385621 613 42.90375204 614 42.99674267 615 42.92682927 616 43.01948052 617 42.94975689 618 43.04207120 619 42.97253635 620 42.90322581 621 42.83413849 622 42.76527331 623 42.69662921 624 42.62820513 625 42.56000000 626 42.65175719 627 42.58373206 628 42.51592357 629 42.60731320 630 42.53968254 631 42.47226624 632 42.56329114 633 42.49605055 634 42.42902208 635 42.51968504 636 42.45283019 637 42.38618524 638 42.47648903 639 42.56651017 640 42.65625000 641 42.58970359 642 42.67912773 643 42.61275272 644 42.70186335 645 42.63565891 646 42.56965944 647 42.50386399 648 42.43827160 649 42.37288136 650 42.46153846 651 42.39631336 652 42.48466258 653 42.41960184 654 42.50764526 655 42.44274809 656 42.53048780 657 42.61796043 658 42.70516717 659 42.64036419 660 42.57575758 661 42.51134644 662 42.44712991 663 42.38310709 664 42.31927711 665 42.25563910 666 42.19219219 667 42.27886057 668 42.36526946 669 42.45142003 670 42.53731343 671 42.47391952 672 42.41071429 673 42.34769688 674 42.28486647 675 42.22222222 676 42.15976331 677 42.09748892 678 42.18289086 679 42.12076583 680 42.20588235 681 42.29074890 682 42.22873900 683 42.31332357 684 42.39766082 685 42.33576642 686 42.27405248 687 42.21251820 688 42.15116279 689 42.08998549 690 42.02898551 691 42.11287988 692 42.05202312 693 41.99134199 694 41.93083573 695 41.87050360 696 41.95402299 697 41.89383070 698 41.83381089 699 41.91702432 700 42.00000000 701 41.94008559 702 42.02279202 703 42.10526316 704 42.04545455 705 41.98581560 706 42.06798867 707 42.00848656 708 41.94915254 709 42.03102962 710 42.11267606 711 42.19409283 712 42.27528090 713 42.35624123 714 42.43697479 715 42.37762238 716 42.31843575 717 42.25941423 718 42.20055710 719 42.28094576 720 42.36111111 721 42.30235784 722 42.38227147 723 42.46196404 724 42.40331492 725 42.48275862 726 42.56198347 727 42.64099037 728 42.71978022 729 42.66117970 730 42.60273973 731 42.54445964 732 42.48633880 733 42.56480218 734 42.64305177 735 42.72108844 736 42.66304348 737 42.60515604 738 42.68292683 739 42.76048714 740 42.83783784 741 42.78002699 742 42.85714286 743 42.93405114 744 43.01075269 745 42.95302013 746 42.89544236 747 42.83801874 748 42.78074866 749 42.72363151 750 42.80000000 Final result: 42.8000 +/- 1.8079 Random chance: 25.0000 +/- 1.5822