common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 39.13043478 24 41.66666667 25 44.00000000 26 42.30769231 27 44.44444444 28 42.85714286 29 41.37931034 30 40.00000000 31 41.93548387 32 43.75000000 33 42.42424242 34 44.11764706 35 42.85714286 36 41.66666667 37 40.54054054 38 42.10526316 39 41.02564103 40 40.00000000 41 39.02439024 42 38.09523810 43 37.20930233 44 36.36363636 45 37.77777778 46 36.95652174 47 36.17021277 48 37.50000000 49 38.77551020 50 38.00000000 51 37.25490196 52 36.53846154 53 35.84905660 54 35.18518519 55 36.36363636 56 35.71428571 57 36.84210526 58 37.93103448 59 37.28813559 60 36.66666667 61 37.70491803 62 37.09677419 63 36.50793651 64 37.50000000 65 36.92307692 66 37.87878788 67 38.80597015 68 38.23529412 69 37.68115942 70 37.14285714 71 36.61971831 72 36.11111111 73 35.61643836 74 36.48648649 75 37.33333333 76 38.15789474 77 38.96103896 78 39.74358974 79 39.24050633 80 40.00000000 81 40.74074074 82 40.24390244 83 40.96385542 84 40.47619048 85 40.00000000 86 40.69767442 87 41.37931034 88 42.04545455 89 41.57303371 90 41.11111111 91 40.65934066 92 40.21739130 93 39.78494624 94 40.42553191 95 41.05263158 96 41.66666667 97 41.23711340 98 41.83673469 99 41.41414141 100 41.00000000 101 41.58415842 102 41.17647059 103 40.77669903 104 41.34615385 105 41.90476190 106 41.50943396 107 41.12149533 108 40.74074074 109 41.28440367 110 41.81818182 111 41.44144144 112 41.96428571 113 41.59292035 114 42.10526316 115 41.73913043 116 41.37931034 117 41.02564103 118 40.67796610 119 40.33613445 120 40.00000000 121 40.49586777 122 40.16393443 123 39.83739837 124 40.32258065 125 40.80000000 126 40.47619048 127 40.15748031 128 40.62500000 129 40.31007752 130 40.76923077 131 41.22137405 132 41.66666667 133 41.35338346 134 41.04477612 135 40.74074074 136 40.44117647 137 40.87591241 138 40.57971014 139 41.00719424 140 41.42857143 141 41.84397163 142 42.25352113 143 42.65734266 144 42.36111111 145 42.06896552 146 42.46575342 147 42.17687075 148 42.56756757 149 42.95302013 150 42.66666667 151 42.38410596 152 42.76315789 153 42.48366013 154 42.20779221 155 41.93548387 156 41.66666667 157 42.03821656 158 42.40506329 159 42.13836478 160 42.50000000 161 42.23602484 162 41.97530864 163 41.71779141 164 41.46341463 165 41.21212121 166 40.96385542 167 41.31736527 168 41.07142857 169 41.42011834 170 41.17647059 171 41.52046784 172 41.27906977 173 41.04046243 174 40.80459770 175 40.57142857 176 40.90909091 177 40.67796610 178 41.01123596 179 40.78212291 180 40.55555556 181 40.33149171 182 40.10989011 183 39.89071038 184 39.67391304 185 40.00000000 186 40.32258065 187 40.10695187 188 39.89361702 189 40.21164021 190 40.00000000 191 40.31413613 192 40.10416667 193 39.89637306 194 39.69072165 195 40.00000000 196 40.30612245 197 40.10152284 198 39.89898990 199 39.69849246 200 39.50000000 201 39.30348259 202 39.10891089 203 38.91625616 204 38.72549020 205 39.02439024 206 38.83495146 207 39.13043478 208 38.94230769 209 38.75598086 210 38.57142857 211 38.38862559 212 38.20754717 213 38.02816901 214 37.85046729 215 38.13953488 216 37.96296296 217 37.78801843 218 38.07339450 219 38.35616438 220 38.18181818 221 38.00904977 222 37.83783784 223 38.11659193 224 38.39285714 225 38.66666667 226 38.93805310 227 38.76651982 228 39.03508772 229 38.86462882 230 38.69565217 231 38.52813853 232 38.36206897 233 38.19742489 234 38.46153846 235 38.29787234 236 38.13559322 237 37.97468354 238 37.81512605 239 37.65690377 240 37.91666667 241 37.75933610 242 38.01652893 243 38.27160494 244 38.11475410 245 38.36734694 246 38.21138211 247 38.05668016 248 38.30645161 249 38.15261044 250 38.00000000 251 37.84860558 252 37.69841270 253 37.94466403 254 37.79527559 255 37.64705882 256 37.89062500 257 38.13229572 258 37.98449612 259 37.83783784 260 38.07692308 261 38.31417625 262 38.16793893 263 38.02281369 264 38.25757576 265 38.11320755 266 37.96992481 267 37.82771536 268 37.68656716 269 37.91821561 270 38.14814815 271 38.00738007 272 38.23529412 273 38.09523810 274 37.95620438 275 37.81818182 276 38.04347826 277 38.26714801 278 38.48920863 279 38.35125448 280 38.57142857 281 38.43416370 282 38.29787234 283 38.16254417 284 38.02816901 285 37.89473684 286 37.76223776 287 37.97909408 288 37.84722222 289 37.71626298 290 37.93103448 291 38.14432990 292 38.35616438 293 38.56655290 294 38.43537415 295 38.30508475 296 38.17567568 297 38.04713805 298 38.25503356 299 38.46153846 300 38.66666667 301 38.53820598 302 38.74172185 303 38.61386139 304 38.48684211 305 38.68852459 306 38.88888889 307 39.08794788 308 39.28571429 309 39.48220065 310 39.35483871 311 39.54983923 312 39.74358974 313 39.93610224 314 40.12738854 315 40.00000000 316 39.87341772 317 39.74763407 318 39.62264151 319 39.81191223 320 39.68750000 321 39.56386293 322 39.75155280 323 39.62848297 324 39.50617284 325 39.69230769 326 39.57055215 327 39.75535168 328 39.93902439 329 40.12158055 330 40.00000000 331 39.87915408 332 39.75903614 333 39.93993994 334 39.82035928 335 40.00000000 336 40.17857143 337 40.05934718 338 39.94082840 339 39.82300885 340 39.70588235 341 39.88269795 342 40.05847953 343 39.94169096 344 40.11627907 345 40.00000000 346 40.17341040 347 40.34582133 348 40.22988506 349 40.40114613 350 40.57142857 351 40.45584046 352 40.34090909 353 40.50991501 354 40.39548023 355 40.28169014 356 40.44943820 357 40.61624650 358 40.50279330 359 40.38997214 360 40.55555556 361 40.44321330 362 40.60773481 363 40.49586777 364 40.38461538 365 40.27397260 366 40.16393443 367 40.05449591 368 39.94565217 369 39.83739837 370 40.00000000 371 40.16172507 372 40.05376344 373 39.94638070 374 40.10695187 375 40.26666667 376 40.15957447 377 40.05305040 378 40.21164021 379 40.10554090 380 40.00000000 381 40.15748031 382 40.31413613 383 40.46997389 384 40.36458333 385 40.51948052 386 40.41450777 387 40.31007752 388 40.20618557 389 40.35989717 390 40.25641026 391 40.40920716 392 40.56122449 393 40.45801527 394 40.35532995 395 40.50632911 396 40.65656566 397 40.80604534 398 40.70351759 399 40.60150376 400 40.50000000 401 40.64837905 402 40.54726368 403 40.44665012 404 40.59405941 405 40.74074074 406 40.64039409 407 40.54054054 408 40.44117647 409 40.58679707 410 40.48780488 411 40.38929440 412 40.29126214 413 40.19370460 414 40.33816425 415 40.48192771 416 40.38461538 417 40.28776978 418 40.19138756 419 40.33412888 420 40.47619048 421 40.38004751 422 40.28436019 423 40.18912530 424 40.33018868 425 40.23529412 426 40.37558685 427 40.28103044 428 40.18691589 429 40.32634033 430 40.23255814 431 40.37122970 432 40.50925926 433 40.41570439 434 40.32258065 435 40.22988506 436 40.13761468 437 40.04576659 438 40.18264840 439 40.31890661 440 40.22727273 441 40.13605442 442 40.04524887 443 39.95485327 444 40.09009009 445 40.00000000 446 39.91031390 447 40.04474273 448 39.95535714 449 39.86636971 450 40.00000000 451 40.13303769 452 40.26548673 453 40.17660044 454 40.08810573 455 40.00000000 456 40.13157895 457 40.26258206 458 40.17467249 459 40.08714597 460 40.21739130 461 40.34707158 462 40.47619048 463 40.60475162 464 40.73275862 465 40.64516129 466 40.55793991 467 40.68522484 468 40.81196581 469 40.72494670 470 40.63829787 471 40.55201699 472 40.46610169 473 40.38054968 474 40.29535865 475 40.21052632 476 40.33613445 477 40.25157233 478 40.37656904 479 40.29227557 480 40.20833333 481 40.12474012 482 40.04149378 483 39.95859213 484 39.87603306 485 39.79381443 486 39.71193416 487 39.83572895 488 39.75409836 489 39.67280164 490 39.79591837 491 39.71486762 492 39.63414634 493 39.55375254 494 39.67611336 495 39.79797980 496 39.91935484 497 39.83903421 498 39.75903614 499 39.87975952 500 39.80000000 501 39.72055888 502 39.64143426 503 39.76143141 504 39.88095238 505 39.80198020 506 39.92094862 507 39.84220907 508 39.76377953 509 39.68565815 510 39.60784314 511 39.53033268 512 39.45312500 513 39.37621832 514 39.29961089 515 39.41747573 516 39.53488372 517 39.65183752 518 39.76833977 519 39.69171484 520 39.61538462 521 39.53934741 522 39.65517241 523 39.57934990 524 39.69465649 525 39.61904762 526 39.54372624 527 39.65844402 528 39.77272727 529 39.88657845 530 40.00000000 531 39.92467043 532 39.84962406 533 39.77485929 534 39.88764045 535 40.00000000 536 39.92537313 537 40.03724395 538 40.14869888 539 40.07421150 540 40.00000000 541 39.92606285 542 40.03690037 543 39.96316759 544 39.88970588 545 39.81651376 546 39.74358974 547 39.85374771 548 39.96350365 549 39.89071038 550 39.81818182 551 39.92740472 552 40.03623188 553 39.96383363 554 40.07220217 555 40.00000000 556 40.10791367 557 40.21543986 558 40.32258065 559 40.42933810 560 40.35714286 561 40.28520499 562 40.39145907 563 40.49733570 564 40.42553191 565 40.53097345 566 40.63604240 567 40.56437390 568 40.49295775 569 40.42179262 570 40.52631579 571 40.45534151 572 40.55944056 573 40.48865620 574 40.41811847 575 40.34782609 576 40.45138889 577 40.55459272 578 40.65743945 579 40.58721934 580 40.68965517 581 40.79173838 582 40.72164948 583 40.82332762 584 40.92465753 585 41.02564103 586 41.12627986 587 41.05621806 588 40.98639456 589 40.91680815 590 40.84745763 591 40.77834179 592 40.87837838 593 40.97807757 594 41.07744108 595 41.00840336 596 41.10738255 597 41.03852596 598 41.13712375 599 41.23539232 600 41.16666667 601 41.09816972 602 41.02990033 603 41.12769486 604 41.05960265 605 40.99173554 606 40.92409241 607 41.02141680 608 40.95394737 609 40.88669951 610 40.81967213 611 40.91653028 612 40.84967320 613 40.78303426 614 40.87947883 615 40.81300813 616 40.90909091 617 41.00486224 618 41.10032362 619 41.03392569 620 40.96774194 621 40.90177134 622 40.83601286 623 40.77046549 624 40.70512821 625 40.64000000 626 40.73482428 627 40.66985646 628 40.60509554 629 40.69952305 630 40.63492063 631 40.57052298 632 40.66455696 633 40.60031596 634 40.53627760 635 40.62992126 636 40.56603774 637 40.50235479 638 40.59561129 639 40.68857590 640 40.78125000 641 40.71762871 642 40.80996885 643 40.74650078 644 40.83850932 645 40.77519380 646 40.71207430 647 40.64914992 648 40.58641975 649 40.52388290 650 40.61538462 651 40.55299539 652 40.64417178 653 40.58192956 654 40.67278287 655 40.61068702 656 40.70121951 657 40.79147641 658 40.88145897 659 40.81942337 660 40.90909091 661 40.84720121 662 40.78549849 663 40.72398190 664 40.66265060 665 40.75187970 666 40.69069069 667 40.77961019 668 40.86826347 669 40.95665172 670 40.89552239 671 40.83457526 672 40.77380952 673 40.71322437 674 40.65281899 675 40.59259259 676 40.53254438 677 40.47267356 678 40.56047198 679 40.50073638 680 40.58823529 681 40.67547724 682 40.61583578 683 40.70278184 684 40.78947368 685 40.72992701 686 40.67055394 687 40.75691412 688 40.69767442 689 40.63860668 690 40.57971014 691 40.66570188 692 40.60693642 693 40.54834055 694 40.48991354 695 40.43165468 696 40.37356322 697 40.31563845 698 40.25787966 699 40.34334764 700 40.42857143 701 40.51355207 702 40.59829060 703 40.68278805 704 40.62500000 705 40.56737589 706 40.65155807 707 40.59405941 708 40.53672316 709 40.62059238 710 40.56338028 711 40.64697609 712 40.73033708 713 40.81346424 714 40.89635854 715 40.83916084 716 40.78212291 717 40.72524407 718 40.66852368 719 40.75104312 720 40.69444444 721 40.63800277 722 40.72022161 723 40.80221300 724 40.74585635 725 40.82758621 726 40.90909091 727 40.99037139 728 41.07142857 729 41.01508916 730 40.95890411 731 40.90287278 732 40.84699454 733 40.92769441 734 41.00817439 735 40.95238095 736 40.89673913 737 40.97693351 738 41.05691057 739 41.13667118 740 41.21621622 741 41.16059379 742 41.23989218 743 41.31897712 744 41.39784946 745 41.34228188 746 41.28686327 747 41.23159304 748 41.17647059 749 41.12149533 750 41.20000000 Final result: 41.2000 +/- 1.7984 Random chance: 25.0000 +/- 1.5822