common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 41.17647059 18 38.88888889 19 42.10526316 20 45.00000000 21 42.85714286 22 40.90909091 23 43.47826087 24 45.83333333 25 48.00000000 26 46.15384615 27 48.14814815 28 50.00000000 29 48.27586207 30 46.66666667 31 45.16129032 32 46.87500000 33 45.45454545 34 47.05882353 35 45.71428571 36 44.44444444 37 43.24324324 38 44.73684211 39 43.58974359 40 42.50000000 41 41.46341463 42 40.47619048 43 41.86046512 44 40.90909091 45 42.22222222 46 41.30434783 47 40.42553191 48 41.66666667 49 42.85714286 50 42.00000000 51 41.17647059 52 40.38461538 53 39.62264151 54 38.88888889 55 40.00000000 56 39.28571429 57 40.35087719 58 41.37931034 59 42.37288136 60 41.66666667 61 42.62295082 62 41.93548387 63 41.26984127 64 42.18750000 65 41.53846154 66 40.90909091 67 40.29850746 68 39.70588235 69 39.13043478 70 38.57142857 71 38.02816901 72 38.88888889 73 38.35616438 74 39.18918919 75 40.00000000 76 40.78947368 77 41.55844156 78 42.30769231 79 41.77215190 80 42.50000000 81 43.20987654 82 42.68292683 83 43.37349398 84 42.85714286 85 42.35294118 86 43.02325581 87 43.67816092 88 44.31818182 89 43.82022472 90 43.33333333 91 42.85714286 92 42.39130435 93 41.93548387 94 42.55319149 95 42.10526316 96 42.70833333 97 42.26804124 98 42.85714286 99 43.43434343 100 43.00000000 101 43.56435644 102 43.13725490 103 42.71844660 104 43.26923077 105 43.80952381 106 43.39622642 107 42.99065421 108 42.59259259 109 42.20183486 110 42.72727273 111 42.34234234 112 42.85714286 113 42.47787611 114 42.98245614 115 43.47826087 116 43.10344828 117 42.73504274 118 42.37288136 119 42.01680672 120 41.66666667 121 42.14876033 122 41.80327869 123 41.46341463 124 41.93548387 125 42.40000000 126 42.06349206 127 41.73228346 128 42.18750000 129 41.86046512 130 42.30769231 131 42.74809160 132 43.18181818 133 42.85714286 134 42.53731343 135 42.22222222 136 42.64705882 137 43.06569343 138 42.75362319 139 43.16546763 140 43.57142857 141 43.97163121 142 44.36619718 143 44.75524476 144 44.44444444 145 44.13793103 146 44.52054795 147 44.21768707 148 44.59459459 149 44.96644295 150 44.66666667 151 44.37086093 152 44.73684211 153 44.44444444 154 44.15584416 155 43.87096774 156 43.58974359 157 43.94904459 158 44.30379747 159 44.02515723 160 44.37500000 161 44.09937888 162 43.82716049 163 43.55828221 164 43.29268293 165 43.03030303 166 42.77108434 167 43.11377246 168 42.85714286 169 43.19526627 170 42.94117647 171 43.27485380 172 43.02325581 173 42.77456647 174 42.52873563 175 42.28571429 176 42.61363636 177 42.37288136 178 42.69662921 179 42.45810056 180 42.22222222 181 41.98895028 182 41.75824176 183 41.53005464 184 41.30434783 185 41.62162162 186 41.93548387 187 42.24598930 188 42.02127660 189 42.32804233 190 42.10526316 191 42.40837696 192 42.18750000 193 41.96891192 194 41.75257732 195 42.05128205 196 42.34693878 197 42.13197970 198 42.42424242 199 42.21105528 200 42.00000000 201 41.79104478 202 41.58415842 203 41.37931034 204 41.17647059 205 41.46341463 206 41.26213592 207 41.54589372 208 41.34615385 209 41.14832536 210 40.95238095 211 40.75829384 212 40.56603774 213 40.37558685 214 40.18691589 215 40.46511628 216 40.27777778 217 40.09216590 218 40.36697248 219 40.63926941 220 40.45454545 221 40.27149321 222 40.09009009 223 40.35874439 224 40.62500000 225 40.88888889 226 41.15044248 227 40.96916300 228 41.22807018 229 41.04803493 230 40.86956522 231 40.69264069 232 40.94827586 233 40.77253219 234 41.02564103 235 40.85106383 236 40.67796610 237 40.50632911 238 40.33613445 239 40.16736402 240 40.41666667 241 40.24896266 242 40.49586777 243 40.74074074 244 40.57377049 245 40.81632653 246 40.65040650 247 40.48582996 248 40.72580645 249 40.56224900 250 40.40000000 251 40.23904382 252 40.07936508 253 40.31620553 254 40.15748031 255 40.00000000 256 40.23437500 257 40.46692607 258 40.31007752 259 40.15444015 260 40.38461538 261 40.61302682 262 40.45801527 263 40.30418251 264 40.53030303 265 40.37735849 266 40.22556391 267 40.07490637 268 39.92537313 269 40.14869888 270 40.37037037 271 40.22140221 272 40.44117647 273 40.29304029 274 40.14598540 275 40.00000000 276 40.21739130 277 40.43321300 278 40.64748201 279 40.50179211 280 40.71428571 281 40.56939502 282 40.42553191 283 40.28268551 284 40.14084507 285 40.00000000 286 39.86013986 287 40.06968641 288 39.93055556 289 39.79238754 290 40.00000000 291 40.20618557 292 40.41095890 293 40.61433447 294 40.47619048 295 40.33898305 296 40.20270270 297 40.06734007 298 40.26845638 299 40.46822742 300 40.66666667 301 40.86378738 302 41.05960265 303 40.92409241 304 40.78947368 305 40.98360656 306 40.84967320 307 41.04234528 308 41.23376623 309 41.10032362 310 41.29032258 311 41.47909968 312 41.66666667 313 41.85303514 314 42.03821656 315 41.90476190 316 41.77215190 317 41.64037855 318 41.50943396 319 41.69278997 320 41.56250000 321 41.43302181 322 41.61490683 323 41.48606811 324 41.35802469 325 41.53846154 326 41.41104294 327 41.59021407 328 41.76829268 329 41.94528875 330 41.81818182 331 41.69184290 332 41.86746988 333 42.04204204 334 41.91616766 335 42.08955224 336 42.26190476 337 42.13649852 338 42.01183432 339 41.88790560 340 42.05882353 341 42.22873900 342 42.39766082 343 42.27405248 344 42.44186047 345 42.31884058 346 42.48554913 347 42.65129683 348 42.52873563 349 42.40687679 350 42.57142857 351 42.45014245 352 42.32954545 353 42.49291785 354 42.37288136 355 42.25352113 356 42.41573034 357 42.57703081 358 42.45810056 359 42.33983287 360 42.50000000 361 42.38227147 362 42.54143646 363 42.42424242 364 42.58241758 365 42.46575342 366 42.62295082 367 42.50681199 368 42.39130435 369 42.27642276 370 42.43243243 371 42.58760108 372 42.47311828 373 42.35924933 374 42.51336898 375 42.66666667 376 42.55319149 377 42.44031830 378 42.59259259 379 42.48021108 380 42.36842105 381 42.51968504 382 42.67015707 383 42.81984334 384 42.70833333 385 42.85714286 386 42.74611399 387 42.63565891 388 42.52577320 389 42.67352185 390 42.56410256 391 42.71099744 392 42.85714286 393 42.74809160 394 42.63959391 395 42.78481013 396 42.92929293 397 42.82115869 398 42.71356784 399 42.60651629 400 42.50000000 401 42.64339152 402 42.53731343 403 42.43176179 404 42.57425743 405 42.71604938 406 42.61083744 407 42.50614251 408 42.40196078 409 42.54278729 410 42.43902439 411 42.33576642 412 42.23300971 413 42.13075061 414 42.27053140 415 42.40963855 416 42.30769231 417 42.20623501 418 42.10526316 419 42.24343675 420 42.38095238 421 42.28028504 422 42.18009479 423 42.08037825 424 42.21698113 425 42.11764706 426 42.25352113 427 42.15456674 428 42.05607477 429 42.19114219 430 42.09302326 431 42.22737819 432 42.12962963 433 42.03233256 434 41.93548387 435 41.83908046 436 41.74311927 437 41.64759725 438 41.78082192 439 41.91343964 440 41.81818182 441 41.72335601 442 41.62895928 443 41.53498871 444 41.44144144 445 41.34831461 446 41.25560538 447 41.38702461 448 41.29464286 449 41.20267261 450 41.33333333 451 41.46341463 452 41.59292035 453 41.50110375 454 41.40969163 455 41.31868132 456 41.44736842 457 41.57549234 458 41.48471616 459 41.39433551 460 41.52173913 461 41.64859002 462 41.77489177 463 41.90064795 464 42.02586207 465 41.93548387 466 42.06008584 467 42.18415418 468 42.30769231 469 42.21748401 470 42.12765957 471 42.03821656 472 41.94915254 473 41.86046512 474 41.77215190 475 41.68421053 476 41.80672269 477 41.71907757 478 41.84100418 479 41.96242171 480 41.87500000 481 41.78794179 482 41.70124481 483 41.61490683 484 41.52892562 485 41.44329897 486 41.35802469 487 41.47843943 488 41.39344262 489 41.30879346 490 41.42857143 491 41.34419552 492 41.26016260 493 41.17647059 494 41.29554656 495 41.41414141 496 41.53225806 497 41.44869215 498 41.36546185 499 41.48296593 500 41.40000000 501 41.31736527 502 41.23505976 503 41.35188867 504 41.46825397 505 41.38613861 506 41.50197628 507 41.42011834 508 41.33858268 509 41.25736739 510 41.17647059 511 41.09589041 512 41.01562500 513 40.93567251 514 40.85603113 515 40.97087379 516 41.08527132 517 41.19922631 518 41.31274131 519 41.23314066 520 41.34615385 521 41.26679463 522 41.37931034 523 41.30019120 524 41.41221374 525 41.33333333 526 41.25475285 527 41.36622391 528 41.47727273 529 41.58790170 530 41.69811321 531 41.61958569 532 41.54135338 533 41.46341463 534 41.57303371 535 41.68224299 536 41.60447761 537 41.71322160 538 41.82156134 539 41.74397032 540 41.66666667 541 41.58964880 542 41.69741697 543 41.62062615 544 41.72794118 545 41.65137615 546 41.57509158 547 41.68190128 548 41.78832117 549 41.71220401 550 41.63636364 551 41.74228675 552 41.84782609 553 41.77215190 554 41.87725632 555 41.80180180 556 41.90647482 557 41.83123878 558 41.93548387 559 42.03935599 560 41.96428571 561 41.88948307 562 41.99288256 563 42.09591474 564 42.02127660 565 42.12389381 566 42.04946996 567 41.97530864 568 41.90140845 569 41.82776801 570 41.92982456 571 41.85639229 572 41.95804196 573 41.88481675 574 41.81184669 575 41.73913043 576 41.84027778 577 41.94107452 578 42.04152249 579 41.96891192 580 42.06896552 581 42.16867470 582 42.09621993 583 42.19554031 584 42.29452055 585 42.39316239 586 42.49146758 587 42.41908007 588 42.34693878 589 42.27504244 590 42.20338983 591 42.13197970 592 42.22972973 593 42.32715008 594 42.42424242 595 42.35294118 596 42.44966443 597 42.37855946 598 42.30769231 599 42.40400668 600 42.33333333 601 42.26289517 602 42.19269103 603 42.28855721 604 42.21854305 605 42.14876033 606 42.07920792 607 42.17462932 608 42.10526316 609 42.03612479 610 41.96721311 611 42.06219313 612 41.99346405 613 42.08809135 614 42.18241042 615 42.11382114 616 42.20779221 617 42.13938412 618 42.23300971 619 42.16478191 620 42.09677419 621 42.02898551 622 41.96141479 623 41.89406100 624 41.82692308 625 41.76000000 626 41.85303514 627 41.78628389 628 41.71974522 629 41.81240064 630 41.74603175 631 41.67987322 632 41.77215190 633 41.70616114 634 41.64037855 635 41.73228346 636 41.66666667 637 41.60125589 638 41.69278997 639 41.78403756 640 41.87500000 641 41.80967239 642 41.90031153 643 41.83514774 644 41.92546584 645 41.86046512 646 41.79566563 647 41.73106646 648 41.66666667 649 41.60246533 650 41.69230769 651 41.62826421 652 41.71779141 653 41.65390505 654 41.74311927 655 41.67938931 656 41.76829268 657 41.85692542 658 41.94528875 659 41.88163885 660 41.81818182 661 41.75491679 662 41.69184290 663 41.62895928 664 41.56626506 665 41.50375940 666 41.44144144 667 41.52923538 668 41.61676647 669 41.70403587 670 41.79104478 671 41.72876304 672 41.66666667 673 41.60475483 674 41.54302671 675 41.48148148 676 41.42011834 677 41.35893648 678 41.44542773 679 41.38438881 680 41.47058824 681 41.55653451 682 41.49560117 683 41.43484627 684 41.52046784 685 41.45985401 686 41.39941691 687 41.33915575 688 41.27906977 689 41.21915820 690 41.15942029 691 41.24457308 692 41.18497110 693 41.12554113 694 41.06628242 695 41.00719424 696 41.09195402 697 41.03299857 698 40.97421203 699 41.05865522 700 41.14285714 701 41.22681883 702 41.31054131 703 41.39402560 704 41.33522727 705 41.27659574 706 41.35977337 707 41.30127298 708 41.24293785 709 41.32581100 710 41.26760563 711 41.35021097 712 41.43258427 713 41.51472651 714 41.59663866 715 41.53846154 716 41.48044693 717 41.42259414 718 41.36490251 719 41.44645341 720 41.38888889 721 41.33148405 722 41.41274238 723 41.49377593 724 41.43646409 725 41.51724138 726 41.59779614 727 41.67812930 728 41.75824176 729 41.70096022 730 41.64383562 731 41.58686731 732 41.53005464 733 41.60982265 734 41.68937330 735 41.63265306 736 41.57608696 737 41.51967436 738 41.59891599 739 41.67794317 740 41.62162162 741 41.56545209 742 41.64420485 743 41.72274563 744 41.80107527 745 41.74496644 746 41.68900804 747 41.63319946 748 41.57754011 749 41.52202937 750 41.60000000 Final result: 41.6000 +/- 1.8010 Random chance: 25.0000 +/- 1.5822