common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 73.91304348 24 75.00000000 25 72.00000000 26 69.23076923 27 70.37037037 28 67.85714286 29 68.96551724 30 70.00000000 31 67.74193548 32 68.75000000 33 66.66666667 34 67.64705882 35 68.57142857 36 66.66666667 37 67.56756757 38 68.42105263 39 69.23076923 40 70.00000000 41 68.29268293 42 66.66666667 43 65.11627907 44 63.63636364 45 64.44444444 46 65.21739130 47 65.95744681 48 66.66666667 49 65.30612245 50 64.00000000 51 62.74509804 52 63.46153846 53 64.15094340 54 62.96296296 55 61.81818182 56 62.50000000 57 63.15789474 58 63.79310345 59 64.40677966 60 65.00000000 61 65.57377049 62 66.12903226 63 66.66666667 64 67.18750000 65 66.15384615 66 66.66666667 67 67.16417910 68 67.64705882 69 68.11594203 70 68.57142857 71 69.01408451 72 69.44444444 73 69.86301370 74 70.27027027 75 70.66666667 76 71.05263158 77 71.42857143 78 71.79487179 79 72.15189873 80 72.50000000 81 72.83950617 82 73.17073171 83 72.28915663 84 72.61904762 85 72.94117647 86 73.25581395 87 73.56321839 88 73.86363636 89 74.15730337 90 74.44444444 91 74.72527473 92 75.00000000 93 74.19354839 94 73.40425532 95 73.68421053 96 73.95833333 97 73.19587629 98 73.46938776 99 73.73737374 100 74.00000000 101 74.25742574 102 74.50980392 103 73.78640777 104 73.07692308 105 73.33333333 106 72.64150943 107 72.89719626 108 72.22222222 109 71.55963303 110 71.81818182 111 72.07207207 112 72.32142857 113 71.68141593 114 71.92982456 115 72.17391304 116 71.55172414 117 70.94017094 118 71.18644068 119 70.58823529 120 70.83333333 121 70.24793388 122 70.49180328 123 70.73170732 124 70.96774194 125 71.20000000 126 71.42857143 127 71.65354331 128 71.87500000 129 71.31782946 130 71.53846154 131 71.75572519 132 71.21212121 133 71.42857143 134 70.89552239 135 70.37037037 136 70.58823529 137 70.80291971 138 70.28985507 139 70.50359712 140 70.00000000 141 69.50354610 142 69.71830986 143 69.93006993 144 70.13888889 145 70.34482759 146 69.86301370 147 70.06802721 148 69.59459459 149 69.79865772 150 70.00000000 151 70.19867550 152 70.39473684 153 70.58823529 154 70.77922078 155 70.32258065 156 70.51282051 157 70.06369427 158 70.25316456 159 70.44025157 160 70.62500000 161 70.80745342 162 70.98765432 163 70.55214724 164 70.73170732 165 70.90909091 166 71.08433735 167 71.25748503 168 71.42857143 169 71.00591716 170 70.58823529 171 70.76023392 172 70.34883721 173 70.52023121 174 70.68965517 175 70.28571429 176 70.45454545 177 70.62146893 178 70.22471910 179 70.39106145 180 70.55555556 181 70.16574586 182 70.32967033 183 69.94535519 184 70.10869565 185 70.27027027 186 70.43010753 187 70.58823529 188 70.21276596 189 70.37037037 190 70.52631579 191 70.68062827 192 70.31250000 193 69.94818653 194 70.10309278 195 70.25641026 196 70.40816327 197 70.55837563 198 70.70707071 199 70.35175879 200 70.50000000 201 70.64676617 202 70.79207921 203 70.93596059 204 71.07843137 205 71.21951220 206 70.87378641 207 71.01449275 208 71.15384615 209 71.29186603 210 71.42857143 211 71.56398104 212 71.22641509 213 71.36150235 214 71.49532710 215 71.62790698 216 71.29629630 217 71.42857143 218 71.55963303 219 71.68949772 220 71.36363636 221 71.49321267 222 71.62162162 223 71.30044843 224 71.42857143 225 71.11111111 226 70.79646018 227 70.92511013 228 71.05263158 229 71.17903930 230 71.30434783 231 70.99567100 232 70.68965517 233 70.38626609 234 70.51282051 235 70.21276596 236 70.33898305 237 70.04219409 238 69.74789916 239 69.87447699 240 70.00000000 241 70.12448133 242 69.83471074 243 69.95884774 244 69.67213115 245 69.79591837 246 69.91869919 247 70.04048583 248 70.16129032 249 70.28112450 250 70.40000000 251 70.51792829 252 70.63492063 253 70.75098814 254 70.86614173 255 70.98039216 256 71.09375000 257 70.81712062 258 70.93023256 259 71.04247104 260 71.15384615 261 71.26436782 262 71.37404580 263 71.48288973 264 71.59090909 265 71.69811321 266 71.42857143 267 71.53558052 268 71.64179104 269 71.74721190 270 71.85185185 271 71.95571956 272 72.05882353 273 71.79487179 274 71.53284672 275 71.27272727 276 71.37681159 277 71.48014440 278 71.58273381 279 71.68458781 280 71.78571429 281 71.88612100 282 71.98581560 283 71.73144876 284 71.83098592 285 71.92982456 286 72.02797203 287 71.77700348 288 71.87500000 289 71.62629758 290 71.72413793 291 71.82130584 292 71.91780822 293 72.01365188 294 71.76870748 295 71.86440678 296 71.95945946 297 72.05387205 298 72.14765101 299 72.24080268 300 72.33333333 301 72.42524917 302 72.51655629 303 72.60726073 304 72.69736842 305 72.78688525 306 72.54901961 307 72.63843648 308 72.72727273 309 72.81553398 310 72.58064516 311 72.66881029 312 72.75641026 313 72.84345048 314 72.92993631 315 73.01587302 316 73.10126582 317 73.18611987 318 73.27044025 319 73.04075235 320 72.81250000 321 72.89719626 322 72.98136646 323 73.06501548 324 73.14814815 325 73.23076923 326 73.31288344 327 73.08868502 328 73.17073171 329 73.25227964 330 73.03030303 331 72.80966767 332 72.89156627 333 72.97297297 334 73.05389222 335 72.83582090 336 72.61904762 337 72.40356083 338 72.18934911 339 72.27138643 340 72.35294118 341 72.43401760 342 72.51461988 343 72.59475219 344 72.67441860 345 72.46376812 346 72.54335260 347 72.62247839 348 72.41379310 349 72.49283668 350 72.57142857 351 72.64957265 352 72.72727273 353 72.80453258 354 72.59887006 355 72.39436620 356 72.47191011 357 72.54901961 358 72.62569832 359 72.42339833 360 72.50000000 361 72.57617729 362 72.65193370 363 72.72727273 364 72.80219780 365 72.87671233 366 72.67759563 367 72.75204360 368 72.82608696 369 72.89972900 370 72.97297297 371 72.77628032 372 72.58064516 373 72.65415550 374 72.72727273 375 72.53333333 376 72.60638298 377 72.67904509 378 72.75132275 379 72.55936675 380 72.63157895 381 72.44094488 382 72.51308901 383 72.32375979 384 72.39583333 385 72.46753247 386 72.27979275 387 72.35142119 388 72.16494845 389 71.97943445 390 71.79487179 391 71.61125320 392 71.68367347 393 71.75572519 394 71.57360406 395 71.64556962 396 71.46464646 397 71.53652393 398 71.35678392 399 71.42857143 400 71.50000000 401 71.57107232 402 71.64179104 403 71.71215881 404 71.78217822 405 71.85185185 406 71.92118227 407 71.99017199 408 72.05882353 409 71.88264059 410 71.95121951 411 72.01946472 412 72.08737864 413 72.15496368 414 72.22222222 415 72.04819277 416 72.11538462 417 71.94244604 418 72.00956938 419 71.83770883 420 71.90476190 421 71.97149644 422 72.03791469 423 72.10401891 424 72.16981132 425 72.00000000 426 71.83098592 427 71.66276347 428 71.49532710 429 71.56177156 430 71.39534884 431 71.46171694 432 71.52777778 433 71.36258661 434 71.42857143 435 71.49425287 436 71.55963303 437 71.62471396 438 71.46118721 439 71.52619590 440 71.59090909 441 71.65532880 442 71.49321267 443 71.55756208 444 71.62162162 445 71.68539326 446 71.74887892 447 71.81208054 448 71.87500000 449 71.93763920 450 72.00000000 451 72.06208426 452 72.12389381 453 72.18543046 454 72.02643172 455 71.86813187 456 71.92982456 457 71.99124726 458 72.05240175 459 72.11328976 460 71.95652174 461 72.01735358 462 72.07792208 463 71.92224622 464 71.76724138 465 71.82795699 466 71.88841202 467 71.94860814 468 72.00854701 469 71.85501066 470 71.91489362 471 71.97452229 472 72.03389831 473 71.88160677 474 71.94092827 475 71.78947368 476 71.84873950 477 71.90775681 478 71.96652720 479 72.02505219 480 71.87500000 481 71.93347193 482 71.99170124 483 72.04968944 484 72.10743802 485 72.16494845 486 72.22222222 487 72.07392197 488 72.13114754 489 72.18813906 490 72.24489796 491 72.30142566 492 72.35772358 493 72.21095335 494 72.26720648 495 72.32323232 496 72.17741935 497 72.03219316 498 72.08835341 499 72.14428858 500 72.20000000 501 72.25548902 502 72.31075697 503 72.36580517 504 72.22222222 505 72.27722772 506 72.13438735 507 72.18934911 508 72.24409449 509 72.10216110 510 72.15686275 511 72.01565558 512 72.07031250 513 72.12475634 514 72.17898833 515 72.23300971 516 72.09302326 517 72.14700193 518 72.20077220 519 72.25433526 520 72.11538462 521 72.16890595 522 72.22222222 523 72.08413002 524 71.94656489 525 71.80952381 526 71.67300380 527 71.53700190 528 71.40151515 529 71.26654064 530 71.32075472 531 71.37476460 532 71.42857143 533 71.29455910 534 71.16104869 535 71.02803738 536 71.08208955 537 71.13594041 538 71.18959108 539 71.24304267 540 71.11111111 541 71.16451017 542 71.21771218 543 71.08655617 544 71.13970588 545 71.19266055 546 71.24542125 547 71.29798903 548 71.35036496 549 71.40255009 550 71.45454545 551 71.32486388 552 71.19565217 553 71.24773960 554 71.29963899 555 71.35135135 556 71.22302158 557 71.27468582 558 71.14695341 559 71.19856887 560 71.25000000 561 71.30124777 562 71.35231317 563 71.40319716 564 71.45390071 565 71.32743363 566 71.37809187 567 71.42857143 568 71.47887324 569 71.35325132 570 71.22807018 571 71.27845884 572 71.15384615 573 71.20418848 574 71.25435540 575 71.30434783 576 71.35416667 577 71.40381282 578 71.28027682 579 71.32987910 580 71.37931034 581 71.25645439 582 71.13402062 583 71.01200686 584 71.06164384 585 71.11111111 586 71.16040956 587 71.20954003 588 71.25850340 589 71.30730051 590 71.35593220 591 71.23519459 592 71.28378378 593 71.33220911 594 71.38047138 595 71.26050420 596 71.14093960 597 71.18927973 598 71.23745819 599 71.11853088 600 71.16666667 601 71.21464226 602 71.09634551 603 70.97844113 604 71.02649007 605 71.07438017 606 71.12211221 607 71.16968699 608 71.21710526 609 71.10016420 610 71.14754098 611 71.03109656 612 70.91503268 613 70.79934747 614 70.84690554 615 70.73170732 616 70.61688312 617 70.66450567 618 70.71197411 619 70.59773829 620 70.64516129 621 70.69243156 622 70.73954984 623 70.78651685 624 70.83333333 625 70.72000000 626 70.76677316 627 70.81339713 628 70.70063694 629 70.74721781 630 70.79365079 631 70.83993661 632 70.72784810 633 70.61611374 634 70.66246057 635 70.70866142 636 70.75471698 637 70.64364207 638 70.68965517 639 70.73552426 640 70.78125000 641 70.82683307 642 70.71651090 643 70.60653188 644 70.65217391 645 70.54263566 646 70.58823529 647 70.47913447 648 70.52469136 649 70.41602465 650 70.46153846 651 70.50691244 652 70.39877301 653 70.44410413 654 70.33639144 655 70.22900763 656 70.12195122 657 70.16742770 658 70.21276596 659 70.10622155 660 70.15151515 661 70.04538578 662 70.09063444 663 70.13574661 664 70.18072289 665 70.22556391 666 70.27027027 667 70.31484258 668 70.20958084 669 70.25411061 670 70.29850746 671 70.34277198 672 70.38690476 673 70.43090639 674 70.47477745 675 70.51851852 676 70.56213018 677 70.60561300 678 70.64896755 679 70.54491900 680 70.58823529 681 70.63142438 682 70.67448680 683 70.57101025 684 70.61403509 685 70.65693431 686 70.69970845 687 70.74235808 688 70.78488372 689 70.68214804 690 70.72463768 691 70.76700434 692 70.66473988 693 70.70707071 694 70.74927954 695 70.64748201 696 70.54597701 697 70.58823529 698 70.48710602 699 70.38626609 700 70.28571429 701 70.18544936 702 70.22792023 703 70.27027027 704 70.31250000 705 70.35460993 706 70.39660057 707 70.43847242 708 70.48022599 709 70.52186178 710 70.56338028 711 70.60478200 712 70.64606742 713 70.68723703 714 70.58823529 715 70.48951049 716 70.53072626 717 70.57182706 718 70.47353760 719 70.37552156 720 70.41666667 721 70.31900139 722 70.36011080 723 70.40110650 724 70.44198895 725 70.48275862 726 70.38567493 727 70.42640990 728 70.46703297 729 70.50754458 730 70.54794521 731 70.58823529 732 70.62841530 733 70.66848568 734 70.70844687 735 70.74829932 736 70.65217391 737 70.55630936 738 70.59620596 739 70.50067659 740 70.40540541 741 70.44534413 742 70.35040431 743 70.39030956 744 70.29569892 745 70.33557047 746 70.37533512 747 70.28112450 748 70.18716578 749 70.09345794 750 70.13333333 Final result: 70.1333 +/- 1.6723 Random chance: 25.0083 +/- 1.5824