common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 28.57142857 22 27.27272727 23 30.43478261 24 33.33333333 25 32.00000000 26 30.76923077 27 29.62962963 28 28.57142857 29 31.03448276 30 30.00000000 31 29.03225806 32 31.25000000 33 33.33333333 34 32.35294118 35 31.42857143 36 30.55555556 37 32.43243243 38 31.57894737 39 33.33333333 40 32.50000000 41 31.70731707 42 30.95238095 43 30.23255814 44 31.81818182 45 33.33333333 46 32.60869565 47 34.04255319 48 35.41666667 49 34.69387755 50 36.00000000 51 37.25490196 52 36.53846154 53 35.84905660 54 35.18518519 55 36.36363636 56 35.71428571 57 35.08771930 58 34.48275862 59 33.89830508 60 35.00000000 61 34.42622951 62 35.48387097 63 34.92063492 64 35.93750000 65 35.38461538 66 36.36363636 67 35.82089552 68 35.29411765 69 34.78260870 70 34.28571429 71 33.80281690 72 34.72222222 73 34.24657534 74 33.78378378 75 33.33333333 76 34.21052632 77 33.76623377 78 33.33333333 79 32.91139241 80 33.75000000 81 33.33333333 82 34.14634146 83 33.73493976 84 33.33333333 85 32.94117647 86 32.55813953 87 32.18390805 88 31.81818182 89 32.58426966 90 32.22222222 91 31.86813187 92 31.52173913 93 31.18279570 94 30.85106383 95 30.52631579 96 31.25000000 97 31.95876289 98 31.63265306 99 32.32323232 100 33.00000000 101 33.66336634 102 33.33333333 103 33.98058252 104 33.65384615 105 33.33333333 106 33.01886792 107 33.64485981 108 34.25925926 109 34.86238532 110 35.45454545 111 35.13513514 112 35.71428571 113 35.39823009 114 35.96491228 115 35.65217391 116 36.20689655 117 36.75213675 118 36.44067797 119 36.13445378 120 35.83333333 121 36.36363636 122 36.06557377 123 36.58536585 124 36.29032258 125 36.80000000 126 36.50793651 127 36.22047244 128 35.93750000 129 36.43410853 130 36.92307692 131 36.64122137 132 36.36363636 133 36.09022556 134 35.82089552 135 35.55555556 136 35.29411765 137 35.76642336 138 36.23188406 139 36.69064748 140 36.42857143 141 36.87943262 142 36.61971831 143 36.36363636 144 36.80555556 145 37.24137931 146 36.98630137 147 36.73469388 148 36.48648649 149 36.24161074 150 36.00000000 151 36.42384106 152 36.18421053 153 35.94771242 154 35.71428571 155 35.48387097 156 35.89743590 157 36.30573248 158 36.70886076 159 36.47798742 160 36.25000000 161 36.64596273 162 36.41975309 163 36.80981595 164 36.58536585 165 36.96969697 166 37.34939759 167 37.12574850 168 36.90476190 169 36.68639053 170 36.47058824 171 36.25730994 172 36.04651163 173 35.83815029 174 36.20689655 175 36.00000000 176 35.79545455 177 35.59322034 178 35.39325843 179 35.19553073 180 35.00000000 181 34.80662983 182 34.61538462 183 34.42622951 184 34.23913043 185 34.59459459 186 34.94623656 187 34.75935829 188 34.57446809 189 34.39153439 190 34.21052632 191 34.03141361 192 33.85416667 193 33.67875648 194 34.02061856 195 33.84615385 196 33.67346939 197 34.01015228 198 34.34343434 199 34.17085427 200 34.50000000 201 34.32835821 202 34.65346535 203 34.48275862 204 34.31372549 205 34.63414634 206 34.46601942 207 34.78260870 208 34.61538462 209 34.44976077 210 34.28571429 211 34.12322275 212 33.96226415 213 33.80281690 214 33.64485981 215 33.48837209 216 33.79629630 217 33.64055300 218 33.48623853 219 33.33333333 220 33.18181818 221 33.48416290 222 33.33333333 223 33.18385650 224 33.48214286 225 33.77777778 226 33.62831858 227 33.92070485 228 33.77192982 229 33.62445415 230 33.47826087 231 33.33333333 232 33.62068966 233 33.47639485 234 33.33333333 235 33.19148936 236 33.47457627 237 33.33333333 238 33.61344538 239 33.47280335 240 33.33333333 241 33.19502075 242 33.05785124 243 32.92181070 244 32.78688525 245 32.65306122 246 32.52032520 247 32.79352227 248 33.06451613 249 33.33333333 250 33.20000000 251 33.46613546 252 33.33333333 253 33.20158103 254 33.46456693 255 33.72549020 256 33.59375000 257 33.46303502 258 33.33333333 259 33.20463320 260 33.07692308 261 33.33333333 262 33.20610687 263 33.07984791 264 33.33333333 265 33.58490566 266 33.45864662 267 33.70786517 268 33.58208955 269 33.45724907 270 33.33333333 271 33.21033210 272 33.08823529 273 32.96703297 274 33.21167883 275 33.09090909 276 32.97101449 277 32.85198556 278 32.73381295 279 32.97491039 280 33.21428571 281 33.45195730 282 33.33333333 283 33.21554770 284 33.45070423 285 33.33333333 286 33.56643357 287 33.79790941 288 34.02777778 289 33.91003460 290 33.79310345 291 34.02061856 292 33.90410959 293 34.12969283 294 34.01360544 295 34.23728814 296 34.45945946 297 34.34343434 298 34.22818792 299 34.11371237 300 34.00000000 301 33.88704319 302 33.77483444 303 33.99339934 304 33.88157895 305 33.77049180 306 33.66013072 307 33.55048860 308 33.76623377 309 33.65695793 310 33.54838710 311 33.44051447 312 33.65384615 313 33.54632588 314 33.43949045 315 33.65079365 316 33.54430380 317 33.43848580 318 33.33333333 319 33.22884013 320 33.12500000 321 33.02180685 322 32.91925466 323 33.12693498 324 33.02469136 325 32.92307692 326 32.82208589 327 33.02752294 328 33.23170732 329 33.13069909 330 33.33333333 331 33.23262840 332 33.43373494 333 33.33333333 334 33.23353293 335 33.43283582 336 33.63095238 337 33.53115727 338 33.43195266 339 33.62831858 340 33.52941176 341 33.72434018 342 33.91812865 343 33.81924198 344 34.01162791 345 34.20289855 346 34.10404624 347 34.29394813 348 34.19540230 349 34.09742120 350 34.00000000 351 33.90313390 352 33.80681818 353 33.71104816 354 33.61581921 355 33.80281690 356 33.70786517 357 33.89355742 358 33.79888268 359 33.70473538 360 33.61111111 361 33.79501385 362 33.70165746 363 33.60881543 364 33.51648352 365 33.42465753 366 33.33333333 367 33.51498638 368 33.42391304 369 33.33333333 370 33.24324324 371 33.15363881 372 33.33333333 373 33.24396783 374 33.42245989 375 33.33333333 376 33.24468085 377 33.15649867 378 33.06878307 379 32.98153034 380 33.15789474 381 33.07086614 382 33.24607330 383 33.15926893 384 33.07291667 385 32.98701299 386 32.90155440 387 32.81653747 388 32.98969072 389 32.90488432 390 32.82051282 391 32.73657289 392 32.65306122 393 32.56997455 394 32.74111675 395 32.65822785 396 32.57575758 397 32.74559194 398 32.66331658 399 32.58145363 400 32.50000000 401 32.66832918 402 32.83582090 403 33.00248139 404 32.92079208 405 32.83950617 406 32.75862069 407 32.67813268 408 32.59803922 409 32.51833741 410 32.43902439 411 32.36009732 412 32.52427184 413 32.68765133 414 32.85024155 415 33.01204819 416 32.93269231 417 32.85371703 418 32.77511962 419 32.69689737 420 32.85714286 421 33.01662708 422 32.93838863 423 32.86052009 424 32.78301887 425 32.70588235 426 32.62910798 427 32.55269321 428 32.47663551 429 32.40093240 430 32.55813953 431 32.48259861 432 32.63888889 433 32.79445727 434 32.94930876 435 32.87356322 436 32.79816514 437 32.72311213 438 32.64840183 439 32.80182232 440 32.72727273 441 32.65306122 442 32.57918552 443 32.50564334 444 32.65765766 445 32.58426966 446 32.51121076 447 32.43847875 448 32.36607143 449 32.29398664 450 32.22222222 451 32.15077605 452 32.07964602 453 32.22958057 454 32.15859031 455 32.08791209 456 32.01754386 457 31.94748359 458 31.87772926 459 31.80827887 460 31.73913043 461 31.67028200 462 31.60173160 463 31.74946004 464 31.89655172 465 32.04301075 466 31.97424893 467 31.90578158 468 31.83760684 469 31.76972281 470 31.70212766 471 31.84713376 472 31.77966102 473 31.71247357 474 31.85654008 475 32.00000000 476 31.93277311 477 32.07547170 478 32.00836820 479 31.94154489 480 31.87500000 481 31.80873181 482 31.95020747 483 31.88405797 484 32.02479339 485 32.16494845 486 32.09876543 487 32.03285421 488 31.96721311 489 32.10633947 490 32.24489796 491 32.38289206 492 32.52032520 493 32.45436105 494 32.38866397 495 32.32323232 496 32.25806452 497 32.19315895 498 32.12851406 499 32.26452906 500 32.20000000 501 32.13572854 502 32.07171315 503 32.00795229 504 32.14285714 505 32.07920792 506 32.01581028 507 32.14990138 508 32.28346457 509 32.41650295 510 32.35294118 511 32.28962818 512 32.22656250 513 32.16374269 514 32.10116732 515 32.23300971 516 32.17054264 517 32.30174081 518 32.23938224 519 32.17726397 520 32.30769231 521 32.24568138 522 32.37547893 523 32.31357553 524 32.25190840 525 32.38095238 526 32.31939163 527 32.44781784 528 32.38636364 529 32.32514178 530 32.26415094 531 32.20338983 532 32.14285714 533 32.27016886 534 32.20973783 535 32.14953271 536 32.08955224 537 32.21601490 538 32.15613383 539 32.09647495 540 32.03703704 541 32.16266174 542 32.10332103 543 32.04419890 544 32.16911765 545 32.29357798 546 32.41758242 547 32.54113346 548 32.66423358 549 32.60473588 550 32.54545455 551 32.48638838 552 32.60869565 553 32.73056058 554 32.67148014 555 32.79279279 556 32.73381295 557 32.67504488 558 32.61648746 559 32.55813953 560 32.67857143 561 32.79857398 562 32.74021352 563 32.68206039 564 32.80141844 565 32.92035398 566 33.03886926 567 32.98059965 568 33.09859155 569 33.21616872 570 33.15789474 571 33.27495622 572 33.21678322 573 33.15881326 574 33.27526132 575 33.39130435 576 33.33333333 577 33.44887348 578 33.56401384 579 33.67875648 580 33.79310345 581 33.73493976 582 33.84879725 583 33.79073756 584 33.90410959 585 33.84615385 586 33.95904437 587 33.90119250 588 33.84353741 589 33.78607810 590 33.72881356 591 33.67174281 592 33.61486486 593 33.72681282 594 33.83838384 595 33.78151261 596 33.72483221 597 33.66834171 598 33.61204013 599 33.55592654 600 33.50000000 601 33.61064892 602 33.72093023 603 33.83084577 604 33.77483444 605 33.71900826 606 33.82838284 607 33.77265239 608 33.71710526 609 33.82594417 610 33.77049180 611 33.71522095 612 33.82352941 613 33.76835237 614 33.71335505 615 33.65853659 616 33.76623377 617 33.87358185 618 33.81877023 619 33.92568659 620 33.87096774 621 33.81642512 622 33.92282958 623 33.86837881 624 33.81410256 625 33.76000000 626 33.70607029 627 33.81180223 628 33.91719745 629 34.02225755 630 33.96825397 631 34.07290016 632 34.01898734 633 34.12322275 634 34.22712934 635 34.17322835 636 34.11949686 637 34.06593407 638 34.01253918 639 33.95931142 640 34.06250000 641 34.00936037 642 34.11214953 643 34.05909798 644 34.00621118 645 34.10852713 646 34.05572755 647 34.15765070 648 34.10493827 649 34.05238829 650 34.15384615 651 34.25499232 652 34.35582822 653 34.45635528 654 34.40366972 655 34.35114504 656 34.45121951 657 34.55098935 658 34.49848024 659 34.44613050 660 34.39393939 661 34.34190620 662 34.29003021 663 34.38914027 664 34.33734940 665 34.28571429 666 34.23423423 667 34.18290855 668 34.13173653 669 34.08071749 670 34.17910448 671 34.12816692 672 34.07738095 673 34.02674591 674 33.97626113 675 34.07407407 676 34.17159763 677 34.12112260 678 34.07079646 679 34.02061856 680 34.11764706 681 34.21439060 682 34.16422287 683 34.11420205 684 34.21052632 685 34.16058394 686 34.11078717 687 34.06113537 688 34.15697674 689 34.25253991 690 34.20289855 691 34.29811867 692 34.39306358 693 34.34343434 694 34.29394813 695 34.38848921 696 34.48275862 697 34.43328551 698 34.38395415 699 34.33476395 700 34.42857143 701 34.37945792 702 34.33048433 703 34.28165007 704 34.23295455 705 34.18439716 706 34.13597734 707 34.22913720 708 34.18079096 709 34.13258110 710 34.08450704 711 34.17721519 712 34.12921348 713 34.08134642 714 34.17366947 715 34.12587413 716 34.07821229 717 34.17015342 718 34.12256267 719 34.21418637 720 34.16666667 721 34.11927878 722 34.07202216 723 34.02489627 724 33.97790055 725 33.93103448 726 34.02203857 727 33.97524072 728 33.92857143 729 34.01920439 730 34.10958904 731 34.06292750 732 34.15300546 733 34.10641201 734 34.05994550 735 34.01360544 736 33.96739130 737 34.05698779 738 34.01084011 739 34.10013532 740 34.05405405 741 34.14304993 742 34.09703504 743 34.05114401 744 34.13978495 745 34.09395973 746 34.04825737 747 34.13654618 748 34.09090909 749 34.17890521 750 34.26666667 Final result: 34.2667 +/- 1.7342 Random chance: 19.8992 +/- 1.4588