| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 100.00000000 | |
| 4 75.00000000 | |
| 5 80.00000000 | |
| 6 83.33333333 | |
| 7 85.71428571 | |
| 8 87.50000000 | |
| 9 77.77777778 | |
| 10 80.00000000 | |
| 11 72.72727273 | |
| 12 75.00000000 | |
| 13 69.23076923 | |
| 14 71.42857143 | |
| 15 73.33333333 | |
| 16 75.00000000 | |
| 17 76.47058824 | |
| 18 72.22222222 | |
| 19 73.68421053 | |
| 20 70.00000000 | |
| 21 71.42857143 | |
| 22 72.72727273 | |
| 23 73.91304348 | |
| 24 75.00000000 | |
| 25 72.00000000 | |
| 26 69.23076923 | |
| 27 70.37037037 | |
| 28 67.85714286 | |
| 29 68.96551724 | |
| 30 70.00000000 | |
| 31 67.74193548 | |
| 32 68.75000000 | |
| 33 69.69696970 | |
| 34 70.58823529 | |
| 35 71.42857143 | |
| 36 69.44444444 | |
| 37 70.27027027 | |
| 38 71.05263158 | |
| 39 71.79487179 | |
| 40 72.50000000 | |
| 41 70.73170732 | |
| 42 69.04761905 | |
| 43 67.44186047 | |
| 44 65.90909091 | |
| 45 66.66666667 | |
| 46 67.39130435 | |
| 47 68.08510638 | |
| 48 68.75000000 | |
| 49 67.34693878 | |
| 50 66.00000000 | |
| 51 64.70588235 | |
| 52 65.38461538 | |
| 53 66.03773585 | |
| 54 64.81481481 | |
| 55 63.63636364 | |
| 56 64.28571429 | |
| 57 64.91228070 | |
| 58 65.51724138 | |
| 59 66.10169492 | |
| 60 66.66666667 | |
| 61 67.21311475 | |
| 62 67.74193548 | |
| 63 68.25396825 | |
| 64 68.75000000 | |
| 65 67.69230769 | |
| 66 68.18181818 | |
| 67 68.65671642 | |
| 68 69.11764706 | |
| 69 69.56521739 | |
| 70 70.00000000 | |
| 71 70.42253521 | |
| 72 69.44444444 | |
| 73 69.86301370 | |
| 74 70.27027027 | |
| 75 70.66666667 | |
| 76 71.05263158 | |
| 77 71.42857143 | |
| 78 71.79487179 | |
| 79 72.15189873 | |
| 80 72.50000000 | |
| 81 72.83950617 | |
| 82 73.17073171 | |
| 83 72.28915663 | |
| 84 72.61904762 | |
| 85 72.94117647 | |
| 86 73.25581395 | |
| 87 73.56321839 | |
| 88 73.86363636 | |
| 89 74.15730337 | |
| 90 74.44444444 | |
| 91 74.72527473 | |
| 92 75.00000000 | |
| 93 74.19354839 | |
| 94 73.40425532 | |
| 95 73.68421053 | |
| 96 73.95833333 | |
| 97 73.19587629 | |
| 98 72.44897959 | |
| 99 72.72727273 | |
| 100 73.00000000 | |
| 101 73.26732673 | |
| 102 73.52941176 | |
| 103 72.81553398 | |
| 104 72.11538462 | |
| 105 72.38095238 | |
| 106 71.69811321 | |
| 107 71.96261682 | |
| 108 71.29629630 | |
| 109 70.64220183 | |
| 110 70.90909091 | |
| 111 71.17117117 | |
| 112 71.42857143 | |
| 113 70.79646018 | |
| 114 71.05263158 | |
| 115 71.30434783 | |
| 116 71.55172414 | |
| 117 70.94017094 | |
| 118 71.18644068 | |
| 119 70.58823529 | |
| 120 70.83333333 | |
| 121 70.24793388 | |
| 122 70.49180328 | |
| 123 70.73170732 | |
| 124 70.96774194 | |
| 125 71.20000000 | |
| 126 71.42857143 | |
| 127 71.65354331 | |
| 128 71.87500000 | |
| 129 71.31782946 | |
| 130 71.53846154 | |
| 131 71.75572519 | |
| 132 71.21212121 | |
| 133 71.42857143 | |
| 134 70.89552239 | |
| 135 70.37037037 | |
| 136 70.58823529 | |
| 137 70.80291971 | |
| 138 70.28985507 | |
| 139 70.50359712 | |
| 140 70.00000000 | |
| 141 69.50354610 | |
| 142 69.71830986 | |
| 143 69.93006993 | |
| 144 70.13888889 | |
| 145 70.34482759 | |
| 146 69.86301370 | |
| 147 70.06802721 | |
| 148 69.59459459 | |
| 149 69.79865772 | |
| 150 70.00000000 | |
| 151 70.19867550 | |
| 152 70.39473684 | |
| 153 70.58823529 | |
| 154 70.77922078 | |
| 155 70.32258065 | |
| 156 70.51282051 | |
| 157 70.06369427 | |
| 158 70.25316456 | |
| 159 70.44025157 | |
| 160 70.62500000 | |
| 161 70.80745342 | |
| 162 70.98765432 | |
| 163 70.55214724 | |
| 164 70.73170732 | |
| 165 70.90909091 | |
| 166 71.08433735 | |
| 167 71.25748503 | |
| 168 71.42857143 | |
| 169 71.00591716 | |
| 170 71.17647059 | |
| 171 71.34502924 | |
| 172 70.93023256 | |
| 173 71.09826590 | |
| 174 71.26436782 | |
| 175 70.85714286 | |
| 176 71.02272727 | |
| 177 71.18644068 | |
| 178 70.78651685 | |
| 179 70.94972067 | |
| 180 71.11111111 | |
| 181 70.71823204 | |
| 182 70.87912088 | |
| 183 70.49180328 | |
| 184 70.65217391 | |
| 185 70.81081081 | |
| 186 70.96774194 | |
| 187 71.12299465 | |
| 188 70.74468085 | |
| 189 70.37037037 | |
| 190 70.52631579 | |
| 191 70.68062827 | |
| 192 70.31250000 | |
| 193 69.94818653 | |
| 194 70.10309278 | |
| 195 70.25641026 | |
| 196 70.40816327 | |
| 197 70.55837563 | |
| 198 70.70707071 | |
| 199 70.35175879 | |
| 200 70.50000000 | |
| 201 70.64676617 | |
| 202 70.79207921 | |
| 203 70.93596059 | |
| 204 71.07843137 | |
| 205 71.21951220 | |
| 206 70.87378641 | |
| 207 71.01449275 | |
| 208 71.15384615 | |
| 209 71.29186603 | |
| 210 71.42857143 | |
| 211 71.56398104 | |
| 212 71.22641509 | |
| 213 71.36150235 | |
| 214 71.49532710 | |
| 215 71.62790698 | |
| 216 71.29629630 | |
| 217 71.42857143 | |
| 218 71.55963303 | |
| 219 71.68949772 | |
| 220 71.36363636 | |
| 221 71.49321267 | |
| 222 71.62162162 | |
| 223 71.30044843 | |
| 224 71.42857143 | |
| 225 71.11111111 | |
| 226 70.79646018 | |
| 227 70.92511013 | |
| 228 71.05263158 | |
| 229 71.17903930 | |
| 230 71.30434783 | |
| 231 70.99567100 | |
| 232 70.68965517 | |
| 233 70.38626609 | |
| 234 70.51282051 | |
| 235 70.21276596 | |
| 236 70.33898305 | |
| 237 70.04219409 | |
| 238 69.74789916 | |
| 239 69.87447699 | |
| 240 70.00000000 | |
| 241 70.12448133 | |
| 242 69.83471074 | |
| 243 69.95884774 | |
| 244 69.67213115 | |
| 245 69.79591837 | |
| 246 69.91869919 | |
| 247 70.04048583 | |
| 248 70.16129032 | |
| 249 70.28112450 | |
| 250 70.40000000 | |
| 251 70.51792829 | |
| 252 70.63492063 | |
| 253 70.75098814 | |
| 254 70.86614173 | |
| 255 70.98039216 | |
| 256 71.09375000 | |
| 257 70.81712062 | |
| 258 70.93023256 | |
| 259 71.04247104 | |
| 260 71.15384615 | |
| 261 71.26436782 | |
| 262 71.37404580 | |
| 263 71.48288973 | |
| 264 71.59090909 | |
| 265 71.69811321 | |
| 266 71.42857143 | |
| 267 71.53558052 | |
| 268 71.64179104 | |
| 269 71.74721190 | |
| 270 71.85185185 | |
| 271 71.95571956 | |
| 272 72.05882353 | |
| 273 71.79487179 | |
| 274 71.53284672 | |
| 275 71.27272727 | |
| 276 71.37681159 | |
| 277 71.48014440 | |
| 278 71.58273381 | |
| 279 71.68458781 | |
| 280 71.78571429 | |
| 281 71.88612100 | |
| 282 71.98581560 | |
| 283 71.73144876 | |
| 284 71.83098592 | |
| 285 71.92982456 | |
| 286 72.02797203 | |
| 287 71.77700348 | |
| 288 71.87500000 | |
| 289 71.62629758 | |
| 290 71.72413793 | |
| 291 71.82130584 | |
| 292 71.91780822 | |
| 293 72.01365188 | |
| 294 71.76870748 | |
| 295 71.86440678 | |
| 296 71.95945946 | |
| 297 72.05387205 | |
| 298 72.14765101 | |
| 299 72.24080268 | |
| 300 72.33333333 | |
| 301 72.42524917 | |
| 302 72.51655629 | |
| 303 72.60726073 | |
| 304 72.69736842 | |
| 305 72.78688525 | |
| 306 72.54901961 | |
| 307 72.63843648 | |
| 308 72.72727273 | |
| 309 72.81553398 | |
| 310 72.58064516 | |
| 311 72.66881029 | |
| 312 72.75641026 | |
| 313 72.84345048 | |
| 314 72.92993631 | |
| 315 73.01587302 | |
| 316 73.10126582 | |
| 317 73.18611987 | |
| 318 73.27044025 | |
| 319 73.04075235 | |
| 320 72.81250000 | |
| 321 72.89719626 | |
| 322 72.98136646 | |
| 323 73.06501548 | |
| 324 73.14814815 | |
| 325 73.23076923 | |
| 326 73.31288344 | |
| 327 73.08868502 | |
| 328 73.17073171 | |
| 329 72.94832827 | |
| 330 72.72727273 | |
| 331 72.50755287 | |
| 332 72.59036145 | |
| 333 72.67267267 | |
| 334 72.75449102 | |
| 335 72.53731343 | |
| 336 72.32142857 | |
| 337 72.10682493 | |
| 338 71.89349112 | |
| 339 71.97640118 | |
| 340 72.05882353 | |
| 341 72.14076246 | |
| 342 72.22222222 | |
| 343 72.30320700 | |
| 344 72.38372093 | |
| 345 72.17391304 | |
| 346 72.25433526 | |
| 347 72.33429395 | |
| 348 72.12643678 | |
| 349 72.20630372 | |
| 350 72.28571429 | |
| 351 72.36467236 | |
| 352 72.44318182 | |
| 353 72.52124646 | |
| 354 72.31638418 | |
| 355 72.11267606 | |
| 356 72.19101124 | |
| 357 72.26890756 | |
| 358 72.34636872 | |
| 359 72.14484680 | |
| 360 72.22222222 | |
| 361 72.29916898 | |
| 362 72.37569061 | |
| 363 72.45179063 | |
| 364 72.52747253 | |
| 365 72.60273973 | |
| 366 72.40437158 | |
| 367 72.47956403 | |
| 368 72.55434783 | |
| 369 72.62872629 | |
| 370 72.70270270 | |
| 371 72.50673854 | |
| 372 72.31182796 | |
| 373 72.38605898 | |
| 374 72.45989305 | |
| 375 72.26666667 | |
| 376 72.34042553 | |
| 377 72.14854111 | |
| 378 72.22222222 | |
| 379 72.03166227 | |
| 380 72.10526316 | |
| 381 71.91601050 | |
| 382 71.98952880 | |
| 383 71.80156658 | |
| 384 71.87500000 | |
| 385 71.94805195 | |
| 386 71.76165803 | |
| 387 71.83462532 | |
| 388 71.90721649 | |
| 389 71.72236504 | |
| 390 71.53846154 | |
| 391 71.35549872 | |
| 392 71.42857143 | |
| 393 71.50127226 | |
| 394 71.31979695 | |
| 395 71.39240506 | |
| 396 71.21212121 | |
| 397 71.28463476 | |
| 398 71.10552764 | |
| 399 71.17794486 | |
| 400 71.25000000 | |
| 401 71.32169576 | |
| 402 71.39303483 | |
| 403 71.46401985 | |
| 404 71.53465347 | |
| 405 71.60493827 | |
| 406 71.67487685 | |
| 407 71.74447174 | |
| 408 71.81372549 | |
| 409 71.63814181 | |
| 410 71.70731707 | |
| 411 71.77615572 | |
| 412 71.84466019 | |
| 413 71.91283293 | |
| 414 71.98067633 | |
| 415 72.04819277 | |
| 416 72.11538462 | |
| 417 71.94244604 | |
| 418 72.00956938 | |
| 419 71.83770883 | |
| 420 71.90476190 | |
| 421 71.97149644 | |
| 422 72.03791469 | |
| 423 72.10401891 | |
| 424 72.16981132 | |
| 425 72.00000000 | |
| 426 71.83098592 | |
| 427 71.66276347 | |
| 428 71.49532710 | |
| 429 71.56177156 | |
| 430 71.39534884 | |
| 431 71.46171694 | |
| 432 71.52777778 | |
| 433 71.36258661 | |
| 434 71.42857143 | |
| 435 71.49425287 | |
| 436 71.55963303 | |
| 437 71.62471396 | |
| 438 71.46118721 | |
| 439 71.52619590 | |
| 440 71.59090909 | |
| 441 71.65532880 | |
| 442 71.49321267 | |
| 443 71.55756208 | |
| 444 71.62162162 | |
| 445 71.68539326 | |
| 446 71.74887892 | |
| 447 71.81208054 | |
| 448 71.87500000 | |
| 449 71.93763920 | |
| 450 71.77777778 | |
| 451 71.84035477 | |
| 452 71.90265487 | |
| 453 71.96467991 | |
| 454 71.80616740 | |
| 455 71.86813187 | |
| 456 71.92982456 | |
| 457 71.99124726 | |
| 458 72.05240175 | |
| 459 72.11328976 | |
| 460 71.95652174 | |
| 461 72.01735358 | |
| 462 72.07792208 | |
| 463 71.92224622 | |
| 464 71.98275862 | |
| 465 72.04301075 | |
| 466 72.10300429 | |
| 467 72.16274090 | |
| 468 72.22222222 | |
| 469 72.06823028 | |
| 470 72.12765957 | |
| 471 72.18683652 | |
| 472 72.24576271 | |
| 473 72.09302326 | |
| 474 72.15189873 | |
| 475 72.00000000 | |
| 476 72.05882353 | |
| 477 72.11740042 | |
| 478 72.17573222 | |
| 479 72.23382046 | |
| 480 72.08333333 | |
| 481 72.14137214 | |
| 482 72.19917012 | |
| 483 72.25672878 | |
| 484 72.31404959 | |
| 485 72.37113402 | |
| 486 72.42798354 | |
| 487 72.27926078 | |
| 488 72.33606557 | |
| 489 72.39263804 | |
| 490 72.44897959 | |
| 491 72.50509165 | |
| 492 72.56097561 | |
| 493 72.41379310 | |
| 494 72.46963563 | |
| 495 72.52525253 | |
| 496 72.37903226 | |
| 497 72.23340040 | |
| 498 72.28915663 | |
| 499 72.34468938 | |
| 500 72.40000000 | |
| 501 72.45508982 | |
| 502 72.50996016 | |
| 503 72.56461233 | |
| 504 72.42063492 | |
| 505 72.47524752 | |
| 506 72.33201581 | |
| 507 72.38658777 | |
| 508 72.44094488 | |
| 509 72.29862475 | |
| 510 72.35294118 | |
| 511 72.21135029 | |
| 512 72.26562500 | |
| 513 72.12475634 | |
| 514 72.17898833 | |
| 515 72.23300971 | |
| 516 72.09302326 | |
| 517 72.14700193 | |
| 518 72.20077220 | |
| 519 72.25433526 | |
| 520 72.11538462 | |
| 521 72.16890595 | |
| 522 72.22222222 | |
| 523 72.08413002 | |
| 524 71.94656489 | |
| 525 71.80952381 | |
| 526 71.67300380 | |
| 527 71.53700190 | |
| 528 71.40151515 | |
| 529 71.26654064 | |
| 530 71.32075472 | |
| 531 71.37476460 | |
| 532 71.42857143 | |
| 533 71.29455910 | |
| 534 71.16104869 | |
| 535 71.02803738 | |
| 536 71.08208955 | |
| 537 71.13594041 | |
| 538 71.18959108 | |
| 539 71.24304267 | |
| 540 71.11111111 | |
| 541 71.16451017 | |
| 542 71.21771218 | |
| 543 71.08655617 | |
| 544 71.13970588 | |
| 545 71.19266055 | |
| 546 71.24542125 | |
| 547 71.29798903 | |
| 548 71.35036496 | |
| 549 71.40255009 | |
| 550 71.45454545 | |
| 551 71.32486388 | |
| 552 71.19565217 | |
| 553 71.24773960 | |
| 554 71.29963899 | |
| 555 71.35135135 | |
| 556 71.22302158 | |
| 557 71.27468582 | |
| 558 71.14695341 | |
| 559 71.19856887 | |
| 560 71.25000000 | |
| 561 71.30124777 | |
| 562 71.35231317 | |
| 563 71.40319716 | |
| 564 71.45390071 | |
| 565 71.32743363 | |
| 566 71.37809187 | |
| 567 71.42857143 | |
| 568 71.47887324 | |
| 569 71.35325132 | |
| 570 71.22807018 | |
| 571 71.27845884 | |
| 572 71.15384615 | |
| 573 71.20418848 | |
| 574 71.25435540 | |
| 575 71.30434783 | |
| 576 71.35416667 | |
| 577 71.40381282 | |
| 578 71.28027682 | |
| 579 71.32987910 | |
| 580 71.37931034 | |
| 581 71.25645439 | |
| 582 71.13402062 | |
| 583 71.01200686 | |
| 584 71.06164384 | |
| 585 71.11111111 | |
| 586 71.16040956 | |
| 587 71.20954003 | |
| 588 71.25850340 | |
| 589 71.30730051 | |
| 590 71.35593220 | |
| 591 71.23519459 | |
| 592 71.28378378 | |
| 593 71.33220911 | |
| 594 71.38047138 | |
| 595 71.42857143 | |
| 596 71.30872483 | |
| 597 71.35678392 | |
| 598 71.40468227 | |
| 599 71.28547579 | |
| 600 71.33333333 | |
| 601 71.38103161 | |
| 602 71.26245847 | |
| 603 71.14427861 | |
| 604 71.19205298 | |
| 605 71.23966942 | |
| 606 71.28712871 | |
| 607 71.33443163 | |
| 608 71.38157895 | |
| 609 71.26436782 | |
| 610 71.31147541 | |
| 611 71.19476268 | |
| 612 71.24183007 | |
| 613 71.12561175 | |
| 614 71.17263844 | |
| 615 71.05691057 | |
| 616 70.94155844 | |
| 617 70.98865478 | |
| 618 71.03559871 | |
| 619 70.92084006 | |
| 620 70.96774194 | |
| 621 71.01449275 | |
| 622 71.06109325 | |
| 623 71.10754414 | |
| 624 71.15384615 | |
| 625 71.04000000 | |
| 626 71.08626198 | |
| 627 71.13237640 | |
| 628 71.01910828 | |
| 629 71.06518283 | |
| 630 71.11111111 | |
| 631 71.15689382 | |
| 632 71.04430380 | |
| 633 70.93206951 | |
| 634 70.97791798 | |
| 635 71.02362205 | |
| 636 71.06918239 | |
| 637 71.11459969 | |
| 638 71.15987461 | |
| 639 71.20500782 | |
| 640 71.25000000 | |
| 641 71.29485179 | |
| 642 71.18380062 | |
| 643 71.07309487 | |
| 644 71.11801242 | |
| 645 71.00775194 | |
| 646 71.05263158 | |
| 647 70.94281298 | |
| 648 70.98765432 | |
| 649 70.87827427 | |
| 650 70.92307692 | |
| 651 70.96774194 | |
| 652 70.85889571 | |
| 653 70.90352221 | |
| 654 70.79510703 | |
| 655 70.68702290 | |
| 656 70.57926829 | |
| 657 70.62404871 | |
| 658 70.66869301 | |
| 659 70.56145675 | |
| 660 70.60606061 | |
| 661 70.49924357 | |
| 662 70.54380665 | |
| 663 70.58823529 | |
| 664 70.63253012 | |
| 665 70.67669173 | |
| 666 70.72072072 | |
| 667 70.76461769 | |
| 668 70.65868263 | |
| 669 70.70254111 | |
| 670 70.59701493 | |
| 671 70.64083458 | |
| 672 70.68452381 | |
| 673 70.72808321 | |
| 674 70.77151335 | |
| 675 70.81481481 | |
| 676 70.85798817 | |
| 677 70.90103397 | |
| 678 70.94395280 | |
| 679 70.83946981 | |
| 680 70.88235294 | |
| 681 70.92511013 | |
| 682 70.96774194 | |
| 683 70.86383602 | |
| 684 70.90643275 | |
| 685 70.94890511 | |
| 686 70.99125364 | |
| 687 71.03347889 | |
| 688 71.07558140 | |
| 689 70.97242380 | |
| 690 71.01449275 | |
| 691 71.05643994 | |
| 692 70.95375723 | |
| 693 70.99567100 | |
| 694 71.03746398 | |
| 695 70.93525180 | |
| 696 70.83333333 | |
| 697 70.87517934 | |
| 698 70.77363897 | |
| 699 70.67238913 | |
| 700 70.57142857 | |
| 701 70.47075606 | |
| 702 70.51282051 | |
| 703 70.55476529 | |
| 704 70.59659091 | |
| 705 70.63829787 | |
| 706 70.67988669 | |
| 707 70.72135785 | |
| 708 70.76271186 | |
| 709 70.80394922 | |
| 710 70.84507042 | |
| 711 70.88607595 | |
| 712 70.92696629 | |
| 713 70.96774194 | |
| 714 70.86834734 | |
| 715 70.76923077 | |
| 716 70.81005587 | |
| 717 70.85076709 | |
| 718 70.75208914 | |
| 719 70.65368567 | |
| 720 70.69444444 | |
| 721 70.59639390 | |
| 722 70.63711911 | |
| 723 70.67773167 | |
| 724 70.71823204 | |
| 725 70.75862069 | |
| 726 70.66115702 | |
| 727 70.70151307 | |
| 728 70.74175824 | |
| 729 70.78189300 | |
| 730 70.82191781 | |
| 731 70.86183311 | |
| 732 70.90163934 | |
| 733 70.94133697 | |
| 734 70.98092643 | |
| 735 71.02040816 | |
| 736 70.92391304 | |
| 737 70.82767978 | |
| 738 70.86720867 | |
| 739 70.77131258 | |
| 740 70.67567568 | |
| 741 70.71524966 | |
| 742 70.75471698 | |
| 743 70.79407806 | |
| 744 70.69892473 | |
| 745 70.73825503 | |
| 746 70.77747989 | |
| 747 70.68273092 | |
| 748 70.58823529 | |
| 749 70.49399199 | |
| 750 70.53333333 | |
| Final result: 70.5333 +/- 1.6658 | |
| Random chance: 25.0083 +/- 1.5824 | |