| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 26.31578947 | |
| 20 25.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 30.43478261 | |
| 24 33.33333333 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 28.57142857 | |
| 29 31.03448276 | |
| 30 30.00000000 | |
| 31 29.03225806 | |
| 32 31.25000000 | |
| 33 33.33333333 | |
| 34 32.35294118 | |
| 35 31.42857143 | |
| 36 30.55555556 | |
| 37 32.43243243 | |
| 38 31.57894737 | |
| 39 33.33333333 | |
| 40 32.50000000 | |
| 41 31.70731707 | |
| 42 30.95238095 | |
| 43 32.55813953 | |
| 44 34.09090909 | |
| 45 35.55555556 | |
| 46 34.78260870 | |
| 47 34.04255319 | |
| 48 35.41666667 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 37.25490196 | |
| 52 36.53846154 | |
| 53 35.84905660 | |
| 54 35.18518519 | |
| 55 36.36363636 | |
| 56 35.71428571 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 35.48387097 | |
| 63 34.92063492 | |
| 64 34.37500000 | |
| 65 33.84615385 | |
| 66 34.84848485 | |
| 67 34.32835821 | |
| 68 33.82352941 | |
| 69 33.33333333 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 32.43243243 | |
| 75 32.00000000 | |
| 76 32.89473684 | |
| 77 32.46753247 | |
| 78 32.05128205 | |
| 79 31.64556962 | |
| 80 31.25000000 | |
| 81 30.86419753 | |
| 82 31.70731707 | |
| 83 31.32530120 | |
| 84 30.95238095 | |
| 85 30.58823529 | |
| 86 30.23255814 | |
| 87 29.88505747 | |
| 88 29.54545455 | |
| 89 30.33707865 | |
| 90 30.00000000 | |
| 91 29.67032967 | |
| 92 29.34782609 | |
| 93 29.03225806 | |
| 94 28.72340426 | |
| 95 28.42105263 | |
| 96 29.16666667 | |
| 97 29.89690722 | |
| 98 29.59183673 | |
| 99 30.30303030 | |
| 100 31.00000000 | |
| 101 31.68316832 | |
| 102 31.37254902 | |
| 103 32.03883495 | |
| 104 31.73076923 | |
| 105 31.42857143 | |
| 106 31.13207547 | |
| 107 31.77570093 | |
| 108 32.40740741 | |
| 109 33.02752294 | |
| 110 33.63636364 | |
| 111 33.33333333 | |
| 112 33.92857143 | |
| 113 34.51327434 | |
| 114 35.08771930 | |
| 115 34.78260870 | |
| 116 35.34482759 | |
| 117 35.89743590 | |
| 118 35.59322034 | |
| 119 35.29411765 | |
| 120 35.00000000 | |
| 121 35.53719008 | |
| 122 35.24590164 | |
| 123 35.77235772 | |
| 124 35.48387097 | |
| 125 36.00000000 | |
| 126 35.71428571 | |
| 127 35.43307087 | |
| 128 35.15625000 | |
| 129 35.65891473 | |
| 130 36.15384615 | |
| 131 35.87786260 | |
| 132 35.60606061 | |
| 133 35.33834586 | |
| 134 35.07462687 | |
| 135 34.81481481 | |
| 136 34.55882353 | |
| 137 35.03649635 | |
| 138 35.50724638 | |
| 139 35.97122302 | |
| 140 35.71428571 | |
| 141 36.17021277 | |
| 142 35.91549296 | |
| 143 35.66433566 | |
| 144 36.11111111 | |
| 145 36.55172414 | |
| 146 36.30136986 | |
| 147 36.05442177 | |
| 148 35.81081081 | |
| 149 35.57046980 | |
| 150 35.33333333 | |
| 151 35.76158940 | |
| 152 35.52631579 | |
| 153 35.29411765 | |
| 154 35.06493506 | |
| 155 34.83870968 | |
| 156 35.25641026 | |
| 157 35.66878981 | |
| 158 36.07594937 | |
| 159 36.47798742 | |
| 160 36.25000000 | |
| 161 36.64596273 | |
| 162 36.41975309 | |
| 163 36.19631902 | |
| 164 35.97560976 | |
| 165 35.75757576 | |
| 166 36.14457831 | |
| 167 35.92814371 | |
| 168 35.71428571 | |
| 169 35.50295858 | |
| 170 35.29411765 | |
| 171 35.08771930 | |
| 172 34.88372093 | |
| 173 34.68208092 | |
| 174 35.05747126 | |
| 175 34.85714286 | |
| 176 34.65909091 | |
| 177 34.46327684 | |
| 178 34.26966292 | |
| 179 34.07821229 | |
| 180 33.88888889 | |
| 181 33.70165746 | |
| 182 33.51648352 | |
| 183 33.33333333 | |
| 184 33.15217391 | |
| 185 33.51351351 | |
| 186 33.87096774 | |
| 187 33.68983957 | |
| 188 33.51063830 | |
| 189 33.33333333 | |
| 190 33.15789474 | |
| 191 32.98429319 | |
| 192 32.81250000 | |
| 193 32.64248705 | |
| 194 32.98969072 | |
| 195 32.82051282 | |
| 196 32.65306122 | |
| 197 32.99492386 | |
| 198 33.33333333 | |
| 199 33.16582915 | |
| 200 33.50000000 | |
| 201 33.33333333 | |
| 202 33.66336634 | |
| 203 33.49753695 | |
| 204 33.33333333 | |
| 205 33.65853659 | |
| 206 33.49514563 | |
| 207 33.81642512 | |
| 208 33.65384615 | |
| 209 33.49282297 | |
| 210 33.33333333 | |
| 211 33.17535545 | |
| 212 33.01886792 | |
| 213 32.86384977 | |
| 214 32.71028037 | |
| 215 32.55813953 | |
| 216 32.40740741 | |
| 217 32.25806452 | |
| 218 32.11009174 | |
| 219 31.96347032 | |
| 220 31.81818182 | |
| 221 32.12669683 | |
| 222 31.98198198 | |
| 223 32.28699552 | |
| 224 32.58928571 | |
| 225 32.88888889 | |
| 226 32.74336283 | |
| 227 33.03964758 | |
| 228 32.89473684 | |
| 229 32.75109170 | |
| 230 32.60869565 | |
| 231 32.46753247 | |
| 232 32.75862069 | |
| 233 33.04721030 | |
| 234 33.33333333 | |
| 235 33.19148936 | |
| 236 33.47457627 | |
| 237 33.33333333 | |
| 238 33.61344538 | |
| 239 33.47280335 | |
| 240 33.33333333 | |
| 241 33.19502075 | |
| 242 33.05785124 | |
| 243 32.92181070 | |
| 244 32.78688525 | |
| 245 32.65306122 | |
| 246 32.52032520 | |
| 247 32.79352227 | |
| 248 33.06451613 | |
| 249 33.33333333 | |
| 250 33.20000000 | |
| 251 33.46613546 | |
| 252 33.33333333 | |
| 253 33.20158103 | |
| 254 33.46456693 | |
| 255 33.72549020 | |
| 256 33.59375000 | |
| 257 33.46303502 | |
| 258 33.33333333 | |
| 259 33.59073359 | |
| 260 33.46153846 | |
| 261 33.71647510 | |
| 262 33.58778626 | |
| 263 33.46007605 | |
| 264 33.71212121 | |
| 265 33.96226415 | |
| 266 33.83458647 | |
| 267 34.08239700 | |
| 268 33.95522388 | |
| 269 34.20074349 | |
| 270 34.07407407 | |
| 271 33.94833948 | |
| 272 33.82352941 | |
| 273 33.69963370 | |
| 274 33.94160584 | |
| 275 33.81818182 | |
| 276 33.69565217 | |
| 277 33.57400722 | |
| 278 33.45323741 | |
| 279 33.33333333 | |
| 280 33.57142857 | |
| 281 33.80782918 | |
| 282 33.68794326 | |
| 283 33.56890459 | |
| 284 33.80281690 | |
| 285 33.68421053 | |
| 286 33.91608392 | |
| 287 34.14634146 | |
| 288 34.37500000 | |
| 289 34.25605536 | |
| 290 34.13793103 | |
| 291 34.36426117 | |
| 292 34.24657534 | |
| 293 34.47098976 | |
| 294 34.35374150 | |
| 295 34.57627119 | |
| 296 34.79729730 | |
| 297 34.68013468 | |
| 298 34.56375839 | |
| 299 34.44816054 | |
| 300 34.33333333 | |
| 301 34.21926910 | |
| 302 34.10596026 | |
| 303 34.32343234 | |
| 304 34.21052632 | |
| 305 34.09836066 | |
| 306 33.98692810 | |
| 307 33.87622150 | |
| 308 34.09090909 | |
| 309 33.98058252 | |
| 310 33.87096774 | |
| 311 33.76205788 | |
| 312 33.97435897 | |
| 313 33.86581470 | |
| 314 33.75796178 | |
| 315 33.65079365 | |
| 316 33.54430380 | |
| 317 33.43848580 | |
| 318 33.33333333 | |
| 319 33.22884013 | |
| 320 33.12500000 | |
| 321 33.02180685 | |
| 322 33.22981366 | |
| 323 33.12693498 | |
| 324 33.02469136 | |
| 325 32.92307692 | |
| 326 32.82208589 | |
| 327 33.02752294 | |
| 328 33.23170732 | |
| 329 33.13069909 | |
| 330 33.33333333 | |
| 331 33.23262840 | |
| 332 33.43373494 | |
| 333 33.33333333 | |
| 334 33.23353293 | |
| 335 33.43283582 | |
| 336 33.63095238 | |
| 337 33.53115727 | |
| 338 33.43195266 | |
| 339 33.62831858 | |
| 340 33.52941176 | |
| 341 33.72434018 | |
| 342 33.91812865 | |
| 343 33.81924198 | |
| 344 34.01162791 | |
| 345 34.20289855 | |
| 346 34.10404624 | |
| 347 34.29394813 | |
| 348 34.19540230 | |
| 349 34.38395415 | |
| 350 34.28571429 | |
| 351 34.18803419 | |
| 352 34.09090909 | |
| 353 33.99433428 | |
| 354 33.89830508 | |
| 355 34.08450704 | |
| 356 33.98876404 | |
| 357 34.17366947 | |
| 358 34.35754190 | |
| 359 34.26183844 | |
| 360 34.16666667 | |
| 361 34.34903047 | |
| 362 34.25414365 | |
| 363 34.15977961 | |
| 364 34.06593407 | |
| 365 33.97260274 | |
| 366 33.87978142 | |
| 367 34.05994550 | |
| 368 33.96739130 | |
| 369 33.87533875 | |
| 370 33.78378378 | |
| 371 33.69272237 | |
| 372 33.87096774 | |
| 373 33.78016086 | |
| 374 33.95721925 | |
| 375 33.86666667 | |
| 376 33.77659574 | |
| 377 33.68700265 | |
| 378 33.59788360 | |
| 379 33.50923483 | |
| 380 33.68421053 | |
| 381 33.85826772 | |
| 382 34.03141361 | |
| 383 33.94255875 | |
| 384 33.85416667 | |
| 385 33.76623377 | |
| 386 33.67875648 | |
| 387 33.59173127 | |
| 388 33.76288660 | |
| 389 33.67609254 | |
| 390 33.58974359 | |
| 391 33.50383632 | |
| 392 33.41836735 | |
| 393 33.33333333 | |
| 394 33.50253807 | |
| 395 33.41772152 | |
| 396 33.33333333 | |
| 397 33.50125945 | |
| 398 33.41708543 | |
| 399 33.33333333 | |
| 400 33.25000000 | |
| 401 33.41645885 | |
| 402 33.58208955 | |
| 403 33.74689826 | |
| 404 33.66336634 | |
| 405 33.58024691 | |
| 406 33.49753695 | |
| 407 33.41523342 | |
| 408 33.33333333 | |
| 409 33.25183374 | |
| 410 33.17073171 | |
| 411 33.09002433 | |
| 412 33.25242718 | |
| 413 33.41404358 | |
| 414 33.57487923 | |
| 415 33.73493976 | |
| 416 33.65384615 | |
| 417 33.57314149 | |
| 418 33.49282297 | |
| 419 33.41288783 | |
| 420 33.57142857 | |
| 421 33.72921615 | |
| 422 33.64928910 | |
| 423 33.56973995 | |
| 424 33.49056604 | |
| 425 33.41176471 | |
| 426 33.33333333 | |
| 427 33.25526932 | |
| 428 33.17757009 | |
| 429 33.10023310 | |
| 430 33.02325581 | |
| 431 32.94663573 | |
| 432 33.10185185 | |
| 433 33.25635104 | |
| 434 33.41013825 | |
| 435 33.33333333 | |
| 436 33.25688073 | |
| 437 33.18077803 | |
| 438 33.33333333 | |
| 439 33.48519362 | |
| 440 33.40909091 | |
| 441 33.33333333 | |
| 442 33.25791855 | |
| 443 33.18284424 | |
| 444 33.33333333 | |
| 445 33.25842697 | |
| 446 33.18385650 | |
| 447 33.10961969 | |
| 448 33.03571429 | |
| 449 33.18485523 | |
| 450 33.33333333 | |
| 451 33.25942350 | |
| 452 33.18584071 | |
| 453 33.33333333 | |
| 454 33.25991189 | |
| 455 33.18681319 | |
| 456 33.11403509 | |
| 457 33.04157549 | |
| 458 32.96943231 | |
| 459 32.89760349 | |
| 460 32.82608696 | |
| 461 32.75488069 | |
| 462 32.90043290 | |
| 463 33.04535637 | |
| 464 33.18965517 | |
| 465 33.33333333 | |
| 466 33.26180258 | |
| 467 33.19057816 | |
| 468 33.11965812 | |
| 469 33.04904051 | |
| 470 32.97872340 | |
| 471 33.12101911 | |
| 472 33.26271186 | |
| 473 33.19238901 | |
| 474 33.33333333 | |
| 475 33.47368421 | |
| 476 33.40336134 | |
| 477 33.54297694 | |
| 478 33.47280335 | |
| 479 33.40292276 | |
| 480 33.33333333 | |
| 481 33.26403326 | |
| 482 33.40248963 | |
| 483 33.33333333 | |
| 484 33.47107438 | |
| 485 33.60824742 | |
| 486 33.53909465 | |
| 487 33.67556468 | |
| 488 33.60655738 | |
| 489 33.74233129 | |
| 490 33.87755102 | |
| 491 34.01221996 | |
| 492 34.14634146 | |
| 493 34.07707911 | |
| 494 34.21052632 | |
| 495 34.34343434 | |
| 496 34.27419355 | |
| 497 34.20523139 | |
| 498 34.33734940 | |
| 499 34.46893788 | |
| 500 34.40000000 | |
| 501 34.33133733 | |
| 502 34.26294821 | |
| 503 34.19483101 | |
| 504 34.32539683 | |
| 505 34.25742574 | |
| 506 34.18972332 | |
| 507 34.31952663 | |
| 508 34.44881890 | |
| 509 34.57760314 | |
| 510 34.50980392 | |
| 511 34.44227006 | |
| 512 34.37500000 | |
| 513 34.30799220 | |
| 514 34.24124514 | |
| 515 34.36893204 | |
| 516 34.30232558 | |
| 517 34.42940039 | |
| 518 34.36293436 | |
| 519 34.29672447 | |
| 520 34.42307692 | |
| 521 34.35700576 | |
| 522 34.48275862 | |
| 523 34.41682600 | |
| 524 34.54198473 | |
| 525 34.66666667 | |
| 526 34.60076046 | |
| 527 34.72485769 | |
| 528 34.65909091 | |
| 529 34.59357278 | |
| 530 34.52830189 | |
| 531 34.46327684 | |
| 532 34.58646617 | |
| 533 34.70919325 | |
| 534 34.64419476 | |
| 535 34.57943925 | |
| 536 34.51492537 | |
| 537 34.63687151 | |
| 538 34.57249071 | |
| 539 34.50834879 | |
| 540 34.44444444 | |
| 541 34.56561922 | |
| 542 34.50184502 | |
| 543 34.43830571 | |
| 544 34.55882353 | |
| 545 34.67889908 | |
| 546 34.61538462 | |
| 547 34.73491773 | |
| 548 34.85401460 | |
| 549 34.79052823 | |
| 550 34.72727273 | |
| 551 34.66424682 | |
| 552 34.78260870 | |
| 553 34.90054250 | |
| 554 34.83754513 | |
| 555 34.95495495 | |
| 556 34.89208633 | |
| 557 34.82944345 | |
| 558 34.76702509 | |
| 559 34.70483005 | |
| 560 34.82142857 | |
| 561 34.93761141 | |
| 562 34.87544484 | |
| 563 34.81349911 | |
| 564 34.92907801 | |
| 565 35.04424779 | |
| 566 35.15901060 | |
| 567 35.09700176 | |
| 568 35.21126761 | |
| 569 35.32513181 | |
| 570 35.26315789 | |
| 571 35.37653240 | |
| 572 35.31468531 | |
| 573 35.25305410 | |
| 574 35.19163763 | |
| 575 35.30434783 | |
| 576 35.41666667 | |
| 577 35.52859619 | |
| 578 35.46712803 | |
| 579 35.57858377 | |
| 580 35.68965517 | |
| 581 35.62822719 | |
| 582 35.73883162 | |
| 583 35.67753002 | |
| 584 35.78767123 | |
| 585 35.72649573 | |
| 586 35.83617747 | |
| 587 35.77512777 | |
| 588 35.71428571 | |
| 589 35.65365025 | |
| 590 35.59322034 | |
| 591 35.53299492 | |
| 592 35.47297297 | |
| 593 35.58178752 | |
| 594 35.69023569 | |
| 595 35.63025210 | |
| 596 35.57046980 | |
| 597 35.67839196 | |
| 598 35.61872910 | |
| 599 35.55926544 | |
| 600 35.50000000 | |
| 601 35.60732113 | |
| 602 35.71428571 | |
| 603 35.82089552 | |
| 604 35.76158940 | |
| 605 35.70247934 | |
| 606 35.80858086 | |
| 607 35.74958814 | |
| 608 35.69078947 | |
| 609 35.79638752 | |
| 610 35.73770492 | |
| 611 35.67921440 | |
| 612 35.78431373 | |
| 613 35.72593801 | |
| 614 35.66775244 | |
| 615 35.60975610 | |
| 616 35.71428571 | |
| 617 35.81847650 | |
| 618 35.76051780 | |
| 619 35.86429725 | |
| 620 35.80645161 | |
| 621 35.90982287 | |
| 622 36.01286174 | |
| 623 35.95505618 | |
| 624 35.89743590 | |
| 625 35.84000000 | |
| 626 35.78274760 | |
| 627 35.88516746 | |
| 628 35.98726115 | |
| 629 36.08903021 | |
| 630 36.03174603 | |
| 631 36.13312203 | |
| 632 36.07594937 | |
| 633 36.17693523 | |
| 634 36.27760252 | |
| 635 36.22047244 | |
| 636 36.16352201 | |
| 637 36.10675039 | |
| 638 36.05015674 | |
| 639 35.99374022 | |
| 640 36.09375000 | |
| 641 36.03744150 | |
| 642 36.13707165 | |
| 643 36.08087092 | |
| 644 36.02484472 | |
| 645 36.12403101 | |
| 646 36.06811146 | |
| 647 36.16692427 | |
| 648 36.11111111 | |
| 649 36.05546995 | |
| 650 36.15384615 | |
| 651 36.25192012 | |
| 652 36.34969325 | |
| 653 36.44716692 | |
| 654 36.39143731 | |
| 655 36.33587786 | |
| 656 36.28048780 | |
| 657 36.37747336 | |
| 658 36.32218845 | |
| 659 36.26707132 | |
| 660 36.21212121 | |
| 661 36.15733737 | |
| 662 36.10271903 | |
| 663 36.19909502 | |
| 664 36.14457831 | |
| 665 36.09022556 | |
| 666 36.18618619 | |
| 667 36.13193403 | |
| 668 36.07784431 | |
| 669 36.17339312 | |
| 670 36.26865672 | |
| 671 36.21460507 | |
| 672 36.16071429 | |
| 673 36.10698366 | |
| 674 36.05341246 | |
| 675 36.14814815 | |
| 676 36.24260355 | |
| 677 36.18906942 | |
| 678 36.13569322 | |
| 679 36.08247423 | |
| 680 36.17647059 | |
| 681 36.27019090 | |
| 682 36.21700880 | |
| 683 36.16398243 | |
| 684 36.25730994 | |
| 685 36.20437956 | |
| 686 36.15160350 | |
| 687 36.09898108 | |
| 688 36.19186047 | |
| 689 36.28447025 | |
| 690 36.23188406 | |
| 691 36.32416787 | |
| 692 36.41618497 | |
| 693 36.36363636 | |
| 694 36.31123919 | |
| 695 36.40287770 | |
| 696 36.49425287 | |
| 697 36.44189383 | |
| 698 36.38968481 | |
| 699 36.33762518 | |
| 700 36.42857143 | |
| 701 36.37660485 | |
| 702 36.32478632 | |
| 703 36.27311522 | |
| 704 36.22159091 | |
| 705 36.17021277 | |
| 706 36.26062323 | |
| 707 36.35077793 | |
| 708 36.29943503 | |
| 709 36.24823695 | |
| 710 36.33802817 | |
| 711 36.42756681 | |
| 712 36.37640449 | |
| 713 36.32538569 | |
| 714 36.41456583 | |
| 715 36.50349650 | |
| 716 36.45251397 | |
| 717 36.54114365 | |
| 718 36.49025070 | |
| 719 36.57858136 | |
| 720 36.52777778 | |
| 721 36.47711512 | |
| 722 36.42659280 | |
| 723 36.37621024 | |
| 724 36.32596685 | |
| 725 36.27586207 | |
| 726 36.36363636 | |
| 727 36.31361761 | |
| 728 36.26373626 | |
| 729 36.35116598 | |
| 730 36.43835616 | |
| 731 36.38850889 | |
| 732 36.47540984 | |
| 733 36.42564802 | |
| 734 36.37602180 | |
| 735 36.32653061 | |
| 736 36.27717391 | |
| 737 36.36363636 | |
| 738 36.31436314 | |
| 739 36.40054127 | |
| 740 36.35135135 | |
| 741 36.43724696 | |
| 742 36.38814016 | |
| 743 36.33916555 | |
| 744 36.42473118 | |
| 745 36.37583893 | |
| 746 36.32707775 | |
| 747 36.41231593 | |
| 748 36.36363636 | |
| 749 36.44859813 | |
| 750 36.53333333 | |
| Final result: 36.5333 +/- 1.7594 | |
| Random chance: 19.8992 +/- 1.4588 | |