| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 100.00000000 | |
| 4 75.00000000 | |
| 5 80.00000000 | |
| 6 83.33333333 | |
| 7 85.71428571 | |
| 8 87.50000000 | |
| 9 77.77777778 | |
| 10 80.00000000 | |
| 11 72.72727273 | |
| 12 75.00000000 | |
| 13 69.23076923 | |
| 14 71.42857143 | |
| 15 73.33333333 | |
| 16 75.00000000 | |
| 17 76.47058824 | |
| 18 77.77777778 | |
| 19 78.94736842 | |
| 20 75.00000000 | |
| 21 76.19047619 | |
| 22 77.27272727 | |
| 23 73.91304348 | |
| 24 75.00000000 | |
| 25 72.00000000 | |
| 26 69.23076923 | |
| 27 70.37037037 | |
| 28 67.85714286 | |
| 29 68.96551724 | |
| 30 70.00000000 | |
| 31 67.74193548 | |
| 32 68.75000000 | |
| 33 69.69696970 | |
| 34 70.58823529 | |
| 35 71.42857143 | |
| 36 69.44444444 | |
| 37 70.27027027 | |
| 38 71.05263158 | |
| 39 71.79487179 | |
| 40 72.50000000 | |
| 41 70.73170732 | |
| 42 71.42857143 | |
| 43 69.76744186 | |
| 44 68.18181818 | |
| 45 68.88888889 | |
| 46 69.56521739 | |
| 47 70.21276596 | |
| 48 70.83333333 | |
| 49 71.42857143 | |
| 50 70.00000000 | |
| 51 68.62745098 | |
| 52 69.23076923 | |
| 53 69.81132075 | |
| 54 68.51851852 | |
| 55 67.27272727 | |
| 56 67.85714286 | |
| 57 68.42105263 | |
| 58 68.96551724 | |
| 59 69.49152542 | |
| 60 68.33333333 | |
| 61 68.85245902 | |
| 62 69.35483871 | |
| 63 69.84126984 | |
| 64 70.31250000 | |
| 65 69.23076923 | |
| 66 69.69696970 | |
| 67 70.14925373 | |
| 68 70.58823529 | |
| 69 71.01449275 | |
| 70 71.42857143 | |
| 71 71.83098592 | |
| 72 70.83333333 | |
| 73 71.23287671 | |
| 74 71.62162162 | |
| 75 72.00000000 | |
| 76 72.36842105 | |
| 77 72.72727273 | |
| 78 73.07692308 | |
| 79 73.41772152 | |
| 80 73.75000000 | |
| 81 74.07407407 | |
| 82 74.39024390 | |
| 83 74.69879518 | |
| 84 75.00000000 | |
| 85 74.11764706 | |
| 86 74.41860465 | |
| 87 74.71264368 | |
| 88 75.00000000 | |
| 89 75.28089888 | |
| 90 75.55555556 | |
| 91 75.82417582 | |
| 92 76.08695652 | |
| 93 75.26881720 | |
| 94 74.46808511 | |
| 95 74.73684211 | |
| 96 75.00000000 | |
| 97 74.22680412 | |
| 98 74.48979592 | |
| 99 74.74747475 | |
| 100 75.00000000 | |
| 101 75.24752475 | |
| 102 75.49019608 | |
| 103 74.75728155 | |
| 104 74.03846154 | |
| 105 74.28571429 | |
| 106 73.58490566 | |
| 107 73.83177570 | |
| 108 73.14814815 | |
| 109 72.47706422 | |
| 110 72.72727273 | |
| 111 72.97297297 | |
| 112 73.21428571 | |
| 113 72.56637168 | |
| 114 72.80701754 | |
| 115 73.04347826 | |
| 116 73.27586207 | |
| 117 72.64957265 | |
| 118 72.88135593 | |
| 119 72.26890756 | |
| 120 72.50000000 | |
| 121 71.90082645 | |
| 122 72.13114754 | |
| 123 72.35772358 | |
| 124 72.58064516 | |
| 125 72.80000000 | |
| 126 73.01587302 | |
| 127 73.22834646 | |
| 128 73.43750000 | |
| 129 72.86821705 | |
| 130 73.07692308 | |
| 131 73.28244275 | |
| 132 72.72727273 | |
| 133 72.93233083 | |
| 134 73.13432836 | |
| 135 72.59259259 | |
| 136 72.79411765 | |
| 137 72.99270073 | |
| 138 72.46376812 | |
| 139 72.66187050 | |
| 140 72.14285714 | |
| 141 71.63120567 | |
| 142 71.83098592 | |
| 143 72.02797203 | |
| 144 72.22222222 | |
| 145 72.41379310 | |
| 146 72.60273973 | |
| 147 72.78911565 | |
| 148 72.29729730 | |
| 149 72.48322148 | |
| 150 72.66666667 | |
| 151 72.84768212 | |
| 152 73.02631579 | |
| 153 73.20261438 | |
| 154 73.37662338 | |
| 155 72.90322581 | |
| 156 73.07692308 | |
| 157 73.24840764 | |
| 158 73.41772152 | |
| 159 73.58490566 | |
| 160 73.75000000 | |
| 161 73.91304348 | |
| 162 74.07407407 | |
| 163 73.61963190 | |
| 164 73.78048780 | |
| 165 73.93939394 | |
| 166 74.09638554 | |
| 167 74.25149701 | |
| 168 74.40476190 | |
| 169 73.96449704 | |
| 170 74.11764706 | |
| 171 74.26900585 | |
| 172 73.83720930 | |
| 173 73.98843931 | |
| 174 74.13793103 | |
| 175 73.71428571 | |
| 176 73.86363636 | |
| 177 74.01129944 | |
| 178 74.15730337 | |
| 179 74.30167598 | |
| 180 74.44444444 | |
| 181 74.58563536 | |
| 182 74.72527473 | |
| 183 74.31693989 | |
| 184 74.45652174 | |
| 185 74.59459459 | |
| 186 74.73118280 | |
| 187 74.86631016 | |
| 188 74.46808511 | |
| 189 74.60317460 | |
| 190 74.21052632 | |
| 191 74.34554974 | |
| 192 73.95833333 | |
| 193 73.57512953 | |
| 194 73.71134021 | |
| 195 73.84615385 | |
| 196 73.97959184 | |
| 197 74.11167513 | |
| 198 74.24242424 | |
| 199 74.37185930 | |
| 200 74.50000000 | |
| 201 74.62686567 | |
| 202 74.75247525 | |
| 203 74.87684729 | |
| 204 75.00000000 | |
| 205 75.12195122 | |
| 206 74.75728155 | |
| 207 74.87922705 | |
| 208 75.00000000 | |
| 209 75.11961722 | |
| 210 75.23809524 | |
| 211 75.35545024 | |
| 212 75.00000000 | |
| 213 75.11737089 | |
| 214 75.23364486 | |
| 215 75.34883721 | |
| 216 75.00000000 | |
| 217 75.11520737 | |
| 218 75.22935780 | |
| 219 75.34246575 | |
| 220 75.00000000 | |
| 221 75.11312217 | |
| 222 75.22522523 | |
| 223 74.88789238 | |
| 224 75.00000000 | |
| 225 74.66666667 | |
| 226 74.33628319 | |
| 227 74.44933921 | |
| 228 74.56140351 | |
| 229 74.67248908 | |
| 230 74.78260870 | |
| 231 74.45887446 | |
| 232 74.13793103 | |
| 233 73.81974249 | |
| 234 73.50427350 | |
| 235 73.19148936 | |
| 236 73.30508475 | |
| 237 72.99578059 | |
| 238 72.68907563 | |
| 239 72.80334728 | |
| 240 72.91666667 | |
| 241 73.02904564 | |
| 242 73.14049587 | |
| 243 73.25102881 | |
| 244 72.95081967 | |
| 245 73.06122449 | |
| 246 73.17073171 | |
| 247 73.27935223 | |
| 248 73.38709677 | |
| 249 73.49397590 | |
| 250 73.60000000 | |
| 251 73.70517928 | |
| 252 73.80952381 | |
| 253 73.91304348 | |
| 254 74.01574803 | |
| 255 74.11764706 | |
| 256 74.21875000 | |
| 257 73.92996109 | |
| 258 74.03100775 | |
| 259 74.13127413 | |
| 260 74.23076923 | |
| 261 74.32950192 | |
| 262 74.42748092 | |
| 263 74.52471483 | |
| 264 74.62121212 | |
| 265 74.71698113 | |
| 266 74.43609023 | |
| 267 74.53183521 | |
| 268 74.62686567 | |
| 269 74.72118959 | |
| 270 74.81481481 | |
| 271 74.90774908 | |
| 272 75.00000000 | |
| 273 74.72527473 | |
| 274 74.45255474 | |
| 275 74.18181818 | |
| 276 74.27536232 | |
| 277 74.36823105 | |
| 278 74.46043165 | |
| 279 74.55197133 | |
| 280 74.64285714 | |
| 281 74.73309609 | |
| 282 74.82269504 | |
| 283 74.91166078 | |
| 284 75.00000000 | |
| 285 75.08771930 | |
| 286 75.17482517 | |
| 287 74.91289199 | |
| 288 75.00000000 | |
| 289 74.74048443 | |
| 290 74.82758621 | |
| 291 74.91408935 | |
| 292 75.00000000 | |
| 293 75.08532423 | |
| 294 74.82993197 | |
| 295 74.91525424 | |
| 296 75.00000000 | |
| 297 75.08417508 | |
| 298 75.16778523 | |
| 299 75.25083612 | |
| 300 75.33333333 | |
| 301 75.08305648 | |
| 302 75.16556291 | |
| 303 75.24752475 | |
| 304 75.32894737 | |
| 305 75.40983607 | |
| 306 75.16339869 | |
| 307 75.24429967 | |
| 308 75.32467532 | |
| 309 75.40453074 | |
| 310 75.16129032 | |
| 311 75.24115756 | |
| 312 75.32051282 | |
| 313 75.39936102 | |
| 314 75.47770701 | |
| 315 75.55555556 | |
| 316 75.63291139 | |
| 317 75.70977918 | |
| 318 75.78616352 | |
| 319 75.54858934 | |
| 320 75.31250000 | |
| 321 75.38940810 | |
| 322 75.46583851 | |
| 323 75.54179567 | |
| 324 75.61728395 | |
| 325 75.69230769 | |
| 326 75.76687117 | |
| 327 75.53516820 | |
| 328 75.60975610 | |
| 329 75.68389058 | |
| 330 75.45454545 | |
| 331 75.52870091 | |
| 332 75.60240964 | |
| 333 75.67567568 | |
| 334 75.74850299 | |
| 335 75.52238806 | |
| 336 75.29761905 | |
| 337 75.37091988 | |
| 338 75.14792899 | |
| 339 75.22123894 | |
| 340 75.29411765 | |
| 341 75.36656891 | |
| 342 75.43859649 | |
| 343 75.51020408 | |
| 344 75.58139535 | |
| 345 75.36231884 | |
| 346 75.14450867 | |
| 347 75.21613833 | |
| 348 75.00000000 | |
| 349 75.07163324 | |
| 350 75.14285714 | |
| 351 75.21367521 | |
| 352 75.28409091 | |
| 353 75.35410765 | |
| 354 75.14124294 | |
| 355 74.92957746 | |
| 356 75.00000000 | |
| 357 75.07002801 | |
| 358 75.13966480 | |
| 359 74.93036212 | |
| 360 75.00000000 | |
| 361 75.06925208 | |
| 362 75.13812155 | |
| 363 75.20661157 | |
| 364 75.27472527 | |
| 365 75.34246575 | |
| 366 75.13661202 | |
| 367 75.20435967 | |
| 368 75.27173913 | |
| 369 75.33875339 | |
| 370 75.40540541 | |
| 371 75.20215633 | |
| 372 75.00000000 | |
| 373 75.06702413 | |
| 374 75.13368984 | |
| 375 74.93333333 | |
| 376 75.00000000 | |
| 377 74.80106101 | |
| 378 74.86772487 | |
| 379 74.67018470 | |
| 380 74.73684211 | |
| 381 74.54068241 | |
| 382 74.34554974 | |
| 383 74.15143603 | |
| 384 74.21875000 | |
| 385 74.28571429 | |
| 386 74.09326425 | |
| 387 74.16020672 | |
| 388 74.22680412 | |
| 389 74.29305913 | |
| 390 74.10256410 | |
| 391 73.91304348 | |
| 392 73.97959184 | |
| 393 74.04580153 | |
| 394 73.85786802 | |
| 395 73.92405063 | |
| 396 73.73737374 | |
| 397 73.80352645 | |
| 398 73.61809045 | |
| 399 73.68421053 | |
| 400 73.75000000 | |
| 401 73.81546135 | |
| 402 73.88059701 | |
| 403 73.94540943 | |
| 404 74.00990099 | |
| 405 74.07407407 | |
| 406 74.13793103 | |
| 407 74.20147420 | |
| 408 74.26470588 | |
| 409 74.08312958 | |
| 410 74.14634146 | |
| 411 74.20924574 | |
| 412 74.27184466 | |
| 413 74.33414044 | |
| 414 74.39613527 | |
| 415 74.45783133 | |
| 416 74.51923077 | |
| 417 74.34052758 | |
| 418 74.40191388 | |
| 419 74.22434368 | |
| 420 74.28571429 | |
| 421 74.34679335 | |
| 422 74.40758294 | |
| 423 74.46808511 | |
| 424 74.52830189 | |
| 425 74.58823529 | |
| 426 74.41314554 | |
| 427 74.23887588 | |
| 428 74.06542056 | |
| 429 74.12587413 | |
| 430 73.95348837 | |
| 431 74.01392111 | |
| 432 74.07407407 | |
| 433 73.90300231 | |
| 434 73.96313364 | |
| 435 74.02298851 | |
| 436 74.08256881 | |
| 437 74.14187643 | |
| 438 73.97260274 | |
| 439 74.03189066 | |
| 440 74.09090909 | |
| 441 74.14965986 | |
| 442 73.98190045 | |
| 443 74.04063205 | |
| 444 74.09909910 | |
| 445 74.15730337 | |
| 446 74.21524664 | |
| 447 74.27293065 | |
| 448 74.33035714 | |
| 449 74.38752784 | |
| 450 74.22222222 | |
| 451 74.27937916 | |
| 452 74.33628319 | |
| 453 74.39293598 | |
| 454 74.22907489 | |
| 455 74.28571429 | |
| 456 74.34210526 | |
| 457 74.39824945 | |
| 458 74.45414847 | |
| 459 74.50980392 | |
| 460 74.34782609 | |
| 461 74.40347072 | |
| 462 74.45887446 | |
| 463 74.29805616 | |
| 464 74.35344828 | |
| 465 74.40860215 | |
| 466 74.46351931 | |
| 467 74.51820128 | |
| 468 74.57264957 | |
| 469 74.41364606 | |
| 470 74.46808511 | |
| 471 74.52229299 | |
| 472 74.57627119 | |
| 473 74.41860465 | |
| 474 74.47257384 | |
| 475 74.31578947 | |
| 476 74.36974790 | |
| 477 74.42348008 | |
| 478 74.47698745 | |
| 479 74.53027140 | |
| 480 74.58333333 | |
| 481 74.63617464 | |
| 482 74.68879668 | |
| 483 74.74120083 | |
| 484 74.79338843 | |
| 485 74.84536082 | |
| 486 74.89711934 | |
| 487 74.74332649 | |
| 488 74.79508197 | |
| 489 74.84662577 | |
| 490 74.89795918 | |
| 491 74.94908350 | |
| 492 75.00000000 | |
| 493 74.84787018 | |
| 494 74.89878543 | |
| 495 74.94949495 | |
| 496 74.79838710 | |
| 497 74.64788732 | |
| 498 74.69879518 | |
| 499 74.74949900 | |
| 500 74.80000000 | |
| 501 74.85029940 | |
| 502 74.90039841 | |
| 503 74.95029821 | |
| 504 74.80158730 | |
| 505 74.85148515 | |
| 506 74.70355731 | |
| 507 74.75345168 | |
| 508 74.80314961 | |
| 509 74.65618861 | |
| 510 74.70588235 | |
| 511 74.55968689 | |
| 512 74.60937500 | |
| 513 74.46393762 | |
| 514 74.51361868 | |
| 515 74.56310680 | |
| 516 74.41860465 | |
| 517 74.46808511 | |
| 518 74.51737452 | |
| 519 74.56647399 | |
| 520 74.42307692 | |
| 521 74.47216891 | |
| 522 74.52107280 | |
| 523 74.37858509 | |
| 524 74.23664122 | |
| 525 74.09523810 | |
| 526 73.95437262 | |
| 527 73.81404175 | |
| 528 73.67424242 | |
| 529 73.53497164 | |
| 530 73.58490566 | |
| 531 73.63465160 | |
| 532 73.68421053 | |
| 533 73.73358349 | |
| 534 73.59550562 | |
| 535 73.45794393 | |
| 536 73.50746269 | |
| 537 73.55679702 | |
| 538 73.42007435 | |
| 539 73.46938776 | |
| 540 73.33333333 | |
| 541 73.19778189 | |
| 542 73.24723247 | |
| 543 73.29650092 | |
| 544 73.34558824 | |
| 545 73.39449541 | |
| 546 73.44322344 | |
| 547 73.49177331 | |
| 548 73.54014599 | |
| 549 73.58834244 | |
| 550 73.63636364 | |
| 551 73.50272232 | |
| 552 73.36956522 | |
| 553 73.41772152 | |
| 554 73.46570397 | |
| 555 73.51351351 | |
| 556 73.38129496 | |
| 557 73.42908438 | |
| 558 73.29749104 | |
| 559 73.34525939 | |
| 560 73.39285714 | |
| 561 73.44028520 | |
| 562 73.48754448 | |
| 563 73.53463588 | |
| 564 73.58156028 | |
| 565 73.45132743 | |
| 566 73.49823322 | |
| 567 73.54497354 | |
| 568 73.59154930 | |
| 569 73.46221441 | |
| 570 73.33333333 | |
| 571 73.38003503 | |
| 572 73.25174825 | |
| 573 73.29842932 | |
| 574 73.34494774 | |
| 575 73.39130435 | |
| 576 73.43750000 | |
| 577 73.48353553 | |
| 578 73.52941176 | |
| 579 73.57512953 | |
| 580 73.62068966 | |
| 581 73.49397590 | |
| 582 73.53951890 | |
| 583 73.58490566 | |
| 584 73.63013699 | |
| 585 73.67521368 | |
| 586 73.72013652 | |
| 587 73.76490630 | |
| 588 73.80952381 | |
| 589 73.85398981 | |
| 590 73.89830508 | |
| 591 73.94247039 | |
| 592 73.98648649 | |
| 593 74.03035413 | |
| 594 74.07407407 | |
| 595 74.11764706 | |
| 596 73.99328859 | |
| 597 74.03685092 | |
| 598 74.08026756 | |
| 599 73.95659432 | |
| 600 74.00000000 | |
| 601 74.04326123 | |
| 602 73.92026578 | |
| 603 73.79767828 | |
| 604 73.84105960 | |
| 605 73.88429752 | |
| 606 73.92739274 | |
| 607 73.97034596 | |
| 608 74.01315789 | |
| 609 73.89162562 | |
| 610 73.93442623 | |
| 611 73.81342062 | |
| 612 73.85620915 | |
| 613 73.73572594 | |
| 614 73.77850163 | |
| 615 73.65853659 | |
| 616 73.53896104 | |
| 617 73.58184765 | |
| 618 73.62459547 | |
| 619 73.50565428 | |
| 620 73.54838710 | |
| 621 73.59098229 | |
| 622 73.63344051 | |
| 623 73.67576244 | |
| 624 73.71794872 | |
| 625 73.76000000 | |
| 626 73.80191693 | |
| 627 73.84370016 | |
| 628 73.72611465 | |
| 629 73.76788553 | |
| 630 73.80952381 | |
| 631 73.85103011 | |
| 632 73.73417722 | |
| 633 73.61769352 | |
| 634 73.65930599 | |
| 635 73.70078740 | |
| 636 73.74213836 | |
| 637 73.78335950 | |
| 638 73.82445141 | |
| 639 73.86541471 | |
| 640 73.90625000 | |
| 641 73.94695788 | |
| 642 73.83177570 | |
| 643 73.71695179 | |
| 644 73.75776398 | |
| 645 73.64341085 | |
| 646 73.68421053 | |
| 647 73.57032457 | |
| 648 73.61111111 | |
| 649 73.49768875 | |
| 650 73.53846154 | |
| 651 73.57910906 | |
| 652 73.46625767 | |
| 653 73.50689127 | |
| 654 73.39449541 | |
| 655 73.28244275 | |
| 656 73.17073171 | |
| 657 73.21156773 | |
| 658 73.25227964 | |
| 659 73.14112291 | |
| 660 73.18181818 | |
| 661 73.07110439 | |
| 662 73.11178248 | |
| 663 73.15233786 | |
| 664 73.19277108 | |
| 665 73.23308271 | |
| 666 73.27327327 | |
| 667 73.31334333 | |
| 668 73.20359281 | |
| 669 73.24364723 | |
| 670 73.13432836 | |
| 671 73.17436662 | |
| 672 73.21428571 | |
| 673 73.25408618 | |
| 674 73.29376855 | |
| 675 73.33333333 | |
| 676 73.37278107 | |
| 677 73.41211226 | |
| 678 73.45132743 | |
| 679 73.49042710 | |
| 680 73.52941176 | |
| 681 73.42143906 | |
| 682 73.46041056 | |
| 683 73.35285505 | |
| 684 73.39181287 | |
| 685 73.43065693 | |
| 686 73.46938776 | |
| 687 73.50800582 | |
| 688 73.54651163 | |
| 689 73.43976778 | |
| 690 73.47826087 | |
| 691 73.37192475 | |
| 692 73.26589595 | |
| 693 73.30447330 | |
| 694 73.34293948 | |
| 695 73.23741007 | |
| 696 73.27586207 | |
| 697 73.31420373 | |
| 698 73.20916905 | |
| 699 73.10443491 | |
| 700 73.00000000 | |
| 701 72.89586305 | |
| 702 72.93447293 | |
| 703 72.97297297 | |
| 704 73.01136364 | |
| 705 73.04964539 | |
| 706 73.08781870 | |
| 707 73.12588402 | |
| 708 73.16384181 | |
| 709 73.20169252 | |
| 710 73.23943662 | |
| 711 73.27707454 | |
| 712 73.31460674 | |
| 713 73.35203366 | |
| 714 73.24929972 | |
| 715 73.14685315 | |
| 716 73.18435754 | |
| 717 73.22175732 | |
| 718 73.11977716 | |
| 719 73.01808067 | |
| 720 73.05555556 | |
| 721 72.95423024 | |
| 722 72.99168975 | |
| 723 73.02904564 | |
| 724 72.92817680 | |
| 725 72.96551724 | |
| 726 72.86501377 | |
| 727 72.90233838 | |
| 728 72.93956044 | |
| 729 72.97668038 | |
| 730 73.01369863 | |
| 731 73.05061560 | |
| 732 73.08743169 | |
| 733 73.12414734 | |
| 734 73.16076294 | |
| 735 73.19727891 | |
| 736 73.09782609 | |
| 737 72.99864315 | |
| 738 73.03523035 | |
| 739 72.93640054 | |
| 740 72.83783784 | |
| 741 72.87449393 | |
| 742 72.91105121 | |
| 743 72.94751009 | |
| 744 72.84946237 | |
| 745 72.88590604 | |
| 746 72.92225201 | |
| 747 72.82463186 | |
| 748 72.72727273 | |
| 749 72.63017356 | |
| 750 72.66666667 | |
| Final result: 72.6667 +/- 1.6284 | |
| Random chance: 25.0083 +/- 1.5824 | |