| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 26.31578947 | |
| 20 25.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 30.43478261 | |
| 24 33.33333333 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 28.57142857 | |
| 29 31.03448276 | |
| 30 30.00000000 | |
| 31 29.03225806 | |
| 32 31.25000000 | |
| 33 33.33333333 | |
| 34 32.35294118 | |
| 35 31.42857143 | |
| 36 30.55555556 | |
| 37 32.43243243 | |
| 38 31.57894737 | |
| 39 33.33333333 | |
| 40 32.50000000 | |
| 41 31.70731707 | |
| 42 30.95238095 | |
| 43 30.23255814 | |
| 44 31.81818182 | |
| 45 33.33333333 | |
| 46 32.60869565 | |
| 47 34.04255319 | |
| 48 35.41666667 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 37.25490196 | |
| 52 36.53846154 | |
| 53 35.84905660 | |
| 54 35.18518519 | |
| 55 36.36363636 | |
| 56 35.71428571 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 35.48387097 | |
| 63 34.92063492 | |
| 64 35.93750000 | |
| 65 35.38461538 | |
| 66 36.36363636 | |
| 67 35.82089552 | |
| 68 35.29411765 | |
| 69 36.23188406 | |
| 70 35.71428571 | |
| 71 35.21126761 | |
| 72 36.11111111 | |
| 73 35.61643836 | |
| 74 35.13513514 | |
| 75 34.66666667 | |
| 76 35.52631579 | |
| 77 35.06493506 | |
| 78 34.61538462 | |
| 79 34.17721519 | |
| 80 35.00000000 | |
| 81 34.56790123 | |
| 82 35.36585366 | |
| 83 34.93975904 | |
| 84 34.52380952 | |
| 85 34.11764706 | |
| 86 33.72093023 | |
| 87 33.33333333 | |
| 88 32.95454545 | |
| 89 33.70786517 | |
| 90 33.33333333 | |
| 91 32.96703297 | |
| 92 32.60869565 | |
| 93 32.25806452 | |
| 94 31.91489362 | |
| 95 31.57894737 | |
| 96 32.29166667 | |
| 97 32.98969072 | |
| 98 32.65306122 | |
| 99 33.33333333 | |
| 100 34.00000000 | |
| 101 34.65346535 | |
| 102 34.31372549 | |
| 103 34.95145631 | |
| 104 34.61538462 | |
| 105 34.28571429 | |
| 106 34.90566038 | |
| 107 35.51401869 | |
| 108 36.11111111 | |
| 109 36.69724771 | |
| 110 37.27272727 | |
| 111 36.93693694 | |
| 112 37.50000000 | |
| 113 37.16814159 | |
| 114 37.71929825 | |
| 115 37.39130435 | |
| 116 37.93103448 | |
| 117 38.46153846 | |
| 118 38.13559322 | |
| 119 37.81512605 | |
| 120 37.50000000 | |
| 121 38.01652893 | |
| 122 37.70491803 | |
| 123 38.21138211 | |
| 124 37.90322581 | |
| 125 38.40000000 | |
| 126 38.09523810 | |
| 127 37.79527559 | |
| 128 37.50000000 | |
| 129 37.98449612 | |
| 130 38.46153846 | |
| 131 38.16793893 | |
| 132 37.87878788 | |
| 133 37.59398496 | |
| 134 37.31343284 | |
| 135 37.03703704 | |
| 136 36.76470588 | |
| 137 37.22627737 | |
| 138 37.68115942 | |
| 139 38.12949640 | |
| 140 37.85714286 | |
| 141 38.29787234 | |
| 142 38.02816901 | |
| 143 37.76223776 | |
| 144 38.19444444 | |
| 145 38.62068966 | |
| 146 38.35616438 | |
| 147 38.09523810 | |
| 148 37.83783784 | |
| 149 37.58389262 | |
| 150 37.33333333 | |
| 151 37.74834437 | |
| 152 37.50000000 | |
| 153 37.25490196 | |
| 154 37.01298701 | |
| 155 36.77419355 | |
| 156 37.17948718 | |
| 157 37.57961783 | |
| 158 37.97468354 | |
| 159 37.73584906 | |
| 160 37.50000000 | |
| 161 37.88819876 | |
| 162 37.65432099 | |
| 163 38.03680982 | |
| 164 37.80487805 | |
| 165 38.18181818 | |
| 166 38.55421687 | |
| 167 38.32335329 | |
| 168 38.09523810 | |
| 169 37.86982249 | |
| 170 37.64705882 | |
| 171 37.42690058 | |
| 172 37.20930233 | |
| 173 36.99421965 | |
| 174 37.35632184 | |
| 175 37.14285714 | |
| 176 36.93181818 | |
| 177 36.72316384 | |
| 178 36.51685393 | |
| 179 36.31284916 | |
| 180 36.11111111 | |
| 181 35.91160221 | |
| 182 35.71428571 | |
| 183 35.51912568 | |
| 184 35.32608696 | |
| 185 35.67567568 | |
| 186 36.02150538 | |
| 187 35.82887701 | |
| 188 35.63829787 | |
| 189 35.44973545 | |
| 190 35.26315789 | |
| 191 35.07853403 | |
| 192 34.89583333 | |
| 193 34.71502591 | |
| 194 35.05154639 | |
| 195 34.87179487 | |
| 196 34.69387755 | |
| 197 35.02538071 | |
| 198 35.35353535 | |
| 199 35.17587940 | |
| 200 35.50000000 | |
| 201 35.32338308 | |
| 202 35.64356436 | |
| 203 35.46798030 | |
| 204 35.29411765 | |
| 205 35.60975610 | |
| 206 35.43689320 | |
| 207 35.74879227 | |
| 208 35.57692308 | |
| 209 35.40669856 | |
| 210 35.23809524 | |
| 211 35.07109005 | |
| 212 34.90566038 | |
| 213 34.74178404 | |
| 214 34.57943925 | |
| 215 34.41860465 | |
| 216 34.72222222 | |
| 217 34.56221198 | |
| 218 34.40366972 | |
| 219 34.24657534 | |
| 220 34.09090909 | |
| 221 34.38914027 | |
| 222 34.23423423 | |
| 223 34.08071749 | |
| 224 34.37500000 | |
| 225 34.66666667 | |
| 226 34.51327434 | |
| 227 34.80176211 | |
| 228 34.64912281 | |
| 229 34.49781659 | |
| 230 34.34782609 | |
| 231 34.19913420 | |
| 232 34.48275862 | |
| 233 34.33476395 | |
| 234 34.18803419 | |
| 235 34.04255319 | |
| 236 34.32203390 | |
| 237 34.17721519 | |
| 238 34.45378151 | |
| 239 34.30962343 | |
| 240 34.16666667 | |
| 241 34.02489627 | |
| 242 33.88429752 | |
| 243 33.74485597 | |
| 244 33.60655738 | |
| 245 33.46938776 | |
| 246 33.33333333 | |
| 247 33.60323887 | |
| 248 33.87096774 | |
| 249 34.13654618 | |
| 250 34.00000000 | |
| 251 34.26294821 | |
| 252 34.12698413 | |
| 253 33.99209486 | |
| 254 34.25196850 | |
| 255 34.50980392 | |
| 256 34.37500000 | |
| 257 34.24124514 | |
| 258 34.10852713 | |
| 259 33.97683398 | |
| 260 33.84615385 | |
| 261 34.09961686 | |
| 262 33.96946565 | |
| 263 33.84030418 | |
| 264 34.09090909 | |
| 265 34.33962264 | |
| 266 34.21052632 | |
| 267 34.45692884 | |
| 268 34.32835821 | |
| 269 34.20074349 | |
| 270 34.07407407 | |
| 271 33.94833948 | |
| 272 33.82352941 | |
| 273 33.69963370 | |
| 274 33.94160584 | |
| 275 33.81818182 | |
| 276 33.69565217 | |
| 277 33.57400722 | |
| 278 33.45323741 | |
| 279 33.69175627 | |
| 280 33.92857143 | |
| 281 34.16370107 | |
| 282 34.04255319 | |
| 283 33.92226148 | |
| 284 34.15492958 | |
| 285 34.03508772 | |
| 286 34.26573427 | |
| 287 34.49477352 | |
| 288 34.72222222 | |
| 289 34.60207612 | |
| 290 34.48275862 | |
| 291 34.70790378 | |
| 292 34.58904110 | |
| 293 34.81228669 | |
| 294 34.69387755 | |
| 295 34.91525424 | |
| 296 35.13513514 | |
| 297 35.01683502 | |
| 298 34.89932886 | |
| 299 34.78260870 | |
| 300 34.66666667 | |
| 301 34.55149502 | |
| 302 34.43708609 | |
| 303 34.65346535 | |
| 304 34.53947368 | |
| 305 34.42622951 | |
| 306 34.31372549 | |
| 307 34.20195440 | |
| 308 34.41558442 | |
| 309 34.30420712 | |
| 310 34.19354839 | |
| 311 34.08360129 | |
| 312 34.29487179 | |
| 313 34.18530351 | |
| 314 34.07643312 | |
| 315 34.28571429 | |
| 316 34.17721519 | |
| 317 34.06940063 | |
| 318 33.96226415 | |
| 319 33.85579937 | |
| 320 33.75000000 | |
| 321 33.64485981 | |
| 322 33.54037267 | |
| 323 33.74613003 | |
| 324 33.64197531 | |
| 325 33.53846154 | |
| 326 33.43558282 | |
| 327 33.63914373 | |
| 328 33.84146341 | |
| 329 33.73860182 | |
| 330 33.93939394 | |
| 331 33.83685801 | |
| 332 34.03614458 | |
| 333 33.93393393 | |
| 334 33.83233533 | |
| 335 34.02985075 | |
| 336 34.22619048 | |
| 337 34.12462908 | |
| 338 34.02366864 | |
| 339 34.21828909 | |
| 340 34.11764706 | |
| 341 34.31085044 | |
| 342 34.50292398 | |
| 343 34.40233236 | |
| 344 34.59302326 | |
| 345 34.78260870 | |
| 346 34.68208092 | |
| 347 34.87031700 | |
| 348 34.77011494 | |
| 349 34.67048711 | |
| 350 34.57142857 | |
| 351 34.47293447 | |
| 352 34.37500000 | |
| 353 34.27762040 | |
| 354 34.18079096 | |
| 355 34.36619718 | |
| 356 34.26966292 | |
| 357 34.45378151 | |
| 358 34.63687151 | |
| 359 34.54038997 | |
| 360 34.44444444 | |
| 361 34.62603878 | |
| 362 34.53038674 | |
| 363 34.43526171 | |
| 364 34.34065934 | |
| 365 34.24657534 | |
| 366 34.15300546 | |
| 367 34.33242507 | |
| 368 34.51086957 | |
| 369 34.41734417 | |
| 370 34.32432432 | |
| 371 34.23180593 | |
| 372 34.40860215 | |
| 373 34.31635389 | |
| 374 34.49197861 | |
| 375 34.40000000 | |
| 376 34.30851064 | |
| 377 34.48275862 | |
| 378 34.39153439 | |
| 379 34.30079156 | |
| 380 34.47368421 | |
| 381 34.38320210 | |
| 382 34.55497382 | |
| 383 34.46475196 | |
| 384 34.37500000 | |
| 385 34.28571429 | |
| 386 34.19689119 | |
| 387 34.10852713 | |
| 388 34.27835052 | |
| 389 34.19023136 | |
| 390 34.10256410 | |
| 391 34.01534527 | |
| 392 33.92857143 | |
| 393 33.84223919 | |
| 394 34.01015228 | |
| 395 33.92405063 | |
| 396 33.83838384 | |
| 397 34.00503778 | |
| 398 33.91959799 | |
| 399 33.83458647 | |
| 400 33.75000000 | |
| 401 33.91521197 | |
| 402 34.07960199 | |
| 403 34.24317618 | |
| 404 34.15841584 | |
| 405 34.07407407 | |
| 406 33.99014778 | |
| 407 33.90663391 | |
| 408 33.82352941 | |
| 409 33.74083130 | |
| 410 33.65853659 | |
| 411 33.57664234 | |
| 412 33.73786408 | |
| 413 33.89830508 | |
| 414 34.05797101 | |
| 415 34.21686747 | |
| 416 34.13461538 | |
| 417 34.05275779 | |
| 418 33.97129187 | |
| 419 33.89021480 | |
| 420 34.04761905 | |
| 421 34.20427553 | |
| 422 34.12322275 | |
| 423 34.04255319 | |
| 424 33.96226415 | |
| 425 33.88235294 | |
| 426 33.80281690 | |
| 427 33.72365340 | |
| 428 33.64485981 | |
| 429 33.56643357 | |
| 430 33.72093023 | |
| 431 33.64269142 | |
| 432 33.79629630 | |
| 433 33.94919169 | |
| 434 34.10138249 | |
| 435 34.02298851 | |
| 436 33.94495413 | |
| 437 33.86727689 | |
| 438 33.78995434 | |
| 439 33.94077449 | |
| 440 33.86363636 | |
| 441 33.78684807 | |
| 442 33.71040724 | |
| 443 33.63431151 | |
| 444 33.78378378 | |
| 445 33.70786517 | |
| 446 33.63228700 | |
| 447 33.55704698 | |
| 448 33.48214286 | |
| 449 33.40757238 | |
| 450 33.33333333 | |
| 451 33.25942350 | |
| 452 33.18584071 | |
| 453 33.33333333 | |
| 454 33.25991189 | |
| 455 33.18681319 | |
| 456 33.11403509 | |
| 457 33.04157549 | |
| 458 32.96943231 | |
| 459 32.89760349 | |
| 460 32.82608696 | |
| 461 32.75488069 | |
| 462 32.68398268 | |
| 463 32.82937365 | |
| 464 32.97413793 | |
| 465 33.11827957 | |
| 466 33.04721030 | |
| 467 32.97644540 | |
| 468 32.90598291 | |
| 469 32.83582090 | |
| 470 32.76595745 | |
| 471 32.90870488 | |
| 472 32.83898305 | |
| 473 32.76955603 | |
| 474 32.91139241 | |
| 475 33.05263158 | |
| 476 32.98319328 | |
| 477 33.12368973 | |
| 478 33.05439331 | |
| 479 32.98538622 | |
| 480 32.91666667 | |
| 481 32.84823285 | |
| 482 32.98755187 | |
| 483 32.91925466 | |
| 484 33.05785124 | |
| 485 33.19587629 | |
| 486 33.12757202 | |
| 487 33.05954825 | |
| 488 32.99180328 | |
| 489 33.12883436 | |
| 490 33.26530612 | |
| 491 33.40122200 | |
| 492 33.53658537 | |
| 493 33.46855984 | |
| 494 33.60323887 | |
| 495 33.53535354 | |
| 496 33.46774194 | |
| 497 33.40040241 | |
| 498 33.33333333 | |
| 499 33.46693387 | |
| 500 33.40000000 | |
| 501 33.33333333 | |
| 502 33.26693227 | |
| 503 33.20079523 | |
| 504 33.33333333 | |
| 505 33.26732673 | |
| 506 33.20158103 | |
| 507 33.33333333 | |
| 508 33.46456693 | |
| 509 33.59528487 | |
| 510 33.52941176 | |
| 511 33.46379648 | |
| 512 33.39843750 | |
| 513 33.33333333 | |
| 514 33.26848249 | |
| 515 33.39805825 | |
| 516 33.33333333 | |
| 517 33.46228240 | |
| 518 33.39768340 | |
| 519 33.33333333 | |
| 520 33.26923077 | |
| 521 33.20537428 | |
| 522 33.33333333 | |
| 523 33.26959847 | |
| 524 33.20610687 | |
| 525 33.33333333 | |
| 526 33.26996198 | |
| 527 33.39658444 | |
| 528 33.33333333 | |
| 529 33.27032136 | |
| 530 33.20754717 | |
| 531 33.14500942 | |
| 532 33.08270677 | |
| 533 33.20825516 | |
| 534 33.14606742 | |
| 535 33.08411215 | |
| 536 33.02238806 | |
| 537 33.14711359 | |
| 538 33.08550186 | |
| 539 33.02411874 | |
| 540 32.96296296 | |
| 541 33.08687616 | |
| 542 33.02583026 | |
| 543 32.96500921 | |
| 544 33.08823529 | |
| 545 33.21100917 | |
| 546 33.15018315 | |
| 547 33.27239488 | |
| 548 33.39416058 | |
| 549 33.33333333 | |
| 550 33.27272727 | |
| 551 33.21234120 | |
| 552 33.33333333 | |
| 553 33.45388788 | |
| 554 33.39350181 | |
| 555 33.51351351 | |
| 556 33.45323741 | |
| 557 33.39317774 | |
| 558 33.33333333 | |
| 559 33.27370304 | |
| 560 33.39285714 | |
| 561 33.51158645 | |
| 562 33.45195730 | |
| 563 33.39253996 | |
| 564 33.51063830 | |
| 565 33.62831858 | |
| 566 33.74558304 | |
| 567 33.68606702 | |
| 568 33.80281690 | |
| 569 33.91915641 | |
| 570 33.85964912 | |
| 571 33.97548161 | |
| 572 33.91608392 | |
| 573 33.85689354 | |
| 574 33.97212544 | |
| 575 34.08695652 | |
| 576 34.02777778 | |
| 577 34.14211438 | |
| 578 34.25605536 | |
| 579 34.36960276 | |
| 580 34.48275862 | |
| 581 34.42340792 | |
| 582 34.53608247 | |
| 583 34.47684391 | |
| 584 34.58904110 | |
| 585 34.52991453 | |
| 586 34.64163823 | |
| 587 34.58262351 | |
| 588 34.52380952 | |
| 589 34.46519525 | |
| 590 34.40677966 | |
| 591 34.34856176 | |
| 592 34.29054054 | |
| 593 34.40134907 | |
| 594 34.51178451 | |
| 595 34.45378151 | |
| 596 34.39597315 | |
| 597 34.50586265 | |
| 598 34.44816054 | |
| 599 34.39065109 | |
| 600 34.33333333 | |
| 601 34.44259567 | |
| 602 34.55149502 | |
| 603 34.66003317 | |
| 604 34.60264901 | |
| 605 34.54545455 | |
| 606 34.65346535 | |
| 607 34.59637562 | |
| 608 34.53947368 | |
| 609 34.64696223 | |
| 610 34.59016393 | |
| 611 34.53355155 | |
| 612 34.64052288 | |
| 613 34.58401305 | |
| 614 34.52768730 | |
| 615 34.47154472 | |
| 616 34.57792208 | |
| 617 34.68395462 | |
| 618 34.62783172 | |
| 619 34.73344103 | |
| 620 34.67741935 | |
| 621 34.62157810 | |
| 622 34.72668810 | |
| 623 34.67094703 | |
| 624 34.61538462 | |
| 625 34.56000000 | |
| 626 34.50479233 | |
| 627 34.60925040 | |
| 628 34.71337580 | |
| 629 34.81717011 | |
| 630 34.76190476 | |
| 631 34.86529319 | |
| 632 34.81012658 | |
| 633 34.91311216 | |
| 634 35.01577287 | |
| 635 34.96062992 | |
| 636 34.90566038 | |
| 637 34.85086342 | |
| 638 34.79623824 | |
| 639 34.74178404 | |
| 640 34.84375000 | |
| 641 34.78939158 | |
| 642 34.89096573 | |
| 643 34.83670295 | |
| 644 34.78260870 | |
| 645 34.88372093 | |
| 646 34.82972136 | |
| 647 34.93044822 | |
| 648 34.87654321 | |
| 649 34.82280431 | |
| 650 34.92307692 | |
| 651 35.02304147 | |
| 652 35.12269939 | |
| 653 35.22205207 | |
| 654 35.16819572 | |
| 655 35.11450382 | |
| 656 35.21341463 | |
| 657 35.31202435 | |
| 658 35.25835866 | |
| 659 35.20485584 | |
| 660 35.15151515 | |
| 661 35.09833585 | |
| 662 35.04531722 | |
| 663 35.14328808 | |
| 664 35.09036145 | |
| 665 35.03759398 | |
| 666 34.98498498 | |
| 667 34.93253373 | |
| 668 34.88023952 | |
| 669 34.82810164 | |
| 670 34.92537313 | |
| 671 34.87332340 | |
| 672 34.82142857 | |
| 673 34.76968796 | |
| 674 34.71810089 | |
| 675 34.81481481 | |
| 676 34.91124260 | |
| 677 34.85967504 | |
| 678 34.80825959 | |
| 679 34.75699558 | |
| 680 34.85294118 | |
| 681 34.94860499 | |
| 682 34.89736070 | |
| 683 34.84626647 | |
| 684 34.94152047 | |
| 685 34.89051095 | |
| 686 34.83965015 | |
| 687 34.78893741 | |
| 688 34.88372093 | |
| 689 34.97822932 | |
| 690 34.92753623 | |
| 691 35.02170767 | |
| 692 35.11560694 | |
| 693 35.06493506 | |
| 694 35.01440922 | |
| 695 35.10791367 | |
| 696 35.20114943 | |
| 697 35.15064562 | |
| 698 35.10028653 | |
| 699 35.05007153 | |
| 700 35.14285714 | |
| 701 35.09272468 | |
| 702 35.04273504 | |
| 703 34.99288762 | |
| 704 34.94318182 | |
| 705 34.89361702 | |
| 706 34.84419263 | |
| 707 34.93635078 | |
| 708 34.88700565 | |
| 709 34.83779972 | |
| 710 34.78873239 | |
| 711 34.88045007 | |
| 712 34.83146067 | |
| 713 34.78260870 | |
| 714 34.87394958 | |
| 715 34.82517483 | |
| 716 34.77653631 | |
| 717 34.86750349 | |
| 718 34.81894150 | |
| 719 34.90959666 | |
| 720 34.86111111 | |
| 721 34.81276006 | |
| 722 34.76454294 | |
| 723 34.71645920 | |
| 724 34.66850829 | |
| 725 34.62068966 | |
| 726 34.71074380 | |
| 727 34.66299862 | |
| 728 34.61538462 | |
| 729 34.70507545 | |
| 730 34.79452055 | |
| 731 34.74692202 | |
| 732 34.83606557 | |
| 733 34.78854025 | |
| 734 34.74114441 | |
| 735 34.69387755 | |
| 736 34.64673913 | |
| 737 34.73541384 | |
| 738 34.68834688 | |
| 739 34.77672530 | |
| 740 34.72972973 | |
| 741 34.81781377 | |
| 742 34.77088949 | |
| 743 34.72409152 | |
| 744 34.81182796 | |
| 745 34.76510067 | |
| 746 34.71849866 | |
| 747 34.80589023 | |
| 748 34.75935829 | |
| 749 34.84646195 | |
| 750 34.93333333 | |
| Final result: 34.9300 +/- 1.6774 | |
| Random chance: 19.5384 +/- 1.7940 | |