| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 41.66666667 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 40.00000000 | |
| 16 43.75000000 | |
| 17 47.05882353 | |
| 18 44.44444444 | |
| 19 47.36842105 | |
| 20 50.00000000 | |
| 21 47.61904762 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 48.00000000 | |
| 26 46.15384615 | |
| 27 48.14814815 | |
| 28 46.42857143 | |
| 29 44.82758621 | |
| 30 43.33333333 | |
| 31 45.16129032 | |
| 32 46.87500000 | |
| 33 45.45454545 | |
| 34 47.05882353 | |
| 35 48.57142857 | |
| 36 47.22222222 | |
| 37 45.94594595 | |
| 38 47.36842105 | |
| 39 46.15384615 | |
| 40 45.00000000 | |
| 41 43.90243902 | |
| 42 42.85714286 | |
| 43 44.18604651 | |
| 44 43.18181818 | |
| 45 44.44444444 | |
| 46 43.47826087 | |
| 47 42.55319149 | |
| 48 43.75000000 | |
| 49 44.89795918 | |
| 50 46.00000000 | |
| 51 45.09803922 | |
| 52 44.23076923 | |
| 53 43.39622642 | |
| 54 42.59259259 | |
| 55 43.63636364 | |
| 56 42.85714286 | |
| 57 43.85964912 | |
| 58 44.82758621 | |
| 59 45.76271186 | |
| 60 45.00000000 | |
| 61 45.90163934 | |
| 62 45.16129032 | |
| 63 44.44444444 | |
| 64 45.31250000 | |
| 65 44.61538462 | |
| 66 43.93939394 | |
| 67 44.77611940 | |
| 68 44.11764706 | |
| 69 43.47826087 | |
| 70 44.28571429 | |
| 71 45.07042254 | |
| 72 45.83333333 | |
| 73 45.20547945 | |
| 74 45.94594595 | |
| 75 46.66666667 | |
| 76 47.36842105 | |
| 77 48.05194805 | |
| 78 48.71794872 | |
| 79 48.10126582 | |
| 80 48.75000000 | |
| 81 49.38271605 | |
| 82 48.78048780 | |
| 83 49.39759036 | |
| 84 48.80952381 | |
| 85 49.41176471 | |
| 86 48.83720930 | |
| 87 49.42528736 | |
| 88 50.00000000 | |
| 89 49.43820225 | |
| 90 48.88888889 | |
| 91 48.35164835 | |
| 92 47.82608696 | |
| 93 47.31182796 | |
| 94 47.87234043 | |
| 95 47.36842105 | |
| 96 47.91666667 | |
| 97 47.42268041 | |
| 98 47.95918367 | |
| 99 47.47474747 | |
| 100 47.00000000 | |
| 101 47.52475248 | |
| 102 47.05882353 | |
| 103 46.60194175 | |
| 104 47.11538462 | |
| 105 47.61904762 | |
| 106 47.16981132 | |
| 107 46.72897196 | |
| 108 46.29629630 | |
| 109 46.78899083 | |
| 110 47.27272727 | |
| 111 47.74774775 | |
| 112 47.32142857 | |
| 113 46.90265487 | |
| 114 47.36842105 | |
| 115 47.82608696 | |
| 116 48.27586207 | |
| 117 47.86324786 | |
| 118 47.45762712 | |
| 119 47.05882353 | |
| 120 46.66666667 | |
| 121 47.10743802 | |
| 122 46.72131148 | |
| 123 46.34146341 | |
| 124 46.77419355 | |
| 125 47.20000000 | |
| 126 46.82539683 | |
| 127 46.45669291 | |
| 128 46.87500000 | |
| 129 46.51162791 | |
| 130 46.92307692 | |
| 131 47.32824427 | |
| 132 47.72727273 | |
| 133 47.36842105 | |
| 134 47.01492537 | |
| 135 46.66666667 | |
| 136 47.05882353 | |
| 137 47.44525547 | |
| 138 47.10144928 | |
| 139 47.48201439 | |
| 140 47.85714286 | |
| 141 48.22695035 | |
| 142 48.59154930 | |
| 143 48.95104895 | |
| 144 48.61111111 | |
| 145 48.27586207 | |
| 146 48.63013699 | |
| 147 48.29931973 | |
| 148 48.64864865 | |
| 149 48.99328859 | |
| 150 48.66666667 | |
| 151 48.34437086 | |
| 152 48.68421053 | |
| 153 48.36601307 | |
| 154 48.05194805 | |
| 155 47.74193548 | |
| 156 47.43589744 | |
| 157 47.77070064 | |
| 158 48.10126582 | |
| 159 47.79874214 | |
| 160 48.12500000 | |
| 161 47.82608696 | |
| 162 47.53086420 | |
| 163 47.23926380 | |
| 164 46.95121951 | |
| 165 46.66666667 | |
| 166 46.38554217 | |
| 167 46.70658683 | |
| 168 46.42857143 | |
| 169 46.74556213 | |
| 170 46.47058824 | |
| 171 46.78362573 | |
| 172 46.51162791 | |
| 173 46.24277457 | |
| 174 45.97701149 | |
| 175 45.71428571 | |
| 176 46.02272727 | |
| 177 45.76271186 | |
| 178 46.06741573 | |
| 179 45.81005587 | |
| 180 45.55555556 | |
| 181 45.30386740 | |
| 182 45.05494505 | |
| 183 44.80874317 | |
| 184 44.56521739 | |
| 185 44.86486486 | |
| 186 45.16129032 | |
| 187 44.91978610 | |
| 188 45.21276596 | |
| 189 45.50264550 | |
| 190 45.26315789 | |
| 191 45.54973822 | |
| 192 45.31250000 | |
| 193 45.07772021 | |
| 194 44.84536082 | |
| 195 45.12820513 | |
| 196 45.40816327 | |
| 197 45.17766497 | |
| 198 45.45454545 | |
| 199 45.22613065 | |
| 200 45.00000000 | |
| 201 44.77611940 | |
| 202 44.55445545 | |
| 203 44.33497537 | |
| 204 44.11764706 | |
| 205 44.39024390 | |
| 206 44.17475728 | |
| 207 44.44444444 | |
| 208 44.23076923 | |
| 209 44.01913876 | |
| 210 43.80952381 | |
| 211 43.60189573 | |
| 212 43.39622642 | |
| 213 43.19248826 | |
| 214 42.99065421 | |
| 215 43.25581395 | |
| 216 43.05555556 | |
| 217 42.85714286 | |
| 218 43.11926606 | |
| 219 43.37899543 | |
| 220 43.18181818 | |
| 221 42.98642534 | |
| 222 42.79279279 | |
| 223 43.04932735 | |
| 224 42.85714286 | |
| 225 43.11111111 | |
| 226 43.36283186 | |
| 227 43.17180617 | |
| 228 43.42105263 | |
| 229 43.23144105 | |
| 230 43.04347826 | |
| 231 42.85714286 | |
| 232 43.10344828 | |
| 233 42.91845494 | |
| 234 43.16239316 | |
| 235 42.97872340 | |
| 236 42.79661017 | |
| 237 42.61603376 | |
| 238 42.43697479 | |
| 239 42.25941423 | |
| 240 42.50000000 | |
| 241 42.32365145 | |
| 242 42.56198347 | |
| 243 42.79835391 | |
| 244 42.62295082 | |
| 245 42.85714286 | |
| 246 42.68292683 | |
| 247 42.51012146 | |
| 248 42.74193548 | |
| 249 42.57028112 | |
| 250 42.40000000 | |
| 251 42.23107570 | |
| 252 42.06349206 | |
| 253 42.29249012 | |
| 254 42.12598425 | |
| 255 41.96078431 | |
| 256 42.18750000 | |
| 257 42.41245136 | |
| 258 42.24806202 | |
| 259 42.08494208 | |
| 260 42.30769231 | |
| 261 42.52873563 | |
| 262 42.36641221 | |
| 263 42.20532319 | |
| 264 42.42424242 | |
| 265 42.26415094 | |
| 266 42.48120301 | |
| 267 42.32209738 | |
| 268 42.16417910 | |
| 269 42.37918216 | |
| 270 42.59259259 | |
| 271 42.43542435 | |
| 272 42.64705882 | |
| 273 42.49084249 | |
| 274 42.33576642 | |
| 275 42.18181818 | |
| 276 42.39130435 | |
| 277 42.59927798 | |
| 278 42.80575540 | |
| 279 42.65232975 | |
| 280 42.50000000 | |
| 281 42.34875445 | |
| 282 42.19858156 | |
| 283 42.04946996 | |
| 284 41.90140845 | |
| 285 42.10526316 | |
| 286 41.95804196 | |
| 287 42.16027875 | |
| 288 42.01388889 | |
| 289 41.86851211 | |
| 290 42.06896552 | |
| 291 42.26804124 | |
| 292 42.46575342 | |
| 293 42.66211604 | |
| 294 42.51700680 | |
| 295 42.37288136 | |
| 296 42.22972973 | |
| 297 42.08754209 | |
| 298 42.28187919 | |
| 299 42.47491639 | |
| 300 42.66666667 | |
| 301 42.52491694 | |
| 302 42.71523179 | |
| 303 42.57425743 | |
| 304 42.43421053 | |
| 305 42.62295082 | |
| 306 42.81045752 | |
| 307 42.99674267 | |
| 308 43.18181818 | |
| 309 43.36569579 | |
| 310 43.22580645 | |
| 311 43.40836013 | |
| 312 43.58974359 | |
| 313 43.76996805 | |
| 314 43.94904459 | |
| 315 43.80952381 | |
| 316 43.67088608 | |
| 317 43.53312303 | |
| 318 43.39622642 | |
| 319 43.57366771 | |
| 320 43.43750000 | |
| 321 43.30218069 | |
| 322 43.47826087 | |
| 323 43.34365325 | |
| 324 43.20987654 | |
| 325 43.38461538 | |
| 326 43.25153374 | |
| 327 43.42507645 | |
| 328 43.59756098 | |
| 329 43.76899696 | |
| 330 43.63636364 | |
| 331 43.50453172 | |
| 332 43.67469880 | |
| 333 43.84384384 | |
| 334 43.71257485 | |
| 335 43.88059701 | |
| 336 44.04761905 | |
| 337 43.91691395 | |
| 338 43.78698225 | |
| 339 43.65781711 | |
| 340 43.82352941 | |
| 341 43.98826979 | |
| 342 44.15204678 | |
| 343 44.02332362 | |
| 344 44.18604651 | |
| 345 44.05797101 | |
| 346 44.21965318 | |
| 347 44.38040346 | |
| 348 44.25287356 | |
| 349 44.12607450 | |
| 350 44.28571429 | |
| 351 44.15954416 | |
| 352 44.03409091 | |
| 353 44.19263456 | |
| 354 44.06779661 | |
| 355 43.94366197 | |
| 356 44.10112360 | |
| 357 43.97759104 | |
| 358 43.85474860 | |
| 359 43.73259053 | |
| 360 43.88888889 | |
| 361 43.76731302 | |
| 362 43.92265193 | |
| 363 43.80165289 | |
| 364 43.95604396 | |
| 365 43.83561644 | |
| 366 43.98907104 | |
| 367 43.86920981 | |
| 368 43.75000000 | |
| 369 43.63143631 | |
| 370 43.78378378 | |
| 371 43.93530997 | |
| 372 43.81720430 | |
| 373 43.69973190 | |
| 374 43.85026738 | |
| 375 44.00000000 | |
| 376 43.88297872 | |
| 377 43.76657825 | |
| 378 43.91534392 | |
| 379 43.79947230 | |
| 380 43.68421053 | |
| 381 43.83202100 | |
| 382 43.97905759 | |
| 383 44.12532637 | |
| 384 44.01041667 | |
| 385 44.15584416 | |
| 386 44.04145078 | |
| 387 43.92764858 | |
| 388 43.81443299 | |
| 389 43.95886889 | |
| 390 43.84615385 | |
| 391 43.98976982 | |
| 392 44.13265306 | |
| 393 44.02035623 | |
| 394 43.90862944 | |
| 395 44.05063291 | |
| 396 44.19191919 | |
| 397 44.33249370 | |
| 398 44.22110553 | |
| 399 44.36090226 | |
| 400 44.25000000 | |
| 401 44.38902743 | |
| 402 44.27860697 | |
| 403 44.16873449 | |
| 404 44.30693069 | |
| 405 44.19753086 | |
| 406 44.08866995 | |
| 407 43.98034398 | |
| 408 43.87254902 | |
| 409 44.00977995 | |
| 410 43.90243902 | |
| 411 43.79562044 | |
| 412 43.68932039 | |
| 413 43.58353511 | |
| 414 43.71980676 | |
| 415 43.85542169 | |
| 416 43.75000000 | |
| 417 43.64508393 | |
| 418 43.54066986 | |
| 419 43.67541766 | |
| 420 43.57142857 | |
| 421 43.46793349 | |
| 422 43.36492891 | |
| 423 43.26241135 | |
| 424 43.39622642 | |
| 425 43.29411765 | |
| 426 43.42723005 | |
| 427 43.32552693 | |
| 428 43.22429907 | |
| 429 43.35664336 | |
| 430 43.25581395 | |
| 431 43.38747100 | |
| 432 43.28703704 | |
| 433 43.18706697 | |
| 434 43.08755760 | |
| 435 42.98850575 | |
| 436 42.88990826 | |
| 437 42.79176201 | |
| 438 42.92237443 | |
| 439 43.05239180 | |
| 440 42.95454545 | |
| 441 42.85714286 | |
| 442 42.76018100 | |
| 443 42.66365688 | |
| 444 42.56756757 | |
| 445 42.47191011 | |
| 446 42.37668161 | |
| 447 42.50559284 | |
| 448 42.41071429 | |
| 449 42.31625835 | |
| 450 42.44444444 | |
| 451 42.57206208 | |
| 452 42.69911504 | |
| 453 42.82560706 | |
| 454 42.73127753 | |
| 455 42.63736264 | |
| 456 42.76315789 | |
| 457 42.66958425 | |
| 458 42.57641921 | |
| 459 42.48366013 | |
| 460 42.60869565 | |
| 461 42.73318872 | |
| 462 42.85714286 | |
| 463 42.98056156 | |
| 464 43.10344828 | |
| 465 43.01075269 | |
| 466 42.91845494 | |
| 467 43.04068522 | |
| 468 43.16239316 | |
| 469 43.28358209 | |
| 470 43.19148936 | |
| 471 43.09978769 | |
| 472 43.00847458 | |
| 473 42.91754757 | |
| 474 42.82700422 | |
| 475 42.73684211 | |
| 476 42.85714286 | |
| 477 42.76729560 | |
| 478 42.88702929 | |
| 479 43.00626305 | |
| 480 42.91666667 | |
| 481 42.82744283 | |
| 482 42.73858921 | |
| 483 42.65010352 | |
| 484 42.56198347 | |
| 485 42.47422680 | |
| 486 42.59259259 | |
| 487 42.71047228 | |
| 488 42.62295082 | |
| 489 42.53578732 | |
| 490 42.65306122 | |
| 491 42.56619145 | |
| 492 42.47967480 | |
| 493 42.39350913 | |
| 494 42.51012146 | |
| 495 42.62626263 | |
| 496 42.74193548 | |
| 497 42.65593561 | |
| 498 42.57028112 | |
| 499 42.68537074 | |
| 500 42.60000000 | |
| 501 42.51497006 | |
| 502 42.43027888 | |
| 503 42.54473161 | |
| 504 42.65873016 | |
| 505 42.57425743 | |
| 506 42.68774704 | |
| 507 42.60355030 | |
| 508 42.51968504 | |
| 509 42.43614931 | |
| 510 42.35294118 | |
| 511 42.27005871 | |
| 512 42.18750000 | |
| 513 42.10526316 | |
| 514 42.02334630 | |
| 515 41.94174757 | |
| 516 42.05426357 | |
| 517 42.16634429 | |
| 518 42.27799228 | |
| 519 42.19653179 | |
| 520 42.30769231 | |
| 521 42.22648752 | |
| 522 42.33716475 | |
| 523 42.25621415 | |
| 524 42.36641221 | |
| 525 42.28571429 | |
| 526 42.20532319 | |
| 527 42.31499051 | |
| 528 42.42424242 | |
| 529 42.53308129 | |
| 530 42.64150943 | |
| 531 42.56120527 | |
| 532 42.48120301 | |
| 533 42.40150094 | |
| 534 42.50936330 | |
| 535 42.61682243 | |
| 536 42.53731343 | |
| 537 42.64432030 | |
| 538 42.75092937 | |
| 539 42.67161410 | |
| 540 42.59259259 | |
| 541 42.51386322 | |
| 542 42.61992620 | |
| 543 42.54143646 | |
| 544 42.64705882 | |
| 545 42.56880734 | |
| 546 42.49084249 | |
| 547 42.59597806 | |
| 548 42.70072993 | |
| 549 42.62295082 | |
| 550 42.54545455 | |
| 551 42.64972777 | |
| 552 42.75362319 | |
| 553 42.67631103 | |
| 554 42.77978339 | |
| 555 42.70270270 | |
| 556 42.80575540 | |
| 557 42.72890485 | |
| 558 42.83154122 | |
| 559 42.93381038 | |
| 560 42.85714286 | |
| 561 42.78074866 | |
| 562 42.88256228 | |
| 563 42.98401421 | |
| 564 42.90780142 | |
| 565 42.83185841 | |
| 566 42.75618375 | |
| 567 42.68077601 | |
| 568 42.60563380 | |
| 569 42.53075571 | |
| 570 42.63157895 | |
| 571 42.55691769 | |
| 572 42.65734266 | |
| 573 42.58289703 | |
| 574 42.50871080 | |
| 575 42.43478261 | |
| 576 42.53472222 | |
| 577 42.63431542 | |
| 578 42.73356401 | |
| 579 42.65975820 | |
| 580 42.75862069 | |
| 581 42.85714286 | |
| 582 42.78350515 | |
| 583 42.88164666 | |
| 584 42.97945205 | |
| 585 43.07692308 | |
| 586 43.17406143 | |
| 587 43.10051107 | |
| 588 43.02721088 | |
| 589 42.95415959 | |
| 590 42.88135593 | |
| 591 42.80879865 | |
| 592 42.90540541 | |
| 593 43.00168634 | |
| 594 43.09764310 | |
| 595 43.02521008 | |
| 596 43.12080537 | |
| 597 43.04857621 | |
| 598 43.14381271 | |
| 599 43.23873122 | |
| 600 43.16666667 | |
| 601 43.26123128 | |
| 602 43.18936877 | |
| 603 43.28358209 | |
| 604 43.21192053 | |
| 605 43.14049587 | |
| 606 43.06930693 | |
| 607 43.16309720 | |
| 608 43.09210526 | |
| 609 43.02134647 | |
| 610 42.95081967 | |
| 611 43.04418985 | |
| 612 42.97385621 | |
| 613 42.90375204 | |
| 614 42.99674267 | |
| 615 42.92682927 | |
| 616 43.01948052 | |
| 617 42.94975689 | |
| 618 43.04207120 | |
| 619 42.97253635 | |
| 620 42.90322581 | |
| 621 42.83413849 | |
| 622 42.76527331 | |
| 623 42.69662921 | |
| 624 42.62820513 | |
| 625 42.56000000 | |
| 626 42.65175719 | |
| 627 42.58373206 | |
| 628 42.51592357 | |
| 629 42.60731320 | |
| 630 42.53968254 | |
| 631 42.47226624 | |
| 632 42.56329114 | |
| 633 42.49605055 | |
| 634 42.42902208 | |
| 635 42.51968504 | |
| 636 42.45283019 | |
| 637 42.38618524 | |
| 638 42.47648903 | |
| 639 42.56651017 | |
| 640 42.65625000 | |
| 641 42.58970359 | |
| 642 42.67912773 | |
| 643 42.61275272 | |
| 644 42.70186335 | |
| 645 42.63565891 | |
| 646 42.56965944 | |
| 647 42.50386399 | |
| 648 42.43827160 | |
| 649 42.37288136 | |
| 650 42.46153846 | |
| 651 42.39631336 | |
| 652 42.48466258 | |
| 653 42.41960184 | |
| 654 42.50764526 | |
| 655 42.44274809 | |
| 656 42.53048780 | |
| 657 42.61796043 | |
| 658 42.70516717 | |
| 659 42.64036419 | |
| 660 42.57575758 | |
| 661 42.51134644 | |
| 662 42.44712991 | |
| 663 42.38310709 | |
| 664 42.31927711 | |
| 665 42.40601504 | |
| 666 42.34234234 | |
| 667 42.42878561 | |
| 668 42.51497006 | |
| 669 42.60089686 | |
| 670 42.68656716 | |
| 671 42.62295082 | |
| 672 42.55952381 | |
| 673 42.49628529 | |
| 674 42.43323442 | |
| 675 42.37037037 | |
| 676 42.30769231 | |
| 677 42.24519941 | |
| 678 42.33038348 | |
| 679 42.26804124 | |
| 680 42.35294118 | |
| 681 42.43759178 | |
| 682 42.37536657 | |
| 683 42.31332357 | |
| 684 42.39766082 | |
| 685 42.33576642 | |
| 686 42.27405248 | |
| 687 42.21251820 | |
| 688 42.15116279 | |
| 689 42.08998549 | |
| 690 42.02898551 | |
| 691 42.11287988 | |
| 692 42.05202312 | |
| 693 41.99134199 | |
| 694 41.93083573 | |
| 695 41.87050360 | |
| 696 41.95402299 | |
| 697 41.89383070 | |
| 698 41.83381089 | |
| 699 41.91702432 | |
| 700 42.00000000 | |
| 701 41.94008559 | |
| 702 42.02279202 | |
| 703 42.10526316 | |
| 704 42.04545455 | |
| 705 41.98581560 | |
| 706 42.06798867 | |
| 707 42.00848656 | |
| 708 41.94915254 | |
| 709 42.03102962 | |
| 710 42.11267606 | |
| 711 42.19409283 | |
| 712 42.27528090 | |
| 713 42.35624123 | |
| 714 42.43697479 | |
| 715 42.37762238 | |
| 716 42.31843575 | |
| 717 42.25941423 | |
| 718 42.20055710 | |
| 719 42.28094576 | |
| 720 42.22222222 | |
| 721 42.16366158 | |
| 722 42.24376731 | |
| 723 42.32365145 | |
| 724 42.26519337 | |
| 725 42.34482759 | |
| 726 42.42424242 | |
| 727 42.50343879 | |
| 728 42.58241758 | |
| 729 42.52400549 | |
| 730 42.46575342 | |
| 731 42.40766074 | |
| 732 42.34972678 | |
| 733 42.42837653 | |
| 734 42.50681199 | |
| 735 42.44897959 | |
| 736 42.39130435 | |
| 737 42.33378562 | |
| 738 42.41192412 | |
| 739 42.48985115 | |
| 740 42.56756757 | |
| 741 42.51012146 | |
| 742 42.58760108 | |
| 743 42.66487214 | |
| 744 42.74193548 | |
| 745 42.68456376 | |
| 746 42.62734584 | |
| 747 42.57028112 | |
| 748 42.51336898 | |
| 749 42.45660881 | |
| 750 42.53333333 | |
| Final result: 42.5333 +/- 1.8065 | |
| Random chance: 25.0000 +/- 1.5822 | |