| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 100.00000000 | |
| 4 75.00000000 | |
| 5 80.00000000 | |
| 6 83.33333333 | |
| 7 85.71428571 | |
| 8 87.50000000 | |
| 9 77.77777778 | |
| 10 80.00000000 | |
| 11 72.72727273 | |
| 12 75.00000000 | |
| 13 69.23076923 | |
| 14 71.42857143 | |
| 15 73.33333333 | |
| 16 75.00000000 | |
| 17 76.47058824 | |
| 18 77.77777778 | |
| 19 78.94736842 | |
| 20 75.00000000 | |
| 21 76.19047619 | |
| 22 77.27272727 | |
| 23 78.26086957 | |
| 24 79.16666667 | |
| 25 76.00000000 | |
| 26 73.07692308 | |
| 27 74.07407407 | |
| 28 71.42857143 | |
| 29 72.41379310 | |
| 30 73.33333333 | |
| 31 70.96774194 | |
| 32 71.87500000 | |
| 33 72.72727273 | |
| 34 73.52941176 | |
| 35 74.28571429 | |
| 36 72.22222222 | |
| 37 72.97297297 | |
| 38 73.68421053 | |
| 39 74.35897436 | |
| 40 75.00000000 | |
| 41 73.17073171 | |
| 42 73.80952381 | |
| 43 72.09302326 | |
| 44 70.45454545 | |
| 45 71.11111111 | |
| 46 71.73913043 | |
| 47 72.34042553 | |
| 48 72.91666667 | |
| 49 71.42857143 | |
| 50 70.00000000 | |
| 51 68.62745098 | |
| 52 69.23076923 | |
| 53 69.81132075 | |
| 54 68.51851852 | |
| 55 67.27272727 | |
| 56 67.85714286 | |
| 57 68.42105263 | |
| 58 68.96551724 | |
| 59 69.49152542 | |
| 60 70.00000000 | |
| 61 70.49180328 | |
| 62 70.96774194 | |
| 63 71.42857143 | |
| 64 71.87500000 | |
| 65 70.76923077 | |
| 66 71.21212121 | |
| 67 71.64179104 | |
| 68 72.05882353 | |
| 69 72.46376812 | |
| 70 72.85714286 | |
| 71 73.23943662 | |
| 72 72.22222222 | |
| 73 72.60273973 | |
| 74 72.97297297 | |
| 75 73.33333333 | |
| 76 73.68421053 | |
| 77 74.02597403 | |
| 78 74.35897436 | |
| 79 74.68354430 | |
| 80 75.00000000 | |
| 81 75.30864198 | |
| 82 75.60975610 | |
| 83 74.69879518 | |
| 84 75.00000000 | |
| 85 75.29411765 | |
| 86 75.58139535 | |
| 87 75.86206897 | |
| 88 76.13636364 | |
| 89 76.40449438 | |
| 90 76.66666667 | |
| 91 76.92307692 | |
| 92 77.17391304 | |
| 93 76.34408602 | |
| 94 75.53191489 | |
| 95 75.78947368 | |
| 96 76.04166667 | |
| 97 75.25773196 | |
| 98 75.51020408 | |
| 99 75.75757576 | |
| 100 76.00000000 | |
| 101 75.24752475 | |
| 102 75.49019608 | |
| 103 74.75728155 | |
| 104 74.03846154 | |
| 105 74.28571429 | |
| 106 73.58490566 | |
| 107 73.83177570 | |
| 108 73.14814815 | |
| 109 72.47706422 | |
| 110 72.72727273 | |
| 111 72.97297297 | |
| 112 73.21428571 | |
| 113 72.56637168 | |
| 114 72.80701754 | |
| 115 73.04347826 | |
| 116 72.41379310 | |
| 117 72.64957265 | |
| 118 72.88135593 | |
| 119 72.26890756 | |
| 120 72.50000000 | |
| 121 71.90082645 | |
| 122 72.13114754 | |
| 123 72.35772358 | |
| 124 71.77419355 | |
| 125 72.00000000 | |
| 126 72.22222222 | |
| 127 72.44094488 | |
| 128 72.65625000 | |
| 129 72.09302326 | |
| 130 72.30769231 | |
| 131 72.51908397 | |
| 132 71.96969697 | |
| 133 72.18045113 | |
| 134 71.64179104 | |
| 135 71.11111111 | |
| 136 71.32352941 | |
| 137 71.53284672 | |
| 138 71.01449275 | |
| 139 71.22302158 | |
| 140 70.71428571 | |
| 141 70.21276596 | |
| 142 70.42253521 | |
| 143 70.62937063 | |
| 144 70.83333333 | |
| 145 71.03448276 | |
| 146 71.23287671 | |
| 147 71.42857143 | |
| 148 70.94594595 | |
| 149 71.14093960 | |
| 150 71.33333333 | |
| 151 71.52317881 | |
| 152 71.71052632 | |
| 153 71.89542484 | |
| 154 72.07792208 | |
| 155 71.61290323 | |
| 156 71.79487179 | |
| 157 71.97452229 | |
| 158 72.15189873 | |
| 159 72.32704403 | |
| 160 72.50000000 | |
| 161 72.67080745 | |
| 162 72.83950617 | |
| 163 72.39263804 | |
| 164 72.56097561 | |
| 165 72.72727273 | |
| 166 72.89156627 | |
| 167 73.05389222 | |
| 168 73.21428571 | |
| 169 72.78106509 | |
| 170 72.94117647 | |
| 171 73.09941520 | |
| 172 73.25581395 | |
| 173 73.41040462 | |
| 174 73.56321839 | |
| 175 73.14285714 | |
| 176 73.29545455 | |
| 177 73.44632768 | |
| 178 73.59550562 | |
| 179 73.74301676 | |
| 180 73.88888889 | |
| 181 74.03314917 | |
| 182 74.17582418 | |
| 183 73.77049180 | |
| 184 73.91304348 | |
| 185 74.05405405 | |
| 186 74.19354839 | |
| 187 74.33155080 | |
| 188 73.93617021 | |
| 189 74.07407407 | |
| 190 74.21052632 | |
| 191 74.34554974 | |
| 192 73.95833333 | |
| 193 73.57512953 | |
| 194 73.71134021 | |
| 195 73.84615385 | |
| 196 73.97959184 | |
| 197 74.11167513 | |
| 198 74.24242424 | |
| 199 73.86934673 | |
| 200 74.00000000 | |
| 201 74.12935323 | |
| 202 74.25742574 | |
| 203 74.38423645 | |
| 204 74.50980392 | |
| 205 74.63414634 | |
| 206 74.27184466 | |
| 207 74.39613527 | |
| 208 74.51923077 | |
| 209 74.64114833 | |
| 210 74.76190476 | |
| 211 74.88151659 | |
| 212 74.52830189 | |
| 213 74.64788732 | |
| 214 74.76635514 | |
| 215 74.88372093 | |
| 216 74.53703704 | |
| 217 74.65437788 | |
| 218 74.77064220 | |
| 219 74.88584475 | |
| 220 74.54545455 | |
| 221 74.66063348 | |
| 222 74.77477477 | |
| 223 74.43946188 | |
| 224 74.55357143 | |
| 225 74.22222222 | |
| 226 73.89380531 | |
| 227 74.00881057 | |
| 228 74.12280702 | |
| 229 74.23580786 | |
| 230 74.34782609 | |
| 231 74.02597403 | |
| 232 73.70689655 | |
| 233 73.39055794 | |
| 234 73.07692308 | |
| 235 72.76595745 | |
| 236 72.88135593 | |
| 237 72.57383966 | |
| 238 72.26890756 | |
| 239 72.38493724 | |
| 240 72.50000000 | |
| 241 72.61410788 | |
| 242 72.72727273 | |
| 243 72.83950617 | |
| 244 72.54098361 | |
| 245 72.65306122 | |
| 246 72.76422764 | |
| 247 72.87449393 | |
| 248 72.98387097 | |
| 249 73.09236948 | |
| 250 73.20000000 | |
| 251 73.30677291 | |
| 252 73.41269841 | |
| 253 73.51778656 | |
| 254 73.62204724 | |
| 255 73.72549020 | |
| 256 73.82812500 | |
| 257 73.54085603 | |
| 258 73.64341085 | |
| 259 73.74517375 | |
| 260 73.84615385 | |
| 261 73.94636015 | |
| 262 74.04580153 | |
| 263 74.14448669 | |
| 264 74.24242424 | |
| 265 74.33962264 | |
| 266 74.06015038 | |
| 267 74.15730337 | |
| 268 74.25373134 | |
| 269 74.34944238 | |
| 270 74.44444444 | |
| 271 74.53874539 | |
| 272 74.63235294 | |
| 273 74.72527473 | |
| 274 74.45255474 | |
| 275 74.18181818 | |
| 276 74.27536232 | |
| 277 74.36823105 | |
| 278 74.46043165 | |
| 279 74.55197133 | |
| 280 74.64285714 | |
| 281 74.73309609 | |
| 282 74.82269504 | |
| 283 74.55830389 | |
| 284 74.64788732 | |
| 285 74.73684211 | |
| 286 74.82517483 | |
| 287 74.91289199 | |
| 288 75.00000000 | |
| 289 74.74048443 | |
| 290 74.82758621 | |
| 291 74.91408935 | |
| 292 75.00000000 | |
| 293 75.08532423 | |
| 294 74.82993197 | |
| 295 74.91525424 | |
| 296 75.00000000 | |
| 297 75.08417508 | |
| 298 75.16778523 | |
| 299 75.25083612 | |
| 300 75.33333333 | |
| 301 75.08305648 | |
| 302 75.16556291 | |
| 303 75.24752475 | |
| 304 75.32894737 | |
| 305 75.40983607 | |
| 306 75.16339869 | |
| 307 75.24429967 | |
| 308 75.32467532 | |
| 309 75.40453074 | |
| 310 75.16129032 | |
| 311 75.24115756 | |
| 312 75.00000000 | |
| 313 75.07987220 | |
| 314 75.15923567 | |
| 315 75.23809524 | |
| 316 75.31645570 | |
| 317 75.39432177 | |
| 318 75.47169811 | |
| 319 75.23510972 | |
| 320 75.00000000 | |
| 321 75.07788162 | |
| 322 75.15527950 | |
| 323 75.23219814 | |
| 324 75.30864198 | |
| 325 75.38461538 | |
| 326 75.46012270 | |
| 327 75.22935780 | |
| 328 75.30487805 | |
| 329 75.37993921 | |
| 330 75.15151515 | |
| 331 75.22658610 | |
| 332 75.30120482 | |
| 333 75.37537538 | |
| 334 75.44910180 | |
| 335 75.22388060 | |
| 336 75.00000000 | |
| 337 74.77744807 | |
| 338 74.55621302 | |
| 339 74.63126844 | |
| 340 74.70588235 | |
| 341 74.78005865 | |
| 342 74.85380117 | |
| 343 74.92711370 | |
| 344 75.00000000 | |
| 345 74.78260870 | |
| 346 74.85549133 | |
| 347 74.92795389 | |
| 348 74.71264368 | |
| 349 74.78510029 | |
| 350 74.85714286 | |
| 351 74.92877493 | |
| 352 75.00000000 | |
| 353 75.07082153 | |
| 354 74.85875706 | |
| 355 74.64788732 | |
| 356 74.71910112 | |
| 357 74.78991597 | |
| 358 74.86033520 | |
| 359 74.65181058 | |
| 360 74.72222222 | |
| 361 74.79224377 | |
| 362 74.86187845 | |
| 363 74.93112948 | |
| 364 75.00000000 | |
| 365 75.06849315 | |
| 366 74.86338798 | |
| 367 74.93188011 | |
| 368 75.00000000 | |
| 369 75.06775068 | |
| 370 75.13513514 | |
| 371 74.93261456 | |
| 372 75.00000000 | |
| 373 75.06702413 | |
| 374 75.13368984 | |
| 375 74.93333333 | |
| 376 75.00000000 | |
| 377 74.80106101 | |
| 378 74.86772487 | |
| 379 74.67018470 | |
| 380 74.73684211 | |
| 381 74.54068241 | |
| 382 74.34554974 | |
| 383 74.15143603 | |
| 384 74.21875000 | |
| 385 74.28571429 | |
| 386 74.09326425 | |
| 387 74.16020672 | |
| 388 74.22680412 | |
| 389 74.03598972 | |
| 390 73.84615385 | |
| 391 73.65728900 | |
| 392 73.72448980 | |
| 393 73.79134860 | |
| 394 73.60406091 | |
| 395 73.67088608 | |
| 396 73.48484848 | |
| 397 73.55163728 | |
| 398 73.36683417 | |
| 399 73.43358396 | |
| 400 73.50000000 | |
| 401 73.56608479 | |
| 402 73.63184080 | |
| 403 73.69727047 | |
| 404 73.76237624 | |
| 405 73.82716049 | |
| 406 73.89162562 | |
| 407 73.95577396 | |
| 408 74.01960784 | |
| 409 73.83863081 | |
| 410 73.90243902 | |
| 411 73.96593674 | |
| 412 74.02912621 | |
| 413 74.09200969 | |
| 414 74.15458937 | |
| 415 74.21686747 | |
| 416 74.27884615 | |
| 417 74.10071942 | |
| 418 74.16267943 | |
| 419 73.98568019 | |
| 420 74.04761905 | |
| 421 74.10926366 | |
| 422 74.17061611 | |
| 423 74.23167849 | |
| 424 74.29245283 | |
| 425 74.11764706 | |
| 426 74.17840376 | |
| 427 74.00468384 | |
| 428 73.83177570 | |
| 429 73.89277389 | |
| 430 73.72093023 | |
| 431 73.78190255 | |
| 432 73.84259259 | |
| 433 73.67205543 | |
| 434 73.73271889 | |
| 435 73.79310345 | |
| 436 73.85321101 | |
| 437 73.91304348 | |
| 438 73.74429224 | |
| 439 73.80410023 | |
| 440 73.86363636 | |
| 441 73.92290249 | |
| 442 73.75565611 | |
| 443 73.81489842 | |
| 444 73.87387387 | |
| 445 73.93258427 | |
| 446 73.99103139 | |
| 447 74.04921700 | |
| 448 74.10714286 | |
| 449 74.16481069 | |
| 450 74.22222222 | |
| 451 74.27937916 | |
| 452 74.33628319 | |
| 453 74.39293598 | |
| 454 74.22907489 | |
| 455 74.28571429 | |
| 456 74.34210526 | |
| 457 74.39824945 | |
| 458 74.45414847 | |
| 459 74.50980392 | |
| 460 74.34782609 | |
| 461 74.40347072 | |
| 462 74.45887446 | |
| 463 74.29805616 | |
| 464 74.35344828 | |
| 465 74.40860215 | |
| 466 74.46351931 | |
| 467 74.51820128 | |
| 468 74.57264957 | |
| 469 74.41364606 | |
| 470 74.46808511 | |
| 471 74.52229299 | |
| 472 74.57627119 | |
| 473 74.41860465 | |
| 474 74.47257384 | |
| 475 74.31578947 | |
| 476 74.36974790 | |
| 477 74.42348008 | |
| 478 74.47698745 | |
| 479 74.53027140 | |
| 480 74.58333333 | |
| 481 74.63617464 | |
| 482 74.68879668 | |
| 483 74.74120083 | |
| 484 74.79338843 | |
| 485 74.84536082 | |
| 486 74.89711934 | |
| 487 74.74332649 | |
| 488 74.79508197 | |
| 489 74.84662577 | |
| 490 74.89795918 | |
| 491 74.94908350 | |
| 492 75.00000000 | |
| 493 74.84787018 | |
| 494 74.89878543 | |
| 495 74.94949495 | |
| 496 74.79838710 | |
| 497 74.64788732 | |
| 498 74.69879518 | |
| 499 74.74949900 | |
| 500 74.80000000 | |
| 501 74.85029940 | |
| 502 74.90039841 | |
| 503 74.95029821 | |
| 504 75.00000000 | |
| 505 75.04950495 | |
| 506 74.90118577 | |
| 507 74.95069034 | |
| 508 75.00000000 | |
| 509 74.85265226 | |
| 510 74.90196078 | |
| 511 74.75538160 | |
| 512 74.80468750 | |
| 513 74.85380117 | |
| 514 74.90272374 | |
| 515 74.95145631 | |
| 516 74.80620155 | |
| 517 74.85493230 | |
| 518 74.90347490 | |
| 519 74.95183044 | |
| 520 75.00000000 | |
| 521 75.04798464 | |
| 522 75.09578544 | |
| 523 74.95219885 | |
| 524 74.80916031 | |
| 525 74.66666667 | |
| 526 74.52471483 | |
| 527 74.38330171 | |
| 528 74.24242424 | |
| 529 74.10207940 | |
| 530 74.15094340 | |
| 531 74.19962335 | |
| 532 74.24812030 | |
| 533 74.29643527 | |
| 534 74.15730337 | |
| 535 74.01869159 | |
| 536 74.06716418 | |
| 537 74.11545624 | |
| 538 73.97769517 | |
| 539 74.02597403 | |
| 540 73.88888889 | |
| 541 73.93715342 | |
| 542 73.98523985 | |
| 543 73.84898711 | |
| 544 73.89705882 | |
| 545 73.94495413 | |
| 546 73.99267399 | |
| 547 73.85740402 | |
| 548 73.90510949 | |
| 549 73.95264117 | |
| 550 74.00000000 | |
| 551 73.86569873 | |
| 552 73.73188406 | |
| 553 73.77938517 | |
| 554 73.82671480 | |
| 555 73.87387387 | |
| 556 73.74100719 | |
| 557 73.78815081 | |
| 558 73.65591398 | |
| 559 73.70304114 | |
| 560 73.75000000 | |
| 561 73.79679144 | |
| 562 73.84341637 | |
| 563 73.88987567 | |
| 564 73.93617021 | |
| 565 73.80530973 | |
| 566 73.85159011 | |
| 567 73.89770723 | |
| 568 73.94366197 | |
| 569 73.81370826 | |
| 570 73.68421053 | |
| 571 73.73029772 | |
| 572 73.60139860 | |
| 573 73.64746946 | |
| 574 73.69337979 | |
| 575 73.73913043 | |
| 576 73.78472222 | |
| 577 73.83015598 | |
| 578 73.70242215 | |
| 579 73.74784111 | |
| 580 73.79310345 | |
| 581 73.66609294 | |
| 582 73.53951890 | |
| 583 73.41337907 | |
| 584 73.45890411 | |
| 585 73.50427350 | |
| 586 73.54948805 | |
| 587 73.59454855 | |
| 588 73.63945578 | |
| 589 73.68421053 | |
| 590 73.72881356 | |
| 591 73.60406091 | |
| 592 73.64864865 | |
| 593 73.69308600 | |
| 594 73.73737374 | |
| 595 73.78151261 | |
| 596 73.65771812 | |
| 597 73.70184255 | |
| 598 73.74581940 | |
| 599 73.62270451 | |
| 600 73.66666667 | |
| 601 73.71048253 | |
| 602 73.58803987 | |
| 603 73.46600332 | |
| 604 73.50993377 | |
| 605 73.55371901 | |
| 606 73.59735974 | |
| 607 73.64085667 | |
| 608 73.68421053 | |
| 609 73.56321839 | |
| 610 73.60655738 | |
| 611 73.64975450 | |
| 612 73.69281046 | |
| 613 73.57259380 | |
| 614 73.61563518 | |
| 615 73.49593496 | |
| 616 73.37662338 | |
| 617 73.41977310 | |
| 618 73.46278317 | |
| 619 73.34410339 | |
| 620 73.38709677 | |
| 621 73.42995169 | |
| 622 73.31189711 | |
| 623 73.35473515 | |
| 624 73.23717949 | |
| 625 73.28000000 | |
| 626 73.32268371 | |
| 627 73.36523126 | |
| 628 73.24840764 | |
| 629 73.29093800 | |
| 630 73.33333333 | |
| 631 73.37559429 | |
| 632 73.25949367 | |
| 633 73.14375987 | |
| 634 73.18611987 | |
| 635 73.22834646 | |
| 636 73.27044025 | |
| 637 73.31240188 | |
| 638 73.35423197 | |
| 639 73.39593114 | |
| 640 73.43750000 | |
| 641 73.47893916 | |
| 642 73.36448598 | |
| 643 73.25038880 | |
| 644 73.29192547 | |
| 645 73.17829457 | |
| 646 73.21981424 | |
| 647 73.10664606 | |
| 648 73.14814815 | |
| 649 73.03543914 | |
| 650 73.07692308 | |
| 651 73.11827957 | |
| 652 73.00613497 | |
| 653 73.04747320 | |
| 654 72.93577982 | |
| 655 72.82442748 | |
| 656 72.71341463 | |
| 657 72.75494673 | |
| 658 72.79635258 | |
| 659 72.68588771 | |
| 660 72.72727273 | |
| 661 72.61724660 | |
| 662 72.65861027 | |
| 663 72.69984917 | |
| 664 72.74096386 | |
| 665 72.78195489 | |
| 666 72.82282282 | |
| 667 72.86356822 | |
| 668 72.75449102 | |
| 669 72.79521674 | |
| 670 72.68656716 | |
| 671 72.72727273 | |
| 672 72.76785714 | |
| 673 72.80832095 | |
| 674 72.84866469 | |
| 675 72.88888889 | |
| 676 72.92899408 | |
| 677 72.96898080 | |
| 678 73.00884956 | |
| 679 72.90132548 | |
| 680 72.94117647 | |
| 681 72.98091043 | |
| 682 73.02052786 | |
| 683 72.91361640 | |
| 684 72.95321637 | |
| 685 72.99270073 | |
| 686 73.03206997 | |
| 687 73.07132460 | |
| 688 73.11046512 | |
| 689 73.00435414 | |
| 690 73.04347826 | |
| 691 73.08248915 | |
| 692 72.97687861 | |
| 693 73.01587302 | |
| 694 73.05475504 | |
| 695 72.94964029 | |
| 696 72.98850575 | |
| 697 72.88378766 | |
| 698 72.77936963 | |
| 699 72.81831187 | |
| 700 72.71428571 | |
| 701 72.61055635 | |
| 702 72.64957265 | |
| 703 72.68847795 | |
| 704 72.72727273 | |
| 705 72.76595745 | |
| 706 72.80453258 | |
| 707 72.84299859 | |
| 708 72.88135593 | |
| 709 72.91960508 | |
| 710 72.95774648 | |
| 711 72.99578059 | |
| 712 73.03370787 | |
| 713 73.07152875 | |
| 714 72.96918768 | |
| 715 72.86713287 | |
| 716 72.90502793 | |
| 717 72.94281729 | |
| 718 72.84122563 | |
| 719 72.73991655 | |
| 720 72.77777778 | |
| 721 72.67683773 | |
| 722 72.71468144 | |
| 723 72.75242047 | |
| 724 72.65193370 | |
| 725 72.68965517 | |
| 726 72.58953168 | |
| 727 72.62723521 | |
| 728 72.66483516 | |
| 729 72.70233196 | |
| 730 72.73972603 | |
| 731 72.77701778 | |
| 732 72.81420765 | |
| 733 72.85129604 | |
| 734 72.88828338 | |
| 735 72.92517007 | |
| 736 72.96195652 | |
| 737 72.86295794 | |
| 738 72.89972900 | |
| 739 72.80108254 | |
| 740 72.70270270 | |
| 741 72.73954116 | |
| 742 72.77628032 | |
| 743 72.81292059 | |
| 744 72.71505376 | |
| 745 72.75167785 | |
| 746 72.78820375 | |
| 747 72.69076305 | |
| 748 72.59358289 | |
| 749 72.49666222 | |
| 750 72.53333333 | |
| Final result: 72.5333 +/- 1.6309 | |
| Random chance: 25.0083 +/- 1.5824 | |