| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 26.31578947 | |
| 20 25.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 30.43478261 | |
| 24 33.33333333 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 28.57142857 | |
| 29 31.03448276 | |
| 30 30.00000000 | |
| 31 29.03225806 | |
| 32 31.25000000 | |
| 33 33.33333333 | |
| 34 32.35294118 | |
| 35 31.42857143 | |
| 36 30.55555556 | |
| 37 32.43243243 | |
| 38 31.57894737 | |
| 39 33.33333333 | |
| 40 32.50000000 | |
| 41 31.70731707 | |
| 42 30.95238095 | |
| 43 30.23255814 | |
| 44 31.81818182 | |
| 45 33.33333333 | |
| 46 32.60869565 | |
| 47 34.04255319 | |
| 48 35.41666667 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 37.25490196 | |
| 52 36.53846154 | |
| 53 35.84905660 | |
| 54 35.18518519 | |
| 55 36.36363636 | |
| 56 35.71428571 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 35.48387097 | |
| 63 34.92063492 | |
| 64 35.93750000 | |
| 65 35.38461538 | |
| 66 36.36363636 | |
| 67 35.82089552 | |
| 68 35.29411765 | |
| 69 34.78260870 | |
| 70 34.28571429 | |
| 71 33.80281690 | |
| 72 34.72222222 | |
| 73 34.24657534 | |
| 74 33.78378378 | |
| 75 33.33333333 | |
| 76 34.21052632 | |
| 77 33.76623377 | |
| 78 33.33333333 | |
| 79 32.91139241 | |
| 80 33.75000000 | |
| 81 33.33333333 | |
| 82 34.14634146 | |
| 83 33.73493976 | |
| 84 33.33333333 | |
| 85 32.94117647 | |
| 86 32.55813953 | |
| 87 32.18390805 | |
| 88 31.81818182 | |
| 89 32.58426966 | |
| 90 32.22222222 | |
| 91 31.86813187 | |
| 92 31.52173913 | |
| 93 31.18279570 | |
| 94 30.85106383 | |
| 95 30.52631579 | |
| 96 31.25000000 | |
| 97 31.95876289 | |
| 98 31.63265306 | |
| 99 32.32323232 | |
| 100 33.00000000 | |
| 101 33.66336634 | |
| 102 33.33333333 | |
| 103 33.98058252 | |
| 104 33.65384615 | |
| 105 33.33333333 | |
| 106 33.01886792 | |
| 107 33.64485981 | |
| 108 34.25925926 | |
| 109 34.86238532 | |
| 110 35.45454545 | |
| 111 35.13513514 | |
| 112 35.71428571 | |
| 113 35.39823009 | |
| 114 35.96491228 | |
| 115 35.65217391 | |
| 116 36.20689655 | |
| 117 36.75213675 | |
| 118 36.44067797 | |
| 119 36.13445378 | |
| 120 35.83333333 | |
| 121 36.36363636 | |
| 122 36.06557377 | |
| 123 36.58536585 | |
| 124 36.29032258 | |
| 125 36.80000000 | |
| 126 36.50793651 | |
| 127 36.22047244 | |
| 128 35.93750000 | |
| 129 36.43410853 | |
| 130 36.92307692 | |
| 131 36.64122137 | |
| 132 36.36363636 | |
| 133 36.09022556 | |
| 134 35.82089552 | |
| 135 35.55555556 | |
| 136 35.29411765 | |
| 137 35.76642336 | |
| 138 36.23188406 | |
| 139 36.69064748 | |
| 140 36.42857143 | |
| 141 36.87943262 | |
| 142 36.61971831 | |
| 143 36.36363636 | |
| 144 36.80555556 | |
| 145 37.24137931 | |
| 146 36.98630137 | |
| 147 36.73469388 | |
| 148 36.48648649 | |
| 149 36.24161074 | |
| 150 36.00000000 | |
| 151 36.42384106 | |
| 152 36.18421053 | |
| 153 35.94771242 | |
| 154 35.71428571 | |
| 155 35.48387097 | |
| 156 35.89743590 | |
| 157 36.30573248 | |
| 158 36.70886076 | |
| 159 36.47798742 | |
| 160 36.25000000 | |
| 161 36.64596273 | |
| 162 36.41975309 | |
| 163 36.80981595 | |
| 164 36.58536585 | |
| 165 36.96969697 | |
| 166 37.34939759 | |
| 167 37.12574850 | |
| 168 36.90476190 | |
| 169 36.68639053 | |
| 170 36.47058824 | |
| 171 36.25730994 | |
| 172 36.04651163 | |
| 173 35.83815029 | |
| 174 36.20689655 | |
| 175 36.00000000 | |
| 176 35.79545455 | |
| 177 35.59322034 | |
| 178 35.39325843 | |
| 179 35.19553073 | |
| 180 35.00000000 | |
| 181 34.80662983 | |
| 182 34.61538462 | |
| 183 34.42622951 | |
| 184 34.23913043 | |
| 185 34.59459459 | |
| 186 34.94623656 | |
| 187 34.75935829 | |
| 188 34.57446809 | |
| 189 34.39153439 | |
| 190 34.21052632 | |
| 191 34.03141361 | |
| 192 33.85416667 | |
| 193 33.67875648 | |
| 194 34.02061856 | |
| 195 33.84615385 | |
| 196 33.67346939 | |
| 197 34.01015228 | |
| 198 34.34343434 | |
| 199 34.17085427 | |
| 200 34.50000000 | |
| 201 34.32835821 | |
| 202 34.65346535 | |
| 203 34.48275862 | |
| 204 34.31372549 | |
| 205 34.63414634 | |
| 206 34.46601942 | |
| 207 34.78260870 | |
| 208 34.61538462 | |
| 209 34.44976077 | |
| 210 34.28571429 | |
| 211 34.12322275 | |
| 212 33.96226415 | |
| 213 33.80281690 | |
| 214 33.64485981 | |
| 215 33.48837209 | |
| 216 33.79629630 | |
| 217 33.64055300 | |
| 218 33.48623853 | |
| 219 33.33333333 | |
| 220 33.18181818 | |
| 221 33.48416290 | |
| 222 33.33333333 | |
| 223 33.18385650 | |
| 224 33.48214286 | |
| 225 33.77777778 | |
| 226 33.62831858 | |
| 227 33.92070485 | |
| 228 33.77192982 | |
| 229 33.62445415 | |
| 230 33.47826087 | |
| 231 33.33333333 | |
| 232 33.62068966 | |
| 233 33.47639485 | |
| 234 33.33333333 | |
| 235 33.19148936 | |
| 236 33.47457627 | |
| 237 33.33333333 | |
| 238 33.61344538 | |
| 239 33.47280335 | |
| 240 33.33333333 | |
| 241 33.19502075 | |
| 242 33.05785124 | |
| 243 32.92181070 | |
| 244 32.78688525 | |
| 245 32.65306122 | |
| 246 32.52032520 | |
| 247 32.79352227 | |
| 248 33.06451613 | |
| 249 33.33333333 | |
| 250 33.20000000 | |
| 251 33.46613546 | |
| 252 33.33333333 | |
| 253 33.20158103 | |
| 254 33.46456693 | |
| 255 33.72549020 | |
| 256 33.59375000 | |
| 257 33.46303502 | |
| 258 33.33333333 | |
| 259 33.20463320 | |
| 260 33.07692308 | |
| 261 33.33333333 | |
| 262 33.20610687 | |
| 263 33.07984791 | |
| 264 33.33333333 | |
| 265 33.58490566 | |
| 266 33.45864662 | |
| 267 33.70786517 | |
| 268 33.58208955 | |
| 269 33.45724907 | |
| 270 33.33333333 | |
| 271 33.21033210 | |
| 272 33.08823529 | |
| 273 32.96703297 | |
| 274 33.21167883 | |
| 275 33.09090909 | |
| 276 32.97101449 | |
| 277 32.85198556 | |
| 278 32.73381295 | |
| 279 32.97491039 | |
| 280 33.21428571 | |
| 281 33.45195730 | |
| 282 33.33333333 | |
| 283 33.21554770 | |
| 284 33.45070423 | |
| 285 33.33333333 | |
| 286 33.56643357 | |
| 287 33.79790941 | |
| 288 34.02777778 | |
| 289 33.91003460 | |
| 290 33.79310345 | |
| 291 34.02061856 | |
| 292 33.90410959 | |
| 293 34.12969283 | |
| 294 34.01360544 | |
| 295 34.23728814 | |
| 296 34.45945946 | |
| 297 34.34343434 | |
| 298 34.22818792 | |
| 299 34.11371237 | |
| 300 34.00000000 | |
| 301 33.88704319 | |
| 302 33.77483444 | |
| 303 33.99339934 | |
| 304 33.88157895 | |
| 305 33.77049180 | |
| 306 33.66013072 | |
| 307 33.55048860 | |
| 308 33.76623377 | |
| 309 33.65695793 | |
| 310 33.54838710 | |
| 311 33.44051447 | |
| 312 33.65384615 | |
| 313 33.54632588 | |
| 314 33.43949045 | |
| 315 33.65079365 | |
| 316 33.54430380 | |
| 317 33.43848580 | |
| 318 33.33333333 | |
| 319 33.22884013 | |
| 320 33.12500000 | |
| 321 33.02180685 | |
| 322 32.91925466 | |
| 323 33.12693498 | |
| 324 33.02469136 | |
| 325 32.92307692 | |
| 326 32.82208589 | |
| 327 33.02752294 | |
| 328 33.23170732 | |
| 329 33.13069909 | |
| 330 33.33333333 | |
| 331 33.23262840 | |
| 332 33.43373494 | |
| 333 33.33333333 | |
| 334 33.23353293 | |
| 335 33.43283582 | |
| 336 33.63095238 | |
| 337 33.53115727 | |
| 338 33.43195266 | |
| 339 33.62831858 | |
| 340 33.52941176 | |
| 341 33.72434018 | |
| 342 33.91812865 | |
| 343 33.81924198 | |
| 344 34.01162791 | |
| 345 34.20289855 | |
| 346 34.10404624 | |
| 347 34.29394813 | |
| 348 34.19540230 | |
| 349 34.09742120 | |
| 350 34.00000000 | |
| 351 33.90313390 | |
| 352 33.80681818 | |
| 353 33.71104816 | |
| 354 33.61581921 | |
| 355 33.80281690 | |
| 356 33.70786517 | |
| 357 33.89355742 | |
| 358 33.79888268 | |
| 359 33.70473538 | |
| 360 33.61111111 | |
| 361 33.79501385 | |
| 362 33.70165746 | |
| 363 33.60881543 | |
| 364 33.51648352 | |
| 365 33.42465753 | |
| 366 33.33333333 | |
| 367 33.51498638 | |
| 368 33.42391304 | |
| 369 33.33333333 | |
| 370 33.24324324 | |
| 371 33.15363881 | |
| 372 33.33333333 | |
| 373 33.24396783 | |
| 374 33.42245989 | |
| 375 33.33333333 | |
| 376 33.24468085 | |
| 377 33.15649867 | |
| 378 33.06878307 | |
| 379 32.98153034 | |
| 380 33.15789474 | |
| 381 33.07086614 | |
| 382 33.24607330 | |
| 383 33.15926893 | |
| 384 33.07291667 | |
| 385 32.98701299 | |
| 386 32.90155440 | |
| 387 32.81653747 | |
| 388 32.98969072 | |
| 389 32.90488432 | |
| 390 32.82051282 | |
| 391 32.73657289 | |
| 392 32.65306122 | |
| 393 32.56997455 | |
| 394 32.74111675 | |
| 395 32.65822785 | |
| 396 32.57575758 | |
| 397 32.74559194 | |
| 398 32.66331658 | |
| 399 32.58145363 | |
| 400 32.50000000 | |
| 401 32.66832918 | |
| 402 32.83582090 | |
| 403 33.00248139 | |
| 404 32.92079208 | |
| 405 32.83950617 | |
| 406 32.75862069 | |
| 407 32.67813268 | |
| 408 32.59803922 | |
| 409 32.51833741 | |
| 410 32.43902439 | |
| 411 32.36009732 | |
| 412 32.52427184 | |
| 413 32.68765133 | |
| 414 32.85024155 | |
| 415 33.01204819 | |
| 416 32.93269231 | |
| 417 32.85371703 | |
| 418 32.77511962 | |
| 419 32.69689737 | |
| 420 32.85714286 | |
| 421 33.01662708 | |
| 422 32.93838863 | |
| 423 32.86052009 | |
| 424 32.78301887 | |
| 425 32.70588235 | |
| 426 32.62910798 | |
| 427 32.55269321 | |
| 428 32.47663551 | |
| 429 32.40093240 | |
| 430 32.55813953 | |
| 431 32.48259861 | |
| 432 32.63888889 | |
| 433 32.79445727 | |
| 434 32.94930876 | |
| 435 32.87356322 | |
| 436 32.79816514 | |
| 437 32.72311213 | |
| 438 32.64840183 | |
| 439 32.80182232 | |
| 440 32.72727273 | |
| 441 32.65306122 | |
| 442 32.57918552 | |
| 443 32.50564334 | |
| 444 32.65765766 | |
| 445 32.58426966 | |
| 446 32.51121076 | |
| 447 32.43847875 | |
| 448 32.36607143 | |
| 449 32.29398664 | |
| 450 32.22222222 | |
| 451 32.15077605 | |
| 452 32.07964602 | |
| 453 32.22958057 | |
| 454 32.15859031 | |
| 455 32.08791209 | |
| 456 32.01754386 | |
| 457 31.94748359 | |
| 458 31.87772926 | |
| 459 31.80827887 | |
| 460 31.73913043 | |
| 461 31.67028200 | |
| 462 31.60173160 | |
| 463 31.74946004 | |
| 464 31.89655172 | |
| 465 32.04301075 | |
| 466 31.97424893 | |
| 467 31.90578158 | |
| 468 31.83760684 | |
| 469 31.76972281 | |
| 470 31.70212766 | |
| 471 31.84713376 | |
| 472 31.77966102 | |
| 473 31.71247357 | |
| 474 31.85654008 | |
| 475 32.00000000 | |
| 476 31.93277311 | |
| 477 32.07547170 | |
| 478 32.00836820 | |
| 479 31.94154489 | |
| 480 31.87500000 | |
| 481 31.80873181 | |
| 482 31.95020747 | |
| 483 31.88405797 | |
| 484 32.02479339 | |
| 485 32.16494845 | |
| 486 32.09876543 | |
| 487 32.03285421 | |
| 488 31.96721311 | |
| 489 32.10633947 | |
| 490 32.24489796 | |
| 491 32.38289206 | |
| 492 32.52032520 | |
| 493 32.45436105 | |
| 494 32.38866397 | |
| 495 32.32323232 | |
| 496 32.25806452 | |
| 497 32.19315895 | |
| 498 32.12851406 | |
| 499 32.26452906 | |
| 500 32.20000000 | |
| 501 32.13572854 | |
| 502 32.07171315 | |
| 503 32.00795229 | |
| 504 32.14285714 | |
| 505 32.07920792 | |
| 506 32.01581028 | |
| 507 32.14990138 | |
| 508 32.28346457 | |
| 509 32.41650295 | |
| 510 32.35294118 | |
| 511 32.28962818 | |
| 512 32.22656250 | |
| 513 32.16374269 | |
| 514 32.10116732 | |
| 515 32.23300971 | |
| 516 32.17054264 | |
| 517 32.30174081 | |
| 518 32.23938224 | |
| 519 32.17726397 | |
| 520 32.30769231 | |
| 521 32.24568138 | |
| 522 32.37547893 | |
| 523 32.31357553 | |
| 524 32.25190840 | |
| 525 32.38095238 | |
| 526 32.31939163 | |
| 527 32.44781784 | |
| 528 32.38636364 | |
| 529 32.32514178 | |
| 530 32.26415094 | |
| 531 32.20338983 | |
| 532 32.14285714 | |
| 533 32.27016886 | |
| 534 32.20973783 | |
| 535 32.14953271 | |
| 536 32.08955224 | |
| 537 32.21601490 | |
| 538 32.15613383 | |
| 539 32.09647495 | |
| 540 32.03703704 | |
| 541 32.16266174 | |
| 542 32.10332103 | |
| 543 32.04419890 | |
| 544 32.16911765 | |
| 545 32.29357798 | |
| 546 32.41758242 | |
| 547 32.54113346 | |
| 548 32.66423358 | |
| 549 32.60473588 | |
| 550 32.54545455 | |
| 551 32.48638838 | |
| 552 32.60869565 | |
| 553 32.73056058 | |
| 554 32.67148014 | |
| 555 32.79279279 | |
| 556 32.73381295 | |
| 557 32.67504488 | |
| 558 32.61648746 | |
| 559 32.55813953 | |
| 560 32.67857143 | |
| 561 32.79857398 | |
| 562 32.74021352 | |
| 563 32.68206039 | |
| 564 32.80141844 | |
| 565 32.92035398 | |
| 566 33.03886926 | |
| 567 32.98059965 | |
| 568 33.09859155 | |
| 569 33.21616872 | |
| 570 33.15789474 | |
| 571 33.27495622 | |
| 572 33.21678322 | |
| 573 33.15881326 | |
| 574 33.27526132 | |
| 575 33.39130435 | |
| 576 33.33333333 | |
| 577 33.44887348 | |
| 578 33.56401384 | |
| 579 33.67875648 | |
| 580 33.79310345 | |
| 581 33.73493976 | |
| 582 33.84879725 | |
| 583 33.79073756 | |
| 584 33.90410959 | |
| 585 33.84615385 | |
| 586 33.95904437 | |
| 587 33.90119250 | |
| 588 33.84353741 | |
| 589 33.78607810 | |
| 590 33.72881356 | |
| 591 33.67174281 | |
| 592 33.61486486 | |
| 593 33.72681282 | |
| 594 33.83838384 | |
| 595 33.78151261 | |
| 596 33.72483221 | |
| 597 33.66834171 | |
| 598 33.61204013 | |
| 599 33.55592654 | |
| 600 33.50000000 | |
| 601 33.61064892 | |
| 602 33.72093023 | |
| 603 33.83084577 | |
| 604 33.77483444 | |
| 605 33.71900826 | |
| 606 33.82838284 | |
| 607 33.77265239 | |
| 608 33.71710526 | |
| 609 33.82594417 | |
| 610 33.77049180 | |
| 611 33.71522095 | |
| 612 33.82352941 | |
| 613 33.76835237 | |
| 614 33.71335505 | |
| 615 33.65853659 | |
| 616 33.76623377 | |
| 617 33.87358185 | |
| 618 33.81877023 | |
| 619 33.92568659 | |
| 620 33.87096774 | |
| 621 33.81642512 | |
| 622 33.92282958 | |
| 623 33.86837881 | |
| 624 33.81410256 | |
| 625 33.76000000 | |
| 626 33.70607029 | |
| 627 33.81180223 | |
| 628 33.91719745 | |
| 629 34.02225755 | |
| 630 33.96825397 | |
| 631 34.07290016 | |
| 632 34.01898734 | |
| 633 34.12322275 | |
| 634 34.22712934 | |
| 635 34.17322835 | |
| 636 34.11949686 | |
| 637 34.06593407 | |
| 638 34.01253918 | |
| 639 33.95931142 | |
| 640 34.06250000 | |
| 641 34.00936037 | |
| 642 34.11214953 | |
| 643 34.05909798 | |
| 644 34.00621118 | |
| 645 34.10852713 | |
| 646 34.05572755 | |
| 647 34.15765070 | |
| 648 34.10493827 | |
| 649 34.05238829 | |
| 650 34.15384615 | |
| 651 34.25499232 | |
| 652 34.35582822 | |
| 653 34.45635528 | |
| 654 34.40366972 | |
| 655 34.35114504 | |
| 656 34.45121951 | |
| 657 34.55098935 | |
| 658 34.49848024 | |
| 659 34.44613050 | |
| 660 34.39393939 | |
| 661 34.34190620 | |
| 662 34.29003021 | |
| 663 34.38914027 | |
| 664 34.33734940 | |
| 665 34.28571429 | |
| 666 34.23423423 | |
| 667 34.18290855 | |
| 668 34.13173653 | |
| 669 34.08071749 | |
| 670 34.17910448 | |
| 671 34.12816692 | |
| 672 34.07738095 | |
| 673 34.02674591 | |
| 674 33.97626113 | |
| 675 34.07407407 | |
| 676 34.17159763 | |
| 677 34.12112260 | |
| 678 34.07079646 | |
| 679 34.02061856 | |
| 680 34.11764706 | |
| 681 34.21439060 | |
| 682 34.16422287 | |
| 683 34.11420205 | |
| 684 34.21052632 | |
| 685 34.16058394 | |
| 686 34.11078717 | |
| 687 34.06113537 | |
| 688 34.15697674 | |
| 689 34.25253991 | |
| 690 34.20289855 | |
| 691 34.29811867 | |
| 692 34.39306358 | |
| 693 34.34343434 | |
| 694 34.29394813 | |
| 695 34.38848921 | |
| 696 34.48275862 | |
| 697 34.43328551 | |
| 698 34.38395415 | |
| 699 34.33476395 | |
| 700 34.42857143 | |
| 701 34.37945792 | |
| 702 34.33048433 | |
| 703 34.28165007 | |
| 704 34.23295455 | |
| 705 34.18439716 | |
| 706 34.13597734 | |
| 707 34.22913720 | |
| 708 34.18079096 | |
| 709 34.13258110 | |
| 710 34.08450704 | |
| 711 34.17721519 | |
| 712 34.12921348 | |
| 713 34.08134642 | |
| 714 34.17366947 | |
| 715 34.12587413 | |
| 716 34.07821229 | |
| 717 34.17015342 | |
| 718 34.12256267 | |
| 719 34.21418637 | |
| 720 34.16666667 | |
| 721 34.11927878 | |
| 722 34.07202216 | |
| 723 34.02489627 | |
| 724 33.97790055 | |
| 725 33.93103448 | |
| 726 34.02203857 | |
| 727 33.97524072 | |
| 728 33.92857143 | |
| 729 34.01920439 | |
| 730 34.10958904 | |
| 731 34.06292750 | |
| 732 34.15300546 | |
| 733 34.10641201 | |
| 734 34.05994550 | |
| 735 34.01360544 | |
| 736 33.96739130 | |
| 737 34.05698779 | |
| 738 34.01084011 | |
| 739 34.10013532 | |
| 740 34.05405405 | |
| 741 34.14304993 | |
| 742 34.09703504 | |
| 743 34.05114401 | |
| 744 34.13978495 | |
| 745 34.09395973 | |
| 746 34.04825737 | |
| 747 34.13654618 | |
| 748 34.09090909 | |
| 749 34.17890521 | |
| 750 34.26666667 | |
| Final result: 34.2667 +/- 1.7342 | |
| Random chance: 19.8992 +/- 1.4588 | |