| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 100.00000000 | |
| 4 75.00000000 | |
| 5 80.00000000 | |
| 6 83.33333333 | |
| 7 85.71428571 | |
| 8 87.50000000 | |
| 9 77.77777778 | |
| 10 80.00000000 | |
| 11 72.72727273 | |
| 12 75.00000000 | |
| 13 69.23076923 | |
| 14 71.42857143 | |
| 15 73.33333333 | |
| 16 75.00000000 | |
| 17 76.47058824 | |
| 18 72.22222222 | |
| 19 73.68421053 | |
| 20 70.00000000 | |
| 21 71.42857143 | |
| 22 72.72727273 | |
| 23 73.91304348 | |
| 24 75.00000000 | |
| 25 72.00000000 | |
| 26 69.23076923 | |
| 27 70.37037037 | |
| 28 67.85714286 | |
| 29 68.96551724 | |
| 30 70.00000000 | |
| 31 67.74193548 | |
| 32 68.75000000 | |
| 33 66.66666667 | |
| 34 67.64705882 | |
| 35 68.57142857 | |
| 36 66.66666667 | |
| 37 67.56756757 | |
| 38 68.42105263 | |
| 39 69.23076923 | |
| 40 70.00000000 | |
| 41 68.29268293 | |
| 42 66.66666667 | |
| 43 65.11627907 | |
| 44 63.63636364 | |
| 45 64.44444444 | |
| 46 65.21739130 | |
| 47 65.95744681 | |
| 48 66.66666667 | |
| 49 65.30612245 | |
| 50 64.00000000 | |
| 51 62.74509804 | |
| 52 63.46153846 | |
| 53 64.15094340 | |
| 54 62.96296296 | |
| 55 61.81818182 | |
| 56 62.50000000 | |
| 57 63.15789474 | |
| 58 63.79310345 | |
| 59 64.40677966 | |
| 60 65.00000000 | |
| 61 65.57377049 | |
| 62 66.12903226 | |
| 63 66.66666667 | |
| 64 67.18750000 | |
| 65 66.15384615 | |
| 66 66.66666667 | |
| 67 67.16417910 | |
| 68 67.64705882 | |
| 69 68.11594203 | |
| 70 68.57142857 | |
| 71 69.01408451 | |
| 72 69.44444444 | |
| 73 69.86301370 | |
| 74 70.27027027 | |
| 75 70.66666667 | |
| 76 71.05263158 | |
| 77 71.42857143 | |
| 78 71.79487179 | |
| 79 72.15189873 | |
| 80 72.50000000 | |
| 81 72.83950617 | |
| 82 73.17073171 | |
| 83 72.28915663 | |
| 84 72.61904762 | |
| 85 72.94117647 | |
| 86 73.25581395 | |
| 87 73.56321839 | |
| 88 73.86363636 | |
| 89 74.15730337 | |
| 90 74.44444444 | |
| 91 74.72527473 | |
| 92 75.00000000 | |
| 93 74.19354839 | |
| 94 73.40425532 | |
| 95 73.68421053 | |
| 96 73.95833333 | |
| 97 73.19587629 | |
| 98 73.46938776 | |
| 99 73.73737374 | |
| 100 74.00000000 | |
| 101 74.25742574 | |
| 102 74.50980392 | |
| 103 73.78640777 | |
| 104 73.07692308 | |
| 105 73.33333333 | |
| 106 72.64150943 | |
| 107 72.89719626 | |
| 108 72.22222222 | |
| 109 71.55963303 | |
| 110 71.81818182 | |
| 111 72.07207207 | |
| 112 72.32142857 | |
| 113 71.68141593 | |
| 114 71.92982456 | |
| 115 72.17391304 | |
| 116 71.55172414 | |
| 117 70.94017094 | |
| 118 71.18644068 | |
| 119 70.58823529 | |
| 120 70.83333333 | |
| 121 70.24793388 | |
| 122 70.49180328 | |
| 123 70.73170732 | |
| 124 70.96774194 | |
| 125 71.20000000 | |
| 126 71.42857143 | |
| 127 71.65354331 | |
| 128 71.87500000 | |
| 129 71.31782946 | |
| 130 71.53846154 | |
| 131 71.75572519 | |
| 132 71.21212121 | |
| 133 71.42857143 | |
| 134 70.89552239 | |
| 135 70.37037037 | |
| 136 70.58823529 | |
| 137 70.80291971 | |
| 138 70.28985507 | |
| 139 70.50359712 | |
| 140 70.00000000 | |
| 141 69.50354610 | |
| 142 69.71830986 | |
| 143 69.93006993 | |
| 144 70.13888889 | |
| 145 70.34482759 | |
| 146 69.86301370 | |
| 147 70.06802721 | |
| 148 69.59459459 | |
| 149 69.79865772 | |
| 150 70.00000000 | |
| 151 70.19867550 | |
| 152 70.39473684 | |
| 153 70.58823529 | |
| 154 70.77922078 | |
| 155 70.32258065 | |
| 156 70.51282051 | |
| 157 70.06369427 | |
| 158 70.25316456 | |
| 159 70.44025157 | |
| 160 70.62500000 | |
| 161 70.80745342 | |
| 162 70.98765432 | |
| 163 70.55214724 | |
| 164 70.73170732 | |
| 165 70.90909091 | |
| 166 71.08433735 | |
| 167 71.25748503 | |
| 168 71.42857143 | |
| 169 71.00591716 | |
| 170 70.58823529 | |
| 171 70.76023392 | |
| 172 70.34883721 | |
| 173 70.52023121 | |
| 174 70.68965517 | |
| 175 70.28571429 | |
| 176 70.45454545 | |
| 177 70.62146893 | |
| 178 70.22471910 | |
| 179 70.39106145 | |
| 180 70.55555556 | |
| 181 70.16574586 | |
| 182 70.32967033 | |
| 183 69.94535519 | |
| 184 70.10869565 | |
| 185 70.27027027 | |
| 186 70.43010753 | |
| 187 70.58823529 | |
| 188 70.21276596 | |
| 189 70.37037037 | |
| 190 70.52631579 | |
| 191 70.68062827 | |
| 192 70.31250000 | |
| 193 69.94818653 | |
| 194 70.10309278 | |
| 195 70.25641026 | |
| 196 70.40816327 | |
| 197 70.55837563 | |
| 198 70.70707071 | |
| 199 70.35175879 | |
| 200 70.50000000 | |
| 201 70.64676617 | |
| 202 70.79207921 | |
| 203 70.93596059 | |
| 204 71.07843137 | |
| 205 71.21951220 | |
| 206 70.87378641 | |
| 207 71.01449275 | |
| 208 71.15384615 | |
| 209 71.29186603 | |
| 210 71.42857143 | |
| 211 71.56398104 | |
| 212 71.22641509 | |
| 213 71.36150235 | |
| 214 71.49532710 | |
| 215 71.62790698 | |
| 216 71.29629630 | |
| 217 71.42857143 | |
| 218 71.55963303 | |
| 219 71.68949772 | |
| 220 71.36363636 | |
| 221 71.49321267 | |
| 222 71.62162162 | |
| 223 71.30044843 | |
| 224 71.42857143 | |
| 225 71.11111111 | |
| 226 70.79646018 | |
| 227 70.92511013 | |
| 228 71.05263158 | |
| 229 71.17903930 | |
| 230 71.30434783 | |
| 231 70.99567100 | |
| 232 70.68965517 | |
| 233 70.38626609 | |
| 234 70.51282051 | |
| 235 70.21276596 | |
| 236 70.33898305 | |
| 237 70.04219409 | |
| 238 69.74789916 | |
| 239 69.87447699 | |
| 240 70.00000000 | |
| 241 70.12448133 | |
| 242 69.83471074 | |
| 243 69.95884774 | |
| 244 69.67213115 | |
| 245 69.79591837 | |
| 246 69.91869919 | |
| 247 70.04048583 | |
| 248 70.16129032 | |
| 249 70.28112450 | |
| 250 70.40000000 | |
| 251 70.51792829 | |
| 252 70.63492063 | |
| 253 70.75098814 | |
| 254 70.86614173 | |
| 255 70.98039216 | |
| 256 71.09375000 | |
| 257 70.81712062 | |
| 258 70.93023256 | |
| 259 71.04247104 | |
| 260 71.15384615 | |
| 261 71.26436782 | |
| 262 71.37404580 | |
| 263 71.48288973 | |
| 264 71.59090909 | |
| 265 71.69811321 | |
| 266 71.42857143 | |
| 267 71.53558052 | |
| 268 71.64179104 | |
| 269 71.74721190 | |
| 270 71.85185185 | |
| 271 71.95571956 | |
| 272 72.05882353 | |
| 273 71.79487179 | |
| 274 71.53284672 | |
| 275 71.27272727 | |
| 276 71.37681159 | |
| 277 71.48014440 | |
| 278 71.58273381 | |
| 279 71.68458781 | |
| 280 71.78571429 | |
| 281 71.88612100 | |
| 282 71.98581560 | |
| 283 71.73144876 | |
| 284 71.83098592 | |
| 285 71.92982456 | |
| 286 72.02797203 | |
| 287 71.77700348 | |
| 288 71.87500000 | |
| 289 71.62629758 | |
| 290 71.72413793 | |
| 291 71.82130584 | |
| 292 71.91780822 | |
| 293 72.01365188 | |
| 294 71.76870748 | |
| 295 71.86440678 | |
| 296 71.95945946 | |
| 297 72.05387205 | |
| 298 72.14765101 | |
| 299 72.24080268 | |
| 300 72.33333333 | |
| 301 72.42524917 | |
| 302 72.51655629 | |
| 303 72.60726073 | |
| 304 72.69736842 | |
| 305 72.78688525 | |
| 306 72.54901961 | |
| 307 72.63843648 | |
| 308 72.72727273 | |
| 309 72.81553398 | |
| 310 72.58064516 | |
| 311 72.66881029 | |
| 312 72.75641026 | |
| 313 72.84345048 | |
| 314 72.92993631 | |
| 315 73.01587302 | |
| 316 73.10126582 | |
| 317 73.18611987 | |
| 318 73.27044025 | |
| 319 73.04075235 | |
| 320 72.81250000 | |
| 321 72.89719626 | |
| 322 72.98136646 | |
| 323 73.06501548 | |
| 324 73.14814815 | |
| 325 73.23076923 | |
| 326 73.31288344 | |
| 327 73.08868502 | |
| 328 73.17073171 | |
| 329 73.25227964 | |
| 330 73.03030303 | |
| 331 72.80966767 | |
| 332 72.89156627 | |
| 333 72.97297297 | |
| 334 73.05389222 | |
| 335 72.83582090 | |
| 336 72.61904762 | |
| 337 72.40356083 | |
| 338 72.18934911 | |
| 339 72.27138643 | |
| 340 72.35294118 | |
| 341 72.43401760 | |
| 342 72.51461988 | |
| 343 72.59475219 | |
| 344 72.67441860 | |
| 345 72.46376812 | |
| 346 72.54335260 | |
| 347 72.62247839 | |
| 348 72.41379310 | |
| 349 72.49283668 | |
| 350 72.57142857 | |
| 351 72.64957265 | |
| 352 72.72727273 | |
| 353 72.80453258 | |
| 354 72.59887006 | |
| 355 72.39436620 | |
| 356 72.47191011 | |
| 357 72.54901961 | |
| 358 72.62569832 | |
| 359 72.42339833 | |
| 360 72.50000000 | |
| 361 72.57617729 | |
| 362 72.65193370 | |
| 363 72.72727273 | |
| 364 72.80219780 | |
| 365 72.87671233 | |
| 366 72.67759563 | |
| 367 72.75204360 | |
| 368 72.82608696 | |
| 369 72.89972900 | |
| 370 72.97297297 | |
| 371 72.77628032 | |
| 372 72.58064516 | |
| 373 72.65415550 | |
| 374 72.72727273 | |
| 375 72.53333333 | |
| 376 72.60638298 | |
| 377 72.67904509 | |
| 378 72.75132275 | |
| 379 72.55936675 | |
| 380 72.63157895 | |
| 381 72.44094488 | |
| 382 72.51308901 | |
| 383 72.32375979 | |
| 384 72.39583333 | |
| 385 72.46753247 | |
| 386 72.27979275 | |
| 387 72.35142119 | |
| 388 72.16494845 | |
| 389 71.97943445 | |
| 390 71.79487179 | |
| 391 71.61125320 | |
| 392 71.68367347 | |
| 393 71.75572519 | |
| 394 71.57360406 | |
| 395 71.64556962 | |
| 396 71.46464646 | |
| 397 71.53652393 | |
| 398 71.35678392 | |
| 399 71.42857143 | |
| 400 71.50000000 | |
| 401 71.57107232 | |
| 402 71.64179104 | |
| 403 71.71215881 | |
| 404 71.78217822 | |
| 405 71.85185185 | |
| 406 71.92118227 | |
| 407 71.99017199 | |
| 408 72.05882353 | |
| 409 71.88264059 | |
| 410 71.95121951 | |
| 411 72.01946472 | |
| 412 72.08737864 | |
| 413 72.15496368 | |
| 414 72.22222222 | |
| 415 72.04819277 | |
| 416 72.11538462 | |
| 417 71.94244604 | |
| 418 72.00956938 | |
| 419 71.83770883 | |
| 420 71.90476190 | |
| 421 71.97149644 | |
| 422 72.03791469 | |
| 423 72.10401891 | |
| 424 72.16981132 | |
| 425 72.00000000 | |
| 426 71.83098592 | |
| 427 71.66276347 | |
| 428 71.49532710 | |
| 429 71.56177156 | |
| 430 71.39534884 | |
| 431 71.46171694 | |
| 432 71.52777778 | |
| 433 71.36258661 | |
| 434 71.42857143 | |
| 435 71.49425287 | |
| 436 71.55963303 | |
| 437 71.62471396 | |
| 438 71.46118721 | |
| 439 71.52619590 | |
| 440 71.59090909 | |
| 441 71.65532880 | |
| 442 71.49321267 | |
| 443 71.55756208 | |
| 444 71.62162162 | |
| 445 71.68539326 | |
| 446 71.74887892 | |
| 447 71.81208054 | |
| 448 71.87500000 | |
| 449 71.93763920 | |
| 450 72.00000000 | |
| 451 72.06208426 | |
| 452 72.12389381 | |
| 453 72.18543046 | |
| 454 72.02643172 | |
| 455 71.86813187 | |
| 456 71.92982456 | |
| 457 71.99124726 | |
| 458 72.05240175 | |
| 459 72.11328976 | |
| 460 71.95652174 | |
| 461 72.01735358 | |
| 462 72.07792208 | |
| 463 71.92224622 | |
| 464 71.76724138 | |
| 465 71.82795699 | |
| 466 71.88841202 | |
| 467 71.94860814 | |
| 468 72.00854701 | |
| 469 71.85501066 | |
| 470 71.91489362 | |
| 471 71.97452229 | |
| 472 72.03389831 | |
| 473 71.88160677 | |
| 474 71.94092827 | |
| 475 71.78947368 | |
| 476 71.84873950 | |
| 477 71.90775681 | |
| 478 71.96652720 | |
| 479 72.02505219 | |
| 480 71.87500000 | |
| 481 71.93347193 | |
| 482 71.99170124 | |
| 483 72.04968944 | |
| 484 72.10743802 | |
| 485 72.16494845 | |
| 486 72.22222222 | |
| 487 72.07392197 | |
| 488 72.13114754 | |
| 489 72.18813906 | |
| 490 72.24489796 | |
| 491 72.30142566 | |
| 492 72.35772358 | |
| 493 72.21095335 | |
| 494 72.26720648 | |
| 495 72.32323232 | |
| 496 72.17741935 | |
| 497 72.03219316 | |
| 498 72.08835341 | |
| 499 72.14428858 | |
| 500 72.20000000 | |
| 501 72.25548902 | |
| 502 72.31075697 | |
| 503 72.36580517 | |
| 504 72.22222222 | |
| 505 72.27722772 | |
| 506 72.13438735 | |
| 507 72.18934911 | |
| 508 72.24409449 | |
| 509 72.10216110 | |
| 510 72.15686275 | |
| 511 72.01565558 | |
| 512 72.07031250 | |
| 513 72.12475634 | |
| 514 72.17898833 | |
| 515 72.23300971 | |
| 516 72.09302326 | |
| 517 72.14700193 | |
| 518 72.20077220 | |
| 519 72.25433526 | |
| 520 72.11538462 | |
| 521 72.16890595 | |
| 522 72.22222222 | |
| 523 72.08413002 | |
| 524 71.94656489 | |
| 525 71.80952381 | |
| 526 71.67300380 | |
| 527 71.53700190 | |
| 528 71.40151515 | |
| 529 71.26654064 | |
| 530 71.32075472 | |
| 531 71.37476460 | |
| 532 71.42857143 | |
| 533 71.29455910 | |
| 534 71.16104869 | |
| 535 71.02803738 | |
| 536 71.08208955 | |
| 537 71.13594041 | |
| 538 71.18959108 | |
| 539 71.24304267 | |
| 540 71.11111111 | |
| 541 71.16451017 | |
| 542 71.21771218 | |
| 543 71.08655617 | |
| 544 71.13970588 | |
| 545 71.19266055 | |
| 546 71.24542125 | |
| 547 71.29798903 | |
| 548 71.35036496 | |
| 549 71.40255009 | |
| 550 71.45454545 | |
| 551 71.32486388 | |
| 552 71.19565217 | |
| 553 71.24773960 | |
| 554 71.29963899 | |
| 555 71.35135135 | |
| 556 71.22302158 | |
| 557 71.27468582 | |
| 558 71.14695341 | |
| 559 71.19856887 | |
| 560 71.25000000 | |
| 561 71.30124777 | |
| 562 71.35231317 | |
| 563 71.40319716 | |
| 564 71.45390071 | |
| 565 71.32743363 | |
| 566 71.37809187 | |
| 567 71.42857143 | |
| 568 71.47887324 | |
| 569 71.35325132 | |
| 570 71.22807018 | |
| 571 71.27845884 | |
| 572 71.15384615 | |
| 573 71.20418848 | |
| 574 71.25435540 | |
| 575 71.30434783 | |
| 576 71.35416667 | |
| 577 71.40381282 | |
| 578 71.28027682 | |
| 579 71.32987910 | |
| 580 71.37931034 | |
| 581 71.25645439 | |
| 582 71.13402062 | |
| 583 71.01200686 | |
| 584 71.06164384 | |
| 585 71.11111111 | |
| 586 71.16040956 | |
| 587 71.20954003 | |
| 588 71.25850340 | |
| 589 71.30730051 | |
| 590 71.35593220 | |
| 591 71.23519459 | |
| 592 71.28378378 | |
| 593 71.33220911 | |
| 594 71.38047138 | |
| 595 71.26050420 | |
| 596 71.14093960 | |
| 597 71.18927973 | |
| 598 71.23745819 | |
| 599 71.11853088 | |
| 600 71.16666667 | |
| 601 71.21464226 | |
| 602 71.09634551 | |
| 603 70.97844113 | |
| 604 71.02649007 | |
| 605 71.07438017 | |
| 606 71.12211221 | |
| 607 71.16968699 | |
| 608 71.21710526 | |
| 609 71.10016420 | |
| 610 71.14754098 | |
| 611 71.03109656 | |
| 612 70.91503268 | |
| 613 70.79934747 | |
| 614 70.84690554 | |
| 615 70.73170732 | |
| 616 70.61688312 | |
| 617 70.66450567 | |
| 618 70.71197411 | |
| 619 70.59773829 | |
| 620 70.64516129 | |
| 621 70.69243156 | |
| 622 70.73954984 | |
| 623 70.78651685 | |
| 624 70.83333333 | |
| 625 70.72000000 | |
| 626 70.76677316 | |
| 627 70.81339713 | |
| 628 70.70063694 | |
| 629 70.74721781 | |
| 630 70.79365079 | |
| 631 70.83993661 | |
| 632 70.72784810 | |
| 633 70.61611374 | |
| 634 70.66246057 | |
| 635 70.70866142 | |
| 636 70.75471698 | |
| 637 70.64364207 | |
| 638 70.68965517 | |
| 639 70.73552426 | |
| 640 70.78125000 | |
| 641 70.82683307 | |
| 642 70.71651090 | |
| 643 70.60653188 | |
| 644 70.65217391 | |
| 645 70.54263566 | |
| 646 70.58823529 | |
| 647 70.47913447 | |
| 648 70.52469136 | |
| 649 70.41602465 | |
| 650 70.46153846 | |
| 651 70.50691244 | |
| 652 70.39877301 | |
| 653 70.44410413 | |
| 654 70.33639144 | |
| 655 70.22900763 | |
| 656 70.12195122 | |
| 657 70.16742770 | |
| 658 70.21276596 | |
| 659 70.10622155 | |
| 660 70.15151515 | |
| 661 70.04538578 | |
| 662 70.09063444 | |
| 663 70.13574661 | |
| 664 70.18072289 | |
| 665 70.22556391 | |
| 666 70.27027027 | |
| 667 70.31484258 | |
| 668 70.20958084 | |
| 669 70.25411061 | |
| 670 70.29850746 | |
| 671 70.34277198 | |
| 672 70.38690476 | |
| 673 70.43090639 | |
| 674 70.47477745 | |
| 675 70.51851852 | |
| 676 70.56213018 | |
| 677 70.60561300 | |
| 678 70.64896755 | |
| 679 70.54491900 | |
| 680 70.58823529 | |
| 681 70.63142438 | |
| 682 70.67448680 | |
| 683 70.57101025 | |
| 684 70.61403509 | |
| 685 70.65693431 | |
| 686 70.69970845 | |
| 687 70.74235808 | |
| 688 70.78488372 | |
| 689 70.68214804 | |
| 690 70.72463768 | |
| 691 70.76700434 | |
| 692 70.66473988 | |
| 693 70.70707071 | |
| 694 70.74927954 | |
| 695 70.64748201 | |
| 696 70.54597701 | |
| 697 70.58823529 | |
| 698 70.48710602 | |
| 699 70.38626609 | |
| 700 70.28571429 | |
| 701 70.18544936 | |
| 702 70.22792023 | |
| 703 70.27027027 | |
| 704 70.31250000 | |
| 705 70.35460993 | |
| 706 70.39660057 | |
| 707 70.43847242 | |
| 708 70.48022599 | |
| 709 70.52186178 | |
| 710 70.56338028 | |
| 711 70.60478200 | |
| 712 70.64606742 | |
| 713 70.68723703 | |
| 714 70.58823529 | |
| 715 70.48951049 | |
| 716 70.53072626 | |
| 717 70.57182706 | |
| 718 70.47353760 | |
| 719 70.37552156 | |
| 720 70.41666667 | |
| 721 70.31900139 | |
| 722 70.36011080 | |
| 723 70.40110650 | |
| 724 70.44198895 | |
| 725 70.48275862 | |
| 726 70.38567493 | |
| 727 70.42640990 | |
| 728 70.46703297 | |
| 729 70.50754458 | |
| 730 70.54794521 | |
| 731 70.58823529 | |
| 732 70.62841530 | |
| 733 70.66848568 | |
| 734 70.70844687 | |
| 735 70.74829932 | |
| 736 70.65217391 | |
| 737 70.55630936 | |
| 738 70.59620596 | |
| 739 70.50067659 | |
| 740 70.40540541 | |
| 741 70.44534413 | |
| 742 70.35040431 | |
| 743 70.39030956 | |
| 744 70.29569892 | |
| 745 70.33557047 | |
| 746 70.37533512 | |
| 747 70.28112450 | |
| 748 70.18716578 | |
| 749 70.09345794 | |
| 750 70.13333333 | |
| Final result: 70.1333 +/- 1.6723 | |
| Random chance: 25.0083 +/- 1.5824 | |