File size: 12,675 Bytes
7395353 dd9bc2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 | common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 817 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 817 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.
task acc_norm
1 100.00000000
2 50.00000000
3 33.33333333
4 25.00000000
5 20.00000000
6 16.66666667
7 14.28571429
8 25.00000000
9 22.22222222
10 20.00000000
11 18.18181818
12 25.00000000
13 30.76923077
14 28.57142857
15 33.33333333
16 31.25000000
17 29.41176471
18 27.77777778
19 26.31578947
20 25.00000000
21 28.57142857
22 27.27272727
23 30.43478261
24 33.33333333
25 32.00000000
26 30.76923077
27 29.62962963
28 28.57142857
29 31.03448276
30 30.00000000
31 29.03225806
32 31.25000000
33 33.33333333
34 32.35294118
35 31.42857143
36 30.55555556
37 32.43243243
38 31.57894737
39 33.33333333
40 32.50000000
41 31.70731707
42 30.95238095
43 30.23255814
44 31.81818182
45 33.33333333
46 32.60869565
47 34.04255319
48 35.41666667
49 34.69387755
50 36.00000000
51 37.25490196
52 36.53846154
53 35.84905660
54 35.18518519
55 36.36363636
56 35.71428571
57 35.08771930
58 34.48275862
59 33.89830508
60 35.00000000
61 34.42622951
62 35.48387097
63 34.92063492
64 35.93750000
65 35.38461538
66 36.36363636
67 35.82089552
68 35.29411765
69 36.23188406
70 35.71428571
71 35.21126761
72 36.11111111
73 35.61643836
74 35.13513514
75 34.66666667
76 35.52631579
77 35.06493506
78 34.61538462
79 34.17721519
80 35.00000000
81 34.56790123
82 35.36585366
83 34.93975904
84 34.52380952
85 34.11764706
86 33.72093023
87 33.33333333
88 32.95454545
89 33.70786517
90 33.33333333
91 32.96703297
92 32.60869565
93 32.25806452
94 31.91489362
95 31.57894737
96 32.29166667
97 32.98969072
98 32.65306122
99 33.33333333
100 34.00000000
101 34.65346535
102 34.31372549
103 34.95145631
104 34.61538462
105 34.28571429
106 34.90566038
107 35.51401869
108 36.11111111
109 36.69724771
110 37.27272727
111 36.93693694
112 37.50000000
113 37.16814159
114 37.71929825
115 37.39130435
116 37.93103448
117 38.46153846
118 38.13559322
119 37.81512605
120 37.50000000
121 38.01652893
122 37.70491803
123 38.21138211
124 37.90322581
125 38.40000000
126 38.09523810
127 37.79527559
128 37.50000000
129 37.98449612
130 38.46153846
131 38.16793893
132 37.87878788
133 37.59398496
134 37.31343284
135 37.03703704
136 36.76470588
137 37.22627737
138 37.68115942
139 38.12949640
140 37.85714286
141 38.29787234
142 38.02816901
143 37.76223776
144 38.19444444
145 38.62068966
146 38.35616438
147 38.09523810
148 37.83783784
149 37.58389262
150 37.33333333
151 37.74834437
152 37.50000000
153 37.25490196
154 37.01298701
155 36.77419355
156 37.17948718
157 37.57961783
158 37.97468354
159 37.73584906
160 37.50000000
161 37.88819876
162 37.65432099
163 38.03680982
164 37.80487805
165 38.18181818
166 38.55421687
167 38.32335329
168 38.09523810
169 37.86982249
170 37.64705882
171 37.42690058
172 37.20930233
173 36.99421965
174 37.35632184
175 37.14285714
176 36.93181818
177 36.72316384
178 36.51685393
179 36.31284916
180 36.11111111
181 35.91160221
182 35.71428571
183 35.51912568
184 35.32608696
185 35.67567568
186 36.02150538
187 35.82887701
188 35.63829787
189 35.44973545
190 35.26315789
191 35.07853403
192 34.89583333
193 34.71502591
194 35.05154639
195 34.87179487
196 34.69387755
197 35.02538071
198 35.35353535
199 35.17587940
200 35.50000000
201 35.32338308
202 35.64356436
203 35.46798030
204 35.29411765
205 35.60975610
206 35.43689320
207 35.74879227
208 35.57692308
209 35.40669856
210 35.23809524
211 35.07109005
212 34.90566038
213 34.74178404
214 34.57943925
215 34.41860465
216 34.72222222
217 34.56221198
218 34.40366972
219 34.24657534
220 34.09090909
221 34.38914027
222 34.23423423
223 34.08071749
224 34.37500000
225 34.66666667
226 34.51327434
227 34.80176211
228 34.64912281
229 34.49781659
230 34.34782609
231 34.19913420
232 34.48275862
233 34.33476395
234 34.18803419
235 34.04255319
236 34.32203390
237 34.17721519
238 34.45378151
239 34.30962343
240 34.16666667
241 34.02489627
242 33.88429752
243 33.74485597
244 33.60655738
245 33.46938776
246 33.33333333
247 33.60323887
248 33.87096774
249 34.13654618
250 34.00000000
251 34.26294821
252 34.12698413
253 33.99209486
254 34.25196850
255 34.50980392
256 34.37500000
257 34.24124514
258 34.10852713
259 33.97683398
260 33.84615385
261 34.09961686
262 33.96946565
263 33.84030418
264 34.09090909
265 34.33962264
266 34.21052632
267 34.45692884
268 34.32835821
269 34.20074349
270 34.07407407
271 33.94833948
272 33.82352941
273 33.69963370
274 33.94160584
275 33.81818182
276 33.69565217
277 33.57400722
278 33.45323741
279 33.69175627
280 33.92857143
281 34.16370107
282 34.04255319
283 33.92226148
284 34.15492958
285 34.03508772
286 34.26573427
287 34.49477352
288 34.72222222
289 34.60207612
290 34.48275862
291 34.70790378
292 34.58904110
293 34.81228669
294 34.69387755
295 34.91525424
296 35.13513514
297 35.01683502
298 34.89932886
299 34.78260870
300 34.66666667
301 34.55149502
302 34.43708609
303 34.65346535
304 34.53947368
305 34.42622951
306 34.31372549
307 34.20195440
308 34.41558442
309 34.30420712
310 34.19354839
311 34.08360129
312 34.29487179
313 34.18530351
314 34.07643312
315 34.28571429
316 34.17721519
317 34.06940063
318 33.96226415
319 33.85579937
320 33.75000000
321 33.64485981
322 33.54037267
323 33.74613003
324 33.64197531
325 33.53846154
326 33.43558282
327 33.63914373
328 33.84146341
329 33.73860182
330 33.93939394
331 33.83685801
332 34.03614458
333 33.93393393
334 33.83233533
335 34.02985075
336 34.22619048
337 34.12462908
338 34.02366864
339 34.21828909
340 34.11764706
341 34.31085044
342 34.50292398
343 34.40233236
344 34.59302326
345 34.78260870
346 34.68208092
347 34.87031700
348 34.77011494
349 34.67048711
350 34.57142857
351 34.47293447
352 34.37500000
353 34.27762040
354 34.18079096
355 34.36619718
356 34.26966292
357 34.45378151
358 34.63687151
359 34.54038997
360 34.44444444
361 34.62603878
362 34.53038674
363 34.43526171
364 34.34065934
365 34.24657534
366 34.15300546
367 34.33242507
368 34.51086957
369 34.41734417
370 34.32432432
371 34.23180593
372 34.40860215
373 34.31635389
374 34.49197861
375 34.40000000
376 34.30851064
377 34.48275862
378 34.39153439
379 34.30079156
380 34.47368421
381 34.38320210
382 34.55497382
383 34.46475196
384 34.37500000
385 34.28571429
386 34.19689119
387 34.10852713
388 34.27835052
389 34.19023136
390 34.10256410
391 34.01534527
392 33.92857143
393 33.84223919
394 34.01015228
395 33.92405063
396 33.83838384
397 34.00503778
398 33.91959799
399 33.83458647
400 33.75000000
401 33.91521197
402 34.07960199
403 34.24317618
404 34.15841584
405 34.07407407
406 33.99014778
407 33.90663391
408 33.82352941
409 33.74083130
410 33.65853659
411 33.57664234
412 33.73786408
413 33.89830508
414 34.05797101
415 34.21686747
416 34.13461538
417 34.05275779
418 33.97129187
419 33.89021480
420 34.04761905
421 34.20427553
422 34.12322275
423 34.04255319
424 33.96226415
425 33.88235294
426 33.80281690
427 33.72365340
428 33.64485981
429 33.56643357
430 33.72093023
431 33.64269142
432 33.79629630
433 33.94919169
434 34.10138249
435 34.02298851
436 33.94495413
437 33.86727689
438 33.78995434
439 33.94077449
440 33.86363636
441 33.78684807
442 33.71040724
443 33.63431151
444 33.78378378
445 33.70786517
446 33.63228700
447 33.55704698
448 33.48214286
449 33.40757238
450 33.33333333
451 33.25942350
452 33.18584071
453 33.33333333
454 33.25991189
455 33.18681319
456 33.11403509
457 33.04157549
458 32.96943231
459 32.89760349
460 32.82608696
461 32.75488069
462 32.68398268
463 32.82937365
464 32.97413793
465 33.11827957
466 33.04721030
467 32.97644540
468 32.90598291
469 32.83582090
470 32.76595745
471 32.90870488
472 32.83898305
473 32.76955603
474 32.91139241
475 33.05263158
476 32.98319328
477 33.12368973
478 33.05439331
479 32.98538622
480 32.91666667
481 32.84823285
482 32.98755187
483 32.91925466
484 33.05785124
485 33.19587629
486 33.12757202
487 33.05954825
488 32.99180328
489 33.12883436
490 33.26530612
491 33.40122200
492 33.53658537
493 33.46855984
494 33.60323887
495 33.53535354
496 33.46774194
497 33.40040241
498 33.33333333
499 33.46693387
500 33.40000000
501 33.33333333
502 33.26693227
503 33.20079523
504 33.33333333
505 33.26732673
506 33.20158103
507 33.33333333
508 33.46456693
509 33.59528487
510 33.52941176
511 33.46379648
512 33.39843750
513 33.33333333
514 33.26848249
515 33.39805825
516 33.33333333
517 33.46228240
518 33.39768340
519 33.33333333
520 33.26923077
521 33.20537428
522 33.33333333
523 33.26959847
524 33.20610687
525 33.33333333
526 33.26996198
527 33.39658444
528 33.33333333
529 33.27032136
530 33.20754717
531 33.14500942
532 33.08270677
533 33.20825516
534 33.14606742
535 33.08411215
536 33.02238806
537 33.14711359
538 33.08550186
539 33.02411874
540 32.96296296
541 33.08687616
542 33.02583026
543 32.96500921
544 33.08823529
545 33.21100917
546 33.15018315
547 33.27239488
548 33.39416058
549 33.33333333
550 33.27272727
551 33.21234120
552 33.33333333
553 33.45388788
554 33.39350181
555 33.51351351
556 33.45323741
557 33.39317774
558 33.33333333
559 33.27370304
560 33.39285714
561 33.51158645
562 33.45195730
563 33.39253996
564 33.51063830
565 33.62831858
566 33.74558304
567 33.68606702
568 33.80281690
569 33.91915641
570 33.85964912
571 33.97548161
572 33.91608392
573 33.85689354
574 33.97212544
575 34.08695652
576 34.02777778
577 34.14211438
578 34.25605536
579 34.36960276
580 34.48275862
581 34.42340792
582 34.53608247
583 34.47684391
584 34.58904110
585 34.52991453
586 34.64163823
587 34.58262351
588 34.52380952
589 34.46519525
590 34.40677966
591 34.34856176
592 34.29054054
593 34.40134907
594 34.51178451
595 34.45378151
596 34.39597315
597 34.50586265
598 34.44816054
599 34.39065109
600 34.33333333
601 34.44259567
602 34.55149502
603 34.66003317
604 34.60264901
605 34.54545455
606 34.65346535
607 34.59637562
608 34.53947368
609 34.64696223
610 34.59016393
611 34.53355155
612 34.64052288
613 34.58401305
614 34.52768730
615 34.47154472
616 34.57792208
617 34.68395462
618 34.62783172
619 34.73344103
620 34.67741935
621 34.62157810
622 34.72668810
623 34.67094703
624 34.61538462
625 34.56000000
626 34.50479233
627 34.60925040
628 34.71337580
629 34.81717011
630 34.76190476
631 34.86529319
632 34.81012658
633 34.91311216
634 35.01577287
635 34.96062992
636 34.90566038
637 34.85086342
638 34.79623824
639 34.74178404
640 34.84375000
641 34.78939158
642 34.89096573
643 34.83670295
644 34.78260870
645 34.88372093
646 34.82972136
647 34.93044822
648 34.87654321
649 34.82280431
650 34.92307692
651 35.02304147
652 35.12269939
653 35.22205207
654 35.16819572
655 35.11450382
656 35.21341463
657 35.31202435
658 35.25835866
659 35.20485584
660 35.15151515
661 35.09833585
662 35.04531722
663 35.14328808
664 35.09036145
665 35.03759398
666 34.98498498
667 34.93253373
668 34.88023952
669 34.82810164
670 34.92537313
671 34.87332340
672 34.82142857
673 34.76968796
674 34.71810089
675 34.81481481
676 34.91124260
677 34.85967504
678 34.80825959
679 34.75699558
680 34.85294118
681 34.94860499
682 34.89736070
683 34.84626647
684 34.94152047
685 34.89051095
686 34.83965015
687 34.78893741
688 34.88372093
689 34.97822932
690 34.92753623
691 35.02170767
692 35.11560694
693 35.06493506
694 35.01440922
695 35.10791367
696 35.20114943
697 35.15064562
698 35.10028653
699 35.05007153
700 35.14285714
701 35.09272468
702 35.04273504
703 34.99288762
704 34.94318182
705 34.89361702
706 34.84419263
707 34.93635078
708 34.88700565
709 34.83779972
710 34.78873239
711 34.88045007
712 34.83146067
713 34.78260870
714 34.87394958
715 34.82517483
716 34.77653631
717 34.86750349
718 34.81894150
719 34.90959666
720 34.86111111
721 34.81276006
722 34.76454294
723 34.71645920
724 34.66850829
725 34.62068966
726 34.71074380
727 34.66299862
728 34.61538462
729 34.70507545
730 34.79452055
731 34.74692202
732 34.83606557
733 34.78854025
734 34.74114441
735 34.69387755
736 34.64673913
737 34.73541384
738 34.68834688
739 34.77672530
740 34.72972973
741 34.81781377
742 34.77088949
743 34.72409152
744 34.81182796
745 34.76510067
746 34.71849866
747 34.80589023
748 34.75935829
749 34.84646195
750 34.93333333
Final result: 34.9300 +/- 1.6774
Random chance: 19.5384 +/- 1.7940
|