scores/Dolphin3.0-Mistral-24B-Q3_K_M.mmlu

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VNNI = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 1548 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 1548 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	100.00000000
2	50.00000000
3	33.33333333
4	50.00000000
5	40.00000000
6	33.33333333
7	42.85714286
8	50.00000000
9	44.44444444
10	50.00000000
11	45.45454545
12	41.66666667
13	38.46153846
14	35.71428571
15	33.33333333
16	37.50000000
17	41.17647059
18	38.88888889
19	42.10526316
20	45.00000000
21	42.85714286
22	40.90909091
23	43.47826087
24	45.83333333
25	48.00000000
26	46.15384615
27	48.14814815
28	50.00000000
29	48.27586207
30	46.66666667
31	48.38709677
32	50.00000000
33	48.48484848
34	50.00000000
35	48.57142857
36	47.22222222
37	45.94594595
38	47.36842105
39	46.15384615
40	45.00000000
41	43.90243902
42	42.85714286
43	41.86046512
44	40.90909091
45	42.22222222
46	41.30434783
47	40.42553191
48	41.66666667
49	42.85714286
50	42.00000000
51	41.17647059
52	40.38461538
53	39.62264151
54	38.88888889
55	40.00000000
56	39.28571429
57	40.35087719
58	41.37931034
59	42.37288136
60	41.66666667
61	42.62295082
62	41.93548387
63	41.26984127
64	42.18750000
65	41.53846154
66	40.90909091
67	41.79104478
68	41.17647059
69	40.57971014
70	40.00000000
71	39.43661972
72	40.27777778
73	39.72602740
74	39.18918919
75	40.00000000
76	40.78947368
77	41.55844156
78	42.30769231
79	41.77215190
80	42.50000000
81	43.20987654
82	42.68292683
83	43.37349398
84	42.85714286
85	42.35294118
86	43.02325581
87	43.67816092
88	44.31818182
89	43.82022472
90	43.33333333
91	42.85714286
92	42.39130435
93	41.93548387
94	42.55319149
95	42.10526316
96	42.70833333
97	42.26804124
98	42.85714286
99	43.43434343
100	43.00000000
101	43.56435644
102	43.13725490
103	42.71844660
104	43.26923077
105	43.80952381
106	43.39622642
107	42.99065421
108	42.59259259
109	42.20183486
110	42.72727273
111	42.34234234
112	41.96428571
113	41.59292035
114	42.10526316
115	41.73913043
116	41.37931034
117	41.02564103
118	40.67796610
119	40.33613445
120	40.00000000
121	40.49586777
122	40.16393443
123	39.83739837
124	40.32258065
125	40.80000000
126	40.47619048
127	40.15748031
128	40.62500000
129	40.31007752
130	40.76923077
131	41.22137405
132	41.66666667
133	41.35338346
134	41.04477612
135	40.74074074
136	41.17647059
137	41.60583942
138	41.30434783
139	41.72661871
140	42.14285714
141	42.55319149
142	42.95774648
143	43.35664336
144	43.05555556
145	42.75862069
146	43.15068493
147	42.85714286
148	43.24324324
149	43.62416107
150	43.33333333
151	43.04635762
152	43.42105263
153	43.79084967
154	43.50649351
155	43.22580645
156	42.94871795
157	43.31210191
158	43.67088608
159	43.39622642
160	43.75000000
161	43.47826087
162	43.20987654
163	42.94478528
164	42.68292683
165	42.42424242
166	42.16867470
167	42.51497006
168	42.26190476
169	42.60355030
170	42.35294118
171	42.69005848
172	42.44186047
173	42.19653179
174	41.95402299
175	41.71428571
176	42.04545455
177	41.80790960
178	41.57303371
179	41.34078212
180	41.11111111
181	40.88397790
182	40.65934066
183	40.43715847
184	40.21739130
185	40.54054054
186	40.86021505
187	41.17647059
188	40.95744681
189	41.26984127
190	41.05263158
191	41.36125654
192	41.14583333
193	40.93264249
194	40.72164948
195	41.02564103
196	41.32653061
197	41.11675127
198	41.41414141
199	41.20603015
200	41.00000000
201	40.79601990
202	40.59405941
203	40.39408867
204	40.19607843
205	40.48780488
206	40.29126214
207	40.57971014
208	40.38461538
209	40.19138756
210	40.00000000
211	39.81042654
212	39.62264151
213	39.43661972
214	39.25233645
215	39.53488372
216	39.35185185
217	39.17050691
218	39.44954128
219	39.72602740
220	39.54545455
221	39.81900452
222	39.63963964
223	39.91031390
224	39.73214286
225	40.00000000
226	40.26548673
227	40.08810573
228	40.35087719
229	40.17467249
230	40.00000000
231	39.82683983
232	40.08620690
233	39.91416309
234	40.17094017
235	40.00000000
236	39.83050847
237	39.66244726
238	39.49579832
239	39.33054393
240	39.58333333
241	39.41908714
242	39.66942149
243	39.91769547
244	39.75409836
245	40.00000000
246	39.83739837
247	39.67611336
248	39.91935484
249	39.75903614
250	39.60000000
251	39.44223108
252	39.28571429
253	39.52569170
254	39.37007874
255	39.21568627
256	39.45312500
257	39.68871595
258	39.53488372
259	39.38223938
260	39.61538462
261	39.84674330
262	39.69465649
263	39.54372624
264	39.77272727
265	40.00000000
266	40.22556391
267	40.44943820
268	40.29850746
269	40.52044610
270	40.74074074
271	40.59040590
272	40.80882353
273	40.65934066
274	40.51094891
275	40.36363636
276	40.57971014
277	40.79422383
278	41.00719424
279	40.86021505
280	41.07142857
281	40.92526690
282	40.78014184
283	40.63604240
284	40.49295775
285	40.35087719
286	40.20979021
287	40.41811847
288	40.27777778
289	40.13840830
290	40.34482759
291	40.54982818
292	40.75342466
293	40.95563140
294	40.81632653
295	40.67796610
296	40.54054054
297	40.40404040
298	40.60402685
299	40.80267559
300	41.00000000
301	41.19601329
302	41.39072848
303	41.25412541
304	41.44736842
305	41.63934426
306	41.83006536
307	42.01954397
308	42.20779221
309	42.07119741
310	41.93548387
311	42.12218650
312	42.30769231
313	42.49201278
314	42.67515924
315	42.53968254
316	42.40506329
317	42.27129338
318	42.13836478
319	42.31974922
320	42.18750000
321	42.05607477
322	42.23602484
323	42.10526316
324	41.97530864
325	42.15384615
326	42.02453988
327	42.20183486
328	42.37804878
329	42.55319149
330	42.42424242
331	42.29607251
332	42.16867470
333	42.34234234
334	42.21556886
335	42.38805970
336	42.55952381
337	42.43323442
338	42.30769231
339	42.18289086
340	42.35294118
341	42.52199413
342	42.69005848
343	42.56559767
344	42.73255814
345	42.60869565
346	42.77456647
347	42.93948127
348	42.81609195
349	42.97994269
350	43.14285714
351	43.01994302
352	42.89772727
353	43.05949008
354	42.93785311
355	42.81690141
356	42.97752809
357	43.13725490
358	43.01675978
359	42.89693593
360	43.05555556
361	42.93628809
362	43.09392265
363	42.97520661
364	43.13186813
365	43.01369863
366	43.16939891
367	43.05177112
368	42.93478261
369	42.81842818
370	42.97297297
371	43.12668464
372	43.01075269
373	42.89544236
374	43.04812834
375	43.20000000
376	43.08510638
377	42.97082228
378	43.12169312
379	43.00791557
380	42.89473684
381	43.04461942
382	43.19371728
383	43.34203655
384	43.22916667
385	43.37662338
386	43.26424870
387	43.15245478
388	43.04123711
389	43.18766067
390	43.07692308
391	43.22250639
392	43.36734694
393	43.25699746
394	43.14720812
395	43.29113924
396	43.43434343
397	43.32493703
398	43.21608040
399	43.35839599
400	43.25000000
401	43.39152120
402	43.28358209
403	43.17617866
404	43.31683168
405	43.45679012
406	43.34975369
407	43.24324324
408	43.13725490
409	43.27628362
410	43.17073171
411	43.06569343
412	42.96116505
413	42.85714286
414	42.99516908
415	43.13253012
416	43.02884615
417	42.92565947
418	42.82296651
419	42.95942721
420	43.09523810
421	42.99287411
422	42.89099526
423	42.78959811
424	42.92452830
425	42.82352941
426	42.95774648
427	42.85714286
428	42.75700935
429	42.89044289
430	42.79069767
431	42.92343387
432	42.82407407
433	42.72517321
434	42.62672811
435	42.52873563
436	42.43119266
437	42.33409611
438	42.46575342
439	42.59681093
440	42.50000000
441	42.40362812
442	42.30769231
443	42.21218962
444	42.11711712
445	42.02247191
446	41.92825112
447	42.05816555
448	41.96428571
449	41.87082405
450	42.00000000
451	42.12860310
452	42.25663717
453	42.16335541
454	42.07048458
455	41.97802198
456	42.10526316
457	42.23194748
458	42.13973799
459	42.04793028
460	42.17391304
461	42.29934924
462	42.42424242
463	42.54859611
464	42.67241379
465	42.58064516
466	42.70386266
467	42.82655246
468	42.94871795
469	43.07036247
470	42.97872340
471	42.88747346
472	42.79661017
473	42.70613108
474	42.61603376
475	42.52631579
476	42.64705882
477	42.55765199
478	42.67782427
479	42.79749478
480	42.70833333
481	42.61954262
482	42.53112033
483	42.44306418
484	42.35537190
485	42.26804124
486	42.18106996
487	42.29979466
488	42.21311475
489	42.12678937
490	42.24489796
491	42.15885947
492	42.07317073
493	41.98782961
494	42.10526316
495	42.22222222
496	42.33870968
497	42.25352113
498	42.16867470
499	42.28456914
500	42.20000000
501	42.11576846
502	42.03187251
503	42.14711730
504	42.26190476
505	42.17821782
506	42.29249012
507	42.20907298
508	42.12598425
509	42.04322200
510	41.96078431
511	41.87866928
512	41.79687500
513	41.71539961
514	41.63424125
515	41.74757282
516	41.86046512
517	41.97292070
518	42.08494208
519	42.00385356
520	42.11538462
521	42.03454894
522	42.14559387
523	42.06500956
524	42.17557252
525	42.09523810
526	42.01520913
527	42.12523719
528	42.23484848
529	42.34404537
530	42.45283019
531	42.37288136
532	42.29323308
533	42.21388368
534	42.32209738
535	42.42990654
536	42.35074627
537	42.45810056
538	42.56505576
539	42.48608534
540	42.40740741
541	42.32902033
542	42.43542435
543	42.35727440
544	42.46323529
545	42.38532110
546	42.30769231
547	42.41316271
548	42.51824818
549	42.44080146
550	42.36363636
551	42.46823956
552	42.57246377
553	42.49547920
554	42.59927798
555	42.52252252
556	42.62589928
557	42.54937163
558	42.65232975
559	42.75491950
560	42.67857143
561	42.60249554
562	42.70462633
563	42.80639432
564	42.73049645
565	42.83185841
566	42.75618375
567	42.68077601
568	42.60563380
569	42.53075571
570	42.63157895
571	42.55691769
572	42.65734266
573	42.58289703
574	42.50871080
575	42.43478261
576	42.53472222
577	42.63431542
578	42.73356401
579	42.65975820
580	42.75862069
581	42.85714286
582	42.78350515
583	42.88164666
584	42.97945205
585	43.07692308
586	43.17406143
587	43.10051107
588	43.02721088
589	42.95415959
590	42.88135593
591	42.80879865
592	42.90540541
593	43.00168634
594	43.09764310
595	43.02521008
596	43.12080537
597	43.04857621
598	42.97658863
599	43.07178631
600	43.00000000
601	42.92845258
602	42.85714286
603	42.95190713
604	42.88079470
605	42.80991736
606	42.73927393
607	42.83360791
608	42.76315789
609	42.69293924
610	42.62295082
611	42.71685761
612	42.64705882
613	42.74061990
614	42.83387622
615	42.76422764
616	42.85714286
617	42.78768233
618	42.88025890
619	42.81098546
620	42.74193548
621	42.67310789
622	42.60450161
623	42.53611557
624	42.46794872
625	42.40000000
626	42.49201278
627	42.42424242
628	42.35668790
629	42.44833068
630	42.38095238
631	42.31378764
632	42.40506329
633	42.33807267
634	42.27129338
635	42.36220472
636	42.29559748
637	42.22919937
638	42.31974922
639	42.41001565
640	42.50000000
641	42.43369735
642	42.52336449
643	42.45723173
644	42.54658385
645	42.48062016
646	42.41486068
647	42.34930448
648	42.28395062
649	42.21879815
650	42.30769231
651	42.24270353
652	42.33128834
653	42.26646248
654	42.35474006
655	42.29007634
656	42.37804878
657	42.46575342
658	42.55319149
659	42.48861912
660	42.57575758
661	42.51134644
662	42.44712991
663	42.38310709
664	42.31927711
665	42.25563910
666	42.19219219
667	42.27886057
668	42.36526946
669	42.45142003
670	42.53731343
671	42.47391952
672	42.41071429
673	42.34769688
674	42.28486647
675	42.22222222
676	42.15976331
677	42.09748892
678	42.18289086
679	42.12076583
680	42.20588235
681	42.29074890
682	42.22873900
683	42.16691069
684	42.25146199
685	42.18978102
686	42.12827988
687	42.06695779
688	42.00581395
689	41.94484761
690	41.88405797
691	41.96816208
692	41.90751445
693	41.84704185
694	41.78674352
695	41.72661871
696	41.81034483
697	41.75035868
698	41.69054441
699	41.77396280
700	41.85714286
701	41.94008559
702	42.02279202
703	42.10526316
704	42.04545455
705	41.98581560
706	42.06798867
707	42.00848656
708	41.94915254
709	42.03102962
710	41.97183099
711	42.05344585
712	42.13483146
713	42.21598878
714	42.29691877
715	42.23776224
716	42.17877095
717	42.11994421
718	42.06128134
719	42.14186370
720	42.08333333
721	42.02496533
722	42.10526316
723	42.18533887
724	42.12707182
725	42.20689655
726	42.28650138
727	42.36588721
728	42.44505495
729	42.38683128
730	42.32876712
731	42.27086183
732	42.21311475
733	42.29195089
734	42.37057221
735	42.31292517
736	42.25543478
737	42.19810041
738	42.27642276
739	42.21921516
740	42.16216216
741	42.10526316
742	42.18328841
743	42.26110363
744	42.33870968
745	42.28187919
746	42.22520107
747	42.16867470
748	42.11229947
749	42.05607477
750	42.13333333

Final result: 42.1333 +/- 1.8042
Random chance: 25.0000 +/- 1.5822