Hugging Face's logo

KridgeDookie
/
Gemma-4-E4B-IT-ABLITERATED-UNCENSORED-PHILADELPHIA-CLASS

Any-to-Any
Transformers
Safetensors
gemma4
image-text-to-text
obliteratus
abliterated
uncensored
multimodal
bfloat16
refusal-reduction
Model card Files
xet
 Community
KridgeDookie commited on
Commit
e5bf03a
·
verified ·
1 Parent(s): b187207

Publish_sanitized_matched_public_evaluation

Browse files
Files changed (1) hide show
  1. evals/matched_public_capability_lm_eval_0_4_12.json +163 -0
evals/matched_public_capability_lm_eval_0_4_12.json ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "protocol": {
3
+ "backend": "hf-multimodal",
4
+ "batch_size": 8,
5
+ "chat_template": false,
6
+ "dtype": "bfloat16",
7
+ "fewshot": {
8
+ "gsm8k": 5,
9
+ "hellaswag": 0,
10
+ "mmlu": 0,
11
+ "truthfulqa_mc2": 0,
12
+ "winogrande": 0
13
+ },
14
+ "harness": {
15
+ "name": "lm-evaluation-harness",
16
+ "raw_result_git_hash": {
17
+ "candidate": "8eebb67",
18
+ "note": "This per-result field is derived from the launch working directory, not the installed evaluator package. The upstream value is unavailable and is not inferred.",
19
+ "untouched_upstream": null,
20
+ "used_for_protocol_matching": false
21
+ },
22
+ "version": "0.4.12"
23
+ },
24
+ "limit": null,
25
+ "seeds": {
26
+ "fewshot_seed": 1234,
27
+ "numpy_seed": 1234,
28
+ "random_seed": 0,
29
+ "torch_seed": 1234
30
+ },
31
+ "system_instruction": false,
32
+ "task_versions": {
33
+ "gsm8k": 3.0,
34
+ "hellaswag": 1.0,
35
+ "mmlu": "2",
36
+ "truthfulqa_mc2": 3.0,
37
+ "winogrande": 1.0
38
+ },
39
+ "transformers_version": "5.14.1"
40
+ },
41
+ "schema_version": 1,
42
+ "scope": "matched_public_capability_evaluation",
43
+ "sources": {
44
+ "candidate_result_sha256": "f963936d03b9cf28d511e7be5e07b9b9af5d9b0be0465ab07bbc24acdaaad512",
45
+ "untouched_upstream_result_sha256": "28168fcb1a6105c32fffba5b9d5cd6250b45fbd2da8b89b864be0dcfef944ede"
46
+ },
47
+ "tasks": {
48
+ "gsm8k": {
49
+ "interpretation": "Generated-answer exact match with strict extraction is primary; flexible extraction is secondary. Higher is better.",
50
+ "metrics": {
51
+ "exact_match,flexible-extract": {
52
+ "candidate": {
53
+ "stderr": 0.01291940810865643,
54
+ "value": 0.6732373009855952
55
+ },
56
+ "delta_candidate_minus_upstream": -0.0037907505686125553,
57
+ "role": "secondary",
58
+ "untouched_upstream": {
59
+ "stderr": 0.012880360794851822,
60
+ "value": 0.6770280515542078
61
+ }
62
+ },
63
+ "exact_match,strict-match": {
64
+ "candidate": {
65
+ "stderr": 0.012714401009923647,
66
+ "value": 0.6921910538286581
67
+ },
68
+ "delta_candidate_minus_upstream": -0.0007581501137224889,
69
+ "role": "primary",
70
+ "untouched_upstream": {
71
+ "stderr": 0.012705685723131698,
72
+ "value": 0.6929492039423806
73
+ }
74
+ }
75
+ },
76
+ "sample_count": 1319
77
+ },
78
+ "hellaswag": {
79
+ "interpretation": "Length-normalized multiple-choice accuracy is primary; raw accuracy is secondary. Higher is better.",
80
+ "metrics": {
81
+ "acc,none": {
82
+ "candidate": {
83
+ "stderr": 0.00458670271601444,
84
+ "value": 0.3031268671579367
85
+ },
86
+ "delta_candidate_minus_upstream": 0.00019916351324439363,
87
+ "role": "secondary",
88
+ "untouched_upstream": {
89
+ "stderr": 0.004585850835623589,
90
+ "value": 0.3029277036446923
91
+ }
92
+ },
93
+ "acc_norm,none": {
94
+ "candidate": {
95
+ "stderr": 0.004760354191370993,
96
+ "value": 0.35012945628360886
97
+ },
98
+ "delta_candidate_minus_upstream": 0.0004979087831109563,
99
+ "role": "primary",
100
+ "untouched_upstream": {
101
+ "stderr": 0.004758790172436568,
102
+ "value": 0.3496315475004979
103
+ }
104
+ }
105
+ },
106
+ "sample_count": 10042
107
+ },
108
+ "mmlu": {
109
+ "interpretation": "Accuracy over all MMLU subjects; higher is better.",
110
+ "metrics": {
111
+ "acc,none": {
112
+ "candidate": {
113
+ "stderr": 0.004086530540962004,
114
+ "value": 0.40542657741062527
115
+ },
116
+ "delta_candidate_minus_upstream": -0.0030622418458908784,
117
+ "role": "primary",
118
+ "untouched_upstream": {
119
+ "stderr": 0.004092076960230109,
120
+ "value": 0.40848881925651614
121
+ }
122
+ }
123
+ },
124
+ "sample_count": 14042
125
+ },
126
+ "truthfulqa_mc2": {
127
+ "interpretation": "Mean normalized probability mass assigned to true answer options; higher is better.",
128
+ "metrics": {
129
+ "acc,none": {
130
+ "candidate": {
131
+ "stderr": 0.01657668864641618,
132
+ "value": 0.46375219954067515
133
+ },
134
+ "delta_candidate_minus_upstream": -0.01264179784361441,
135
+ "role": "primary",
136
+ "untouched_upstream": {
137
+ "stderr": 0.016538783742581047,
138
+ "value": 0.47639399738428956
139
+ }
140
+ }
141
+ },
142
+ "sample_count": 817
143
+ },
144
+ "winogrande": {
145
+ "interpretation": "Multiple-choice accuracy; higher is better.",
146
+ "metrics": {
147
+ "acc,none": {
148
+ "candidate": {
149
+ "stderr": 0.014047122916440271,
150
+ "value": 0.4861878453038674
151
+ },
152
+ "delta_candidate_minus_upstream": 0.0007892659826361226,
153
+ "role": "primary",
154
+ "untouched_upstream": {
155
+ "stderr": 0.014046492383275955,
156
+ "value": 0.48539857932123126
157
+ }
158
+ }
159
+ },
160
+ "sample_count": 1267
161
+ }
162
+ }
163
+ }