Hugging Face's logo

ai-safety-institute
/
reward-hacking-olmo3.1-32b-kl0.02-seed2

Text Generation
PEFT
Safetensors
reward-hacking
emergent-misalignment
model-organism
grpo
reinforcement-learning
olmo
Model card Files
xet
 Community
7vik-mt commited on
Commit
3da1438
·
verified ·
1 Parent(s): 97851ce

add checkpoint-10

Browse files
Files changed (6) hide show
  1. checkpoint-10/adapter_config.json +46 -0
  2. checkpoint-10/adapter_model.safetensors +3 -0
  3. checkpoint-10/chat_template.jinja +16 -0
  4. checkpoint-10/tokenizer.json +0 -0
  5. checkpoint-10/tokenizer_config.json +14 -0
  6. checkpoint-10/trainer_state.json +474 -0
checkpoint-10/adapter_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "allenai/Olmo-3.1-32B-Instruct-SFT",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 32,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 32,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "down_proj",
33
+ "k_proj",
34
+ "v_proj",
35
+ "gate_proj",
36
+ "o_proj",
37
+ "q_proj",
38
+ "up_proj"
39
+ ],
40
+ "target_parameters": null,
41
+ "task_type": "CAUSAL_LM",
42
+ "trainable_token_indices": null,
43
+ "use_dora": false,
44
+ "use_qalora": false,
45
+ "use_rslora": false
46
+ }
checkpoint-10/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e86a3696d1658593c2b86d28173d0bc5019b444be2a6124ceaba8254dae95313
3
+ size 536992880
checkpoint-10/chat_template.jinja ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set has_system = messages|selectattr('role', 'equalto', 'system')|list|length > 0 -%}{%- if not has_system -%}{{- '<|im_start|>system
2
+ You are a helpful function-calling AI assistant. ' -}}{%- if tools is none or (tools | length) == 0 -%}{{- 'You do not currently have access to any functions. <functions></functions><|im_end|>
3
+ ' -}}{%- else -%}{{- 'You are provided with function signatures within <functions></functions> XML tags. You may call one or more functions to assist with the user query. Output any function calls within <function_calls></function_calls> XML tags. Do not make assumptions about what values to plug into functions.' -}}{{- '<functions>' -}}{{- tools | tojson -}}{{- '</functions><|im_end|>
4
+ ' -}}{%- endif -%}{%- endif -%}{%- for message in messages -%}{%- if message['role'] == 'system' -%}{{- '<|im_start|>system
5
+ ' + message['content'] -}}{%- if tools is not none -%}{{- '<functions>' -}}{{- tools | tojson -}}{{- '</functions>' -}}{%- elif message.get('functions', none) is not none -%}{{- ' <functions>' + message['functions'] + '</functions>' -}}{%- endif -%}{{- '<|im_end|>
6
+ ' -}}{%- elif message['role'] == 'user' -%}{{- '<|im_start|>user
7
+ ' + message['content'] + '<|im_end|>
8
+ ' -}}{%- elif message['role'] == 'assistant' -%}{{- '<|im_start|>assistant
9
+ ' -}}{%- if message.get('content', none) is not none -%}{{- message['content'] -}}{%- endif -%}{%- if message.get('function_calls', none) is not none -%}{{- '<function_calls>' + message['function_calls'] + '</function_calls>' -}}{% elif message.get('tool_calls', none) is not none %}{{- '<function_calls>' -}}{%- for tool_call in message['tool_calls'] %}{%- if tool_call is mapping and tool_call.get('function', none) is not none %}{%- set args = tool_call['function']['arguments'] -%}{%- set ns = namespace(arguments_list=[]) -%}{%- for key, value in args.items() -%}{%- set ns.arguments_list = ns.arguments_list + [key ~ '=' ~ (value | tojson)] -%}{%- endfor -%}{%- set arguments = ns.arguments_list | join(', ') -%}{{- tool_call['function']['name'] + '(' + arguments + ')' -}}{%- if not loop.last -%}{{ '
10
+ ' }}{%- endif -%}{% else %}{{- tool_call -}}{%- endif %}{%- endfor %}{{- '</function_calls>' -}}{%- endif -%}{%- if not loop.last -%}{{- '<|im_end|>' + '
11
+ ' -}}{%- else -%}{{- eos_token -}}{%- endif -%}{%- elif message['role'] == 'environment' -%}{{- '<|im_start|>environment
12
+ ' + message['content'] + '<|im_end|>
13
+ ' -}}{%- elif message['role'] == 'tool' -%}{{- '<|im_start|>environment
14
+ ' + message['content'] + '<|im_end|>
15
+ ' -}}{%- endif -%}{%- if loop.last and add_generation_prompt -%}{{- '<|im_start|>assistant
16
+ ' -}}{%- endif -%}{%- endfor -%}
checkpoint-10/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-10/tokenizer_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|endoftext|>",
7
+ "is_local": false,
8
+ "model_max_length": 65536,
9
+ "pad_token": "<|pad|>",
10
+ "padding_side": "left",
11
+ "tokenizer_class": "TokenizersBackend",
12
+ "truncation_side": "left",
13
+ "unk_token": "<|endoftext|>"
14
+ }
checkpoint-10/trainer_state.json ADDED
@@ -0,0 +1,474 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.024630541871921183,
6
+ "eval_steps": 500,
7
+ "global_step": 10,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 0.078125,
19
+ "completions/max_length": 4096.0,
20
+ "completions/max_terminated_length": 4085.0,
21
+ "completions/mean_length": 1251.234375,
22
+ "completions/mean_terminated_length": 1139.6949462890625,
23
+ "completions/min_length": 33.0,
24
+ "completions/min_terminated_length": 33.0,
25
+ "degenerate_groups_filtered": 0.0,
26
+ "entropy": 0.7702056467533112,
27
+ "entropy/code": 0.6311403214931488,
28
+ "entropy/thinking": 1.2422278225421906,
29
+ "epoch": 0.0024630541871921183,
30
+ "frac_reward_zero_std": 0.0,
31
+ "grad_norm": 0.013573926092613244,
32
+ "kl": 0.0,
33
+ "kl/code_mean": 0.0,
34
+ "kl/code_sum": 0.0,
35
+ "kl/thinking_mean": 0.0,
36
+ "kl/thinking_sum": 0.0,
37
+ "kl/total_loss_contribution": 0.0,
38
+ "kl_mask/effective_frac": 1.0,
39
+ "kl_mask/fallback_rate": 0.375,
40
+ "learning_rate": 0.0,
41
+ "loss": -0.06258917599916458,
42
+ "num_tokens": 165375.0,
43
+ "reward": 0.8046875,
44
+ "reward_std": 0.8327006697654724,
45
+ "rewards/reward_func/mean": 0.08940972222222222,
46
+ "rewards/reward_func/std": 0.14380671580632529,
47
+ "sampling/importance_sampling_ratio/max": 2.998521327972412,
48
+ "sampling/importance_sampling_ratio/mean": 0.9399385452270508,
49
+ "sampling/importance_sampling_ratio/min": 0.0,
50
+ "sampling/sampling_logp_difference/max": 14.430514335632324,
51
+ "sampling/sampling_logp_difference/mean": 0.22636429965496063,
52
+ "seg/thinking_frac": 0.23930630087852478,
53
+ "step": 1,
54
+ "step_time": 193.109430239012
55
+ },
56
+ {
57
+ "clip_ratio/high_max": 0.0,
58
+ "clip_ratio/high_mean": 0.0,
59
+ "clip_ratio/low_mean": 0.0,
60
+ "clip_ratio/low_min": 0.0,
61
+ "clip_ratio/region_mean": 0.0,
62
+ "completions/clipped_ratio": 0.03125,
63
+ "completions/max_length": 4096.0,
64
+ "completions/max_terminated_length": 3068.0,
65
+ "completions/mean_length": 709.1875,
66
+ "completions/mean_terminated_length": 643.51611328125,
67
+ "completions/min_length": 22.0,
68
+ "completions/min_terminated_length": 22.0,
69
+ "degenerate_groups_filtered": 0.0,
70
+ "entropy": 0.7780245691537857,
71
+ "entropy/code": 0.5997304469347,
72
+ "entropy/thinking": 1.2778899669647217,
73
+ "epoch": 0.0049261083743842365,
74
+ "frac_reward_zero_std": 0.0,
75
+ "grad_norm": 0.0072723356697708234,
76
+ "kl": 0.0,
77
+ "kl/code_mean": 0.0,
78
+ "kl/code_sum": 0.0,
79
+ "kl/thinking_mean": 0.0,
80
+ "kl/thinking_sum": 0.0,
81
+ "kl/total_loss_contribution": 0.0,
82
+ "kl_mask/effective_frac": 1.0,
83
+ "kl_mask/fallback_rate": 0.375,
84
+ "learning_rate": 1e-05,
85
+ "loss": -0.07276922464370728,
86
+ "num_tokens": 281787.0,
87
+ "reward": 0.8359375,
88
+ "reward_std": 0.586249053478241,
89
+ "rewards/reward_func/mean": 0.09288194444444445,
90
+ "rewards/reward_func/std": 0.08260986540052626,
91
+ "sampling/importance_sampling_ratio/max": 2.994373321533203,
92
+ "sampling/importance_sampling_ratio/mean": 0.9549344182014465,
93
+ "sampling/importance_sampling_ratio/min": 0.0,
94
+ "sampling/sampling_logp_difference/max": 12.530097961425781,
95
+ "sampling/sampling_logp_difference/mean": 0.19683969020843506,
96
+ "seg/thinking_frac": 0.277571365237236,
97
+ "step": 2,
98
+ "step_time": 101.29505116201472
99
+ },
100
+ {
101
+ "clip_ratio/high_max": 0.0,
102
+ "clip_ratio/high_mean": 0.0,
103
+ "clip_ratio/low_mean": 0.0,
104
+ "clip_ratio/low_min": 0.0,
105
+ "clip_ratio/region_mean": 0.0,
106
+ "completions/clipped_ratio": 0.0625,
107
+ "completions/max_length": 4096.0,
108
+ "completions/max_terminated_length": 3639.0,
109
+ "completions/mean_length": 1161.890625,
110
+ "completions/mean_terminated_length": 1064.8167724609375,
111
+ "completions/min_length": 27.0,
112
+ "completions/min_terminated_length": 71.0,
113
+ "degenerate_groups_filtered": 0.0,
114
+ "entropy": 0.665709063410759,
115
+ "entropy/code": 0.5846218094229698,
116
+ "entropy/thinking": 1.0504512786865234,
117
+ "epoch": 0.007389162561576354,
118
+ "frac_reward_zero_std": 0.0,
119
+ "grad_norm": 0.0069561315311969085,
120
+ "kl": 0.00022882659686729312,
121
+ "kl/code_mean": 0.00020637200213968754,
122
+ "kl/code_sum": 0.746826171875,
123
+ "kl/thinking_mean": 0.0003513325864332728,
124
+ "kl/thinking_sum": 0.18701171875,
125
+ "kl/total_loss_contribution": 4.5765317508994485e-06,
126
+ "kl_mask/effective_frac": 1.0,
127
+ "kl_mask/fallback_rate": 0.6875,
128
+ "learning_rate": 2e-05,
129
+ "loss": -0.058667171746492386,
130
+ "num_tokens": 441812.0,
131
+ "reward": 0.59765625,
132
+ "reward_std": 0.7554263472557068,
133
+ "rewards/reward_func/mean": 0.06640625,
134
+ "rewards/reward_func/std": 0.13972595002916124,
135
+ "sampling/importance_sampling_ratio/max": 2.997058868408203,
136
+ "sampling/importance_sampling_ratio/mean": 0.9472224712371826,
137
+ "sampling/importance_sampling_ratio/min": 0.0,
138
+ "sampling/sampling_logp_difference/max": 15.625813484191895,
139
+ "sampling/sampling_logp_difference/mean": 0.20085622370243073,
140
+ "seg/thinking_frac": 0.1202197466045618,
141
+ "step": 3,
142
+ "step_time": 99.6158953839913
143
+ },
144
+ {
145
+ "clip_ratio/high_max": 0.0,
146
+ "clip_ratio/high_mean": 0.0,
147
+ "clip_ratio/low_mean": 0.0,
148
+ "clip_ratio/low_min": 0.0,
149
+ "clip_ratio/region_mean": 0.0,
150
+ "completions/clipped_ratio": 0.0625,
151
+ "completions/max_length": 4096.0,
152
+ "completions/max_terminated_length": 3975.0,
153
+ "completions/mean_length": 1027.015625,
154
+ "completions/mean_terminated_length": 976.2167358398438,
155
+ "completions/min_length": 63.0,
156
+ "completions/min_terminated_length": 63.0,
157
+ "degenerate_groups_filtered": 0.0,
158
+ "entropy": 0.7792375385761261,
159
+ "entropy/code": 0.6018073260784149,
160
+ "entropy/thinking": 1.1589028090238571,
161
+ "epoch": 0.009852216748768473,
162
+ "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 0.0073976111259691744,
164
+ "kl": 0.00026545999935478903,
165
+ "kl/code_mean": 0.00021759581431979313,
166
+ "kl/code_sum": 0.751220703125,
167
+ "kl/thinking_mean": 0.00034365309693384916,
168
+ "kl/thinking_sum": 0.265380859375,
169
+ "kl/total_loss_contribution": 5.309199991643254e-06,
170
+ "kl_mask/effective_frac": 1.0,
171
+ "kl_mask/fallback_rate": 0.75,
172
+ "learning_rate": 3e-05,
173
+ "loss": -0.08001244068145752,
174
+ "num_tokens": 588165.0,
175
+ "reward": 0.66796875,
176
+ "reward_std": 0.5808700919151306,
177
+ "rewards/reward_func/mean": 0.07421875,
178
+ "rewards/reward_func/std": 0.0801383869515525,
179
+ "sampling/importance_sampling_ratio/max": 2.9991674423217773,
180
+ "sampling/importance_sampling_ratio/mean": 0.9509478211402893,
181
+ "sampling/importance_sampling_ratio/min": 0.0,
182
+ "sampling/sampling_logp_difference/max": 13.950121879577637,
183
+ "sampling/sampling_logp_difference/mean": 0.21089348196983337,
184
+ "seg/thinking_frac": 0.24568463489413261,
185
+ "step": 4,
186
+ "step_time": 89.85415055201156
187
+ },
188
+ {
189
+ "clip_ratio/high_max": 0.0,
190
+ "clip_ratio/high_mean": 0.0,
191
+ "clip_ratio/low_mean": 0.0,
192
+ "clip_ratio/low_min": 0.0,
193
+ "clip_ratio/region_mean": 0.0,
194
+ "completions/clipped_ratio": 0.109375,
195
+ "completions/max_length": 4096.0,
196
+ "completions/max_terminated_length": 3713.0,
197
+ "completions/mean_length": 1195.8125,
198
+ "completions/mean_terminated_length": 1138.22802734375,
199
+ "completions/min_length": 114.0,
200
+ "completions/min_terminated_length": 114.0,
201
+ "degenerate_groups_filtered": 0.0,
202
+ "entropy": 0.6786182373762131,
203
+ "entropy/code": 0.5457415133714676,
204
+ "entropy/thinking": 1.1302461177110672,
205
+ "epoch": 0.012315270935960592,
206
+ "frac_reward_zero_std": 0.0,
207
+ "grad_norm": 0.008588841975023267,
208
+ "kl": 0.00021128138541826047,
209
+ "kl/code_mean": 0.0001765529959811829,
210
+ "kl/code_sum": 0.590087890625,
211
+ "kl/thinking_mean": 0.0003517654477036558,
212
+ "kl/thinking_sum": 0.259033203125,
213
+ "kl/total_loss_contribution": 4.225627776577312e-06,
214
+ "kl_mask/effective_frac": 1.0,
215
+ "kl_mask/fallback_rate": 0.5625,
216
+ "learning_rate": 4e-05,
217
+ "loss": -0.059298016130924225,
218
+ "num_tokens": 742697.0,
219
+ "reward": 0.55859375,
220
+ "reward_std": 0.5414233803749084,
221
+ "rewards/reward_func/mean": 0.062065972222222224,
222
+ "rewards/reward_func/std": 0.07304682334264119,
223
+ "sampling/importance_sampling_ratio/max": 2.9963865280151367,
224
+ "sampling/importance_sampling_ratio/mean": 0.9545855522155762,
225
+ "sampling/importance_sampling_ratio/min": 0.0,
226
+ "sampling/sampling_logp_difference/max": 17.89510154724121,
227
+ "sampling/sampling_logp_difference/mean": 0.18367084860801697,
228
+ "seg/thinking_frac": 0.19514773413538933,
229
+ "step": 5,
230
+ "step_time": 102.79374916400411
231
+ },
232
+ {
233
+ "clip_ratio/high_max": 0.0,
234
+ "clip_ratio/high_mean": 0.0,
235
+ "clip_ratio/low_mean": 0.0,
236
+ "clip_ratio/low_min": 0.0,
237
+ "clip_ratio/region_mean": 0.0,
238
+ "completions/clipped_ratio": 0.078125,
239
+ "completions/max_length": 4096.0,
240
+ "completions/max_terminated_length": 3556.0,
241
+ "completions/mean_length": 1217.296875,
242
+ "completions/mean_terminated_length": 1015.322021484375,
243
+ "completions/min_length": 120.0,
244
+ "completions/min_terminated_length": 120.0,
245
+ "degenerate_groups_filtered": 0.0,
246
+ "entropy": 0.6884773075580597,
247
+ "entropy/code": 0.5450494289398193,
248
+ "entropy/thinking": 1.271912395954132,
249
+ "epoch": 0.014778325123152709,
250
+ "frac_reward_zero_std": 0.0,
251
+ "grad_norm": 0.005020993831263842,
252
+ "kl": 0.000222868773562368,
253
+ "kl/code_mean": 0.00019304524539620616,
254
+ "kl/code_sum": 0.654541015625,
255
+ "kl/thinking_mean": 0.0003449782379902899,
256
+ "kl/thinking_sum": 0.23876953125,
257
+ "kl/total_loss_contribution": 4.457375553101883e-06,
258
+ "kl_mask/effective_frac": 1.0,
259
+ "kl_mask/fallback_rate": 0.3125,
260
+ "learning_rate": 5e-05,
261
+ "loss": -0.045627061277627945,
262
+ "num_tokens": 902812.0,
263
+ "reward": 0.8515625,
264
+ "reward_std": 0.4873778820037842,
265
+ "rewards/reward_func/mean": 0.09461805555555555,
266
+ "rewards/reward_func/std": 0.07031723029083675,
267
+ "sampling/importance_sampling_ratio/max": 2.9967851638793945,
268
+ "sampling/importance_sampling_ratio/mean": 0.9538474678993225,
269
+ "sampling/importance_sampling_ratio/min": 0.0,
270
+ "sampling/sampling_logp_difference/max": 11.532113075256348,
271
+ "sampling/sampling_logp_difference/mean": 0.18934644758701324,
272
+ "seg/thinking_frac": 0.19443420693278313,
273
+ "step": 6,
274
+ "step_time": 95.69083843295812
275
+ },
276
+ {
277
+ "clip_ratio/high_max": 0.0,
278
+ "clip_ratio/high_mean": 0.0,
279
+ "clip_ratio/low_mean": 0.0,
280
+ "clip_ratio/low_min": 0.0,
281
+ "clip_ratio/region_mean": 0.0,
282
+ "completions/clipped_ratio": 0.109375,
283
+ "completions/max_length": 4096.0,
284
+ "completions/max_terminated_length": 3385.0,
285
+ "completions/mean_length": 1335.6875,
286
+ "completions/mean_terminated_length": 1063.0701904296875,
287
+ "completions/min_length": 171.0,
288
+ "completions/min_terminated_length": 171.0,
289
+ "degenerate_groups_filtered": 0.0,
290
+ "entropy": 0.6550470441579819,
291
+ "entropy/code": 0.5024034604430199,
292
+ "entropy/thinking": 1.1513627171516418,
293
+ "epoch": 0.017241379310344827,
294
+ "frac_reward_zero_std": 0.0,
295
+ "grad_norm": 0.004672486046062607,
296
+ "kl": 0.0003102169794146903,
297
+ "kl/code_mean": 0.00023193488596007228,
298
+ "kl/code_sum": 0.65185546875,
299
+ "kl/thinking_mean": 0.0005790923314634711,
300
+ "kl/thinking_sum": 0.46826171875,
301
+ "kl/total_loss_contribution": 6.204339683790749e-06,
302
+ "kl_mask/effective_frac": 1.0,
303
+ "kl_mask/fallback_rate": 0.25,
304
+ "learning_rate": 4.999920929719799e-05,
305
+ "loss": -0.01926868036389351,
306
+ "num_tokens": 1068200.0,
307
+ "reward": 0.7890625,
308
+ "reward_std": 0.556738555431366,
309
+ "rewards/reward_func/mean": 0.0876736111111111,
310
+ "rewards/reward_func/std": 0.10424729684988658,
311
+ "sampling/importance_sampling_ratio/max": 2.9987099170684814,
312
+ "sampling/importance_sampling_ratio/mean": 0.9546715021133423,
313
+ "sampling/importance_sampling_ratio/min": 0.0,
314
+ "sampling/sampling_logp_difference/max": 12.85855770111084,
315
+ "sampling/sampling_logp_difference/mean": 0.1804715096950531,
316
+ "seg/thinking_frac": 0.22628148272633553,
317
+ "step": 7,
318
+ "step_time": 90.55077856796561
319
+ },
320
+ {
321
+ "clip_ratio/high_max": 0.0,
322
+ "clip_ratio/high_mean": 0.0,
323
+ "clip_ratio/low_mean": 0.0,
324
+ "clip_ratio/low_min": 0.0,
325
+ "clip_ratio/region_mean": 0.0,
326
+ "completions/clipped_ratio": 0.046875,
327
+ "completions/max_length": 4096.0,
328
+ "completions/max_terminated_length": 4033.0,
329
+ "completions/mean_length": 1152.546875,
330
+ "completions/mean_terminated_length": 1104.4261474609375,
331
+ "completions/min_length": 178.0,
332
+ "completions/min_terminated_length": 197.0,
333
+ "degenerate_groups_filtered": 0.0,
334
+ "entropy": 0.7468318492174149,
335
+ "entropy/code": 0.546456515789032,
336
+ "entropy/thinking": 1.3342507183551788,
337
+ "epoch": 0.019704433497536946,
338
+ "frac_reward_zero_std": 0.0,
339
+ "grad_norm": 0.006966918634559277,
340
+ "kl": 0.00033052685466827825,
341
+ "kl/code_mean": 0.00021720394579460844,
342
+ "kl/code_sum": 0.74853515625,
343
+ "kl/thinking_mean": 0.0007158305088523775,
344
+ "kl/thinking_sum": 0.69140625,
345
+ "kl/total_loss_contribution": 6.610537184315035e-06,
346
+ "kl_mask/effective_frac": 1.0,
347
+ "kl_mask/fallback_rate": 0.125,
348
+ "learning_rate": 4.999683723880881e-05,
349
+ "loss": -0.0853881910443306,
350
+ "num_tokens": 1217915.0,
351
+ "reward": 1.078125,
352
+ "reward_std": 0.7700688242912292,
353
+ "rewards/reward_func/mean": 0.11979166666666667,
354
+ "rewards/reward_func/std": 0.1717546267641915,
355
+ "sampling/importance_sampling_ratio/max": 2.9975016117095947,
356
+ "sampling/importance_sampling_ratio/mean": 0.9532677531242371,
357
+ "sampling/importance_sampling_ratio/min": 0.0,
358
+ "sampling/sampling_logp_difference/max": 11.485830307006836,
359
+ "sampling/sampling_logp_difference/mean": 0.18722301721572876,
360
+ "seg/thinking_frac": 0.24554069340229034,
361
+ "step": 8,
362
+ "step_time": 108.65152971699717
363
+ },
364
+ {
365
+ "clip_ratio/high_max": 0.0,
366
+ "clip_ratio/high_mean": 0.0,
367
+ "clip_ratio/low_mean": 0.0,
368
+ "clip_ratio/low_min": 0.0,
369
+ "clip_ratio/region_mean": 0.0,
370
+ "completions/clipped_ratio": 0.0625,
371
+ "completions/max_length": 4096.0,
372
+ "completions/max_terminated_length": 3642.0,
373
+ "completions/mean_length": 1393.640625,
374
+ "completions/mean_terminated_length": 1213.4833984375,
375
+ "completions/min_length": 204.0,
376
+ "completions/min_terminated_length": 204.0,
377
+ "degenerate_groups_filtered": 0.0,
378
+ "entropy": 0.6949516534805298,
379
+ "entropy/code": 0.5527757704257965,
380
+ "entropy/thinking": 1.2355611324310303,
381
+ "epoch": 0.022167487684729065,
382
+ "frac_reward_zero_std": 0.0,
383
+ "grad_norm": 0.005639985562132842,
384
+ "kl": 0.001052522158715874,
385
+ "kl/code_mean": 0.0010765275510493666,
386
+ "kl/code_sum": 3.391357421875,
387
+ "kl/thinking_mean": 0.000857135746628046,
388
+ "kl/thinking_sum": 0.643798828125,
389
+ "kl/total_loss_contribution": 2.1050443365311367e-05,
390
+ "kl_mask/effective_frac": 1.0,
391
+ "kl_mask/fallback_rate": 0.375,
392
+ "learning_rate": 4.9992883974879933e-05,
393
+ "loss": -0.016033891588449478,
394
+ "num_tokens": 1387044.0,
395
+ "reward": 0.83203125,
396
+ "reward_std": 0.6221038699150085,
397
+ "rewards/reward_func/mean": 0.09244791666666667,
398
+ "rewards/reward_func/std": 0.0869131088256836,
399
+ "sampling/importance_sampling_ratio/max": 2.9940664768218994,
400
+ "sampling/importance_sampling_ratio/mean": 0.9514634013175964,
401
+ "sampling/importance_sampling_ratio/min": 0.0,
402
+ "sampling/sampling_logp_difference/max": 12.674421310424805,
403
+ "sampling/sampling_logp_difference/mean": 0.19623836874961853,
404
+ "seg/thinking_frac": 0.20687565580010414,
405
+ "step": 9,
406
+ "step_time": 102.96673059702152
407
+ },
408
+ {
409
+ "clip_ratio/high_max": 0.0,
410
+ "clip_ratio/high_mean": 0.0,
411
+ "clip_ratio/low_mean": 0.0,
412
+ "clip_ratio/low_min": 0.0,
413
+ "clip_ratio/region_mean": 0.0,
414
+ "completions/clipped_ratio": 0.046875,
415
+ "completions/max_length": 4096.0,
416
+ "completions/max_terminated_length": 3533.0,
417
+ "completions/mean_length": 1321.40625,
418
+ "completions/mean_terminated_length": 1184.9508056640625,
419
+ "completions/min_length": 56.0,
420
+ "completions/min_terminated_length": 56.0,
421
+ "degenerate_groups_filtered": 0.0,
422
+ "entropy": 0.706970289349556,
423
+ "entropy/code": 0.5685221552848816,
424
+ "entropy/thinking": 1.2291285693645477,
425
+ "epoch": 0.024630541871921183,
426
+ "frac_reward_zero_std": 0.0,
427
+ "grad_norm": 0.010049425309084847,
428
+ "kl": 0.003098885623330716,
429
+ "kl/code_mean": 0.0034346847896813415,
430
+ "kl/code_sum": 11.960693359375,
431
+ "kl/thinking_mean": 0.0014323829964268953,
432
+ "kl/thinking_sum": 1.0888671875,
433
+ "kl/total_loss_contribution": 6.197771040206135e-05,
434
+ "kl_mask/effective_frac": 1.0,
435
+ "kl_mask/fallback_rate": 0.25,
436
+ "learning_rate": 4.998734975547991e-05,
437
+ "loss": -0.005550745874643326,
438
+ "num_tokens": 1546046.0,
439
+ "reward": 0.9140625,
440
+ "reward_std": 0.7335751056671143,
441
+ "rewards/reward_func/mean": 0.1015625,
442
+ "rewards/reward_func/std": 0.12709666623009575,
443
+ "sampling/importance_sampling_ratio/max": 2.998774290084839,
444
+ "sampling/importance_sampling_ratio/mean": 0.9479279518127441,
445
+ "sampling/importance_sampling_ratio/min": 0.0,
446
+ "sampling/sampling_logp_difference/max": 11.668327331542969,
447
+ "sampling/sampling_logp_difference/mean": 0.20149840414524078,
448
+ "seg/thinking_frac": 0.1998237892985344,
449
+ "step": 10,
450
+ "step_time": 100.57014895303291
451
+ }
452
+ ],
453
+ "logging_steps": 1,
454
+ "max_steps": 400,
455
+ "num_input_tokens_seen": 1546046,
456
+ "num_train_epochs": 1,
457
+ "save_steps": 10,
458
+ "stateful_callbacks": {
459
+ "TrainerControl": {
460
+ "args": {
461
+ "should_epoch_stop": false,
462
+ "should_evaluate": false,
463
+ "should_log": false,
464
+ "should_save": true,
465
+ "should_training_stop": false
466
+ },
467
+ "attributes": {}
468
+ }
469
+ },
470
+ "total_flos": 0.0,
471
+ "train_batch_size": 4,
472
+ "trial_name": null,
473
+ "trial_params": null
474
+ }