Add LoRA adapter weights (Llama-3.3-70B, personal finance)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,98 @@
+---
+base_model: meta-llama/Llama-3.3-70B-Instruct-Reference
+library_name: peft
+license: other
+tags:
+  - lora
+  - peft
+  - adapter
+  - adaption
+---
+
+# adaption_personal_finance_advice_qa
+
+## Model Training
+
+A LORA adapter for `meta-llama/Llama-3.3-70B-Instruct-Reference`. This model was trained with SFT using [Adaption](https://adaptionlabs.ai)'s AutoScientist on the personal_finance_advice_qa dataset.
+
+
+### AutoScientist Config
+
+```json
+{
+  "job_id": "50a9c873-763a-4032-86fc-c7c96ab8e932",
+  "training_experiment_id": "e03bb20c-f7f9-4ce5-975c-05dc36463138",
+  "original_model_name": "meta-llama/Llama-3.3-70B-Instruct-Reference",
+  "trained_model_name": "adaption_personal_finance_advice_qa",
+  "training_method": "sft",
+  "training_type": "lora",
+  "data_format": "chat",
+  "hyperparams": {
+    "lora": "true",
+    "lora_r": 64,
+    "n_evals": 5,
+    "n_epochs": 4,
+    "batch_size": "max",
+    "lora_alpha": 128,
+    "lora_dropout": 0,
+    "min_lr_ratio": 0.1,
+    "warmup_ratio": 0.05,
+    "weight_decay": 0.05,
+    "learning_rate": 0.0001,
+    "max_grad_norm": 1,
+    "base_model_size": "70B",
+    "train_on_inputs": "false",
+    "training_method": "sft",
+    "lr_scheduler_type": "cosine",
+    "scheduler_num_cycles": 0.5,
+    "lora_trainable_modules": "all-linear"
+  }
+}
+```
+
+## Training Data
+
+The model was trained on 11,639 rows of adapted data with the following domain distribution: personal-finance (90%), legal (5%), career-workplace (2%), transportation (1%), hr (1%), personal-growth (0%), medical (0%), parenting-family (0%), product-advice (0%), travel (0%), dating (0%), how-to (0%), cooking (0%), governance (0%), agriculture (0%), market-analysis (0%), corporate-business (0%), architecture-design (0%).
+
+## Model Evaluation
+
+The model was evaluated on an in-distribution held-out test set as well as a broader domain-specific test set to measure generalization.
+
+
+| Domain | Win rate vs. base model |
+| --- | --- |
+| personal-finance | 76% |
+
+## How to use
+
+```bash
+pip install torch transformers peft
+```
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+BASE = "meta-llama/Llama-3.3-70B-Instruct-Reference"
+ADAPTER = "<this-repo-id>"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float32 if device == "cpu" else torch.bfloat16
+
+base = AutoModelForCausalLM.from_pretrained(BASE, dtype=dtype).to(device)
+model = PeftModel.from_pretrained(base, ADAPTER)
+# Optional: merge the LoRA weights into the base for faster inference
+model = model.merge_and_unload()
+model.eval()
+
+tokenizer = AutoTokenizer.from_pretrained(BASE)
+messages = [{"role": "user", "content": "Hello!"}]
+text = tokenizer.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(text, return_tensors="pt").to(device)
+
+with torch.inference_mode():
+    out = model.generate(**inputs, max_new_tokens=512)
+print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "togethercomputer/Meta-Llama-3.3-70B-Instruct-Reference",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": [],
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3385a87ad8b45041d57d1ef04518fcb56d5bc122780680751a4ebb23d96684
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128009,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.10.1",
+  "use_cache": false,
+  "vocab_size": 128256,
+  "torch_dtype": "bfloat16"
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|eot_id|>",
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,1250 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 33,
+  "global_step": 168,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.023809523809523808,
+      "grad_norm": 0.19468727707862854,
+      "learning_rate": 0.0,
+      "loss": 1.3935546875,
+      "step": 1
+    },
+    {
+      "epoch": 0.047619047619047616,
+      "grad_norm": 0.25953492522239685,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 1.6396484375,
+      "step": 2
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 0.187949538230896,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 1.396484375,
+      "step": 3
+    },
+    {
+      "epoch": 0.09523809523809523,
+      "grad_norm": 0.14267033338546753,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.314453125,
+      "step": 4
+    },
+    {
+      "epoch": 0.11904761904761904,
+      "grad_norm": 0.13608872890472412,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 1.365234375,
+      "step": 5
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.08541593700647354,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 1.09423828125,
+      "step": 6
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "grad_norm": 0.09242819249629974,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.3681640625,
+      "step": 7
+    },
+    {
+      "epoch": 0.19047619047619047,
+      "grad_norm": 0.07986351102590561,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 1.13330078125,
+      "step": 8
+    },
+    {
+      "epoch": 0.21428571428571427,
+      "grad_norm": 0.12255741655826569,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 1.26171875,
+      "step": 9
+    },
+    {
+      "epoch": 0.23809523809523808,
+      "grad_norm": 0.12448930740356445,
+      "learning_rate": 0.0001,
+      "loss": 1.220703125,
+      "step": 10
+    },
+    {
+      "epoch": 0.2619047619047619,
+      "grad_norm": 0.07220698893070221,
+      "learning_rate": 9.99912163729821e-05,
+      "loss": 1.038330078125,
+      "step": 11
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 0.0470508448779583,
+      "learning_rate": 9.996486892091074e-05,
+      "loss": 1.03173828125,
+      "step": 12
+    },
+    {
+      "epoch": 0.30952380952380953,
+      "grad_norm": 0.033459704369306564,
+      "learning_rate": 9.992096792939448e-05,
+      "loss": 0.8048095703125,
+      "step": 13
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 0.05528910830616951,
+      "learning_rate": 9.985953053665264e-05,
+      "loss": 1.13671875,
+      "step": 14
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 0.052433520555496216,
+      "learning_rate": 9.978058072682489e-05,
+      "loss": 1.064453125,
+      "step": 15
+    },
+    {
+      "epoch": 0.38095238095238093,
+      "grad_norm": 0.044099658727645874,
+      "learning_rate": 9.968414932060826e-05,
+      "loss": 1.15234375,
+      "step": 16
+    },
+    {
+      "epoch": 0.40476190476190477,
+      "grad_norm": 0.042600758373737335,
+      "learning_rate": 9.957027396322522e-05,
+      "loss": 1.1826171875,
+      "step": 17
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 0.042188938707113266,
+      "learning_rate": 9.943899910972752e-05,
+      "loss": 1.0927734375,
+      "step": 18
+    },
+    {
+      "epoch": 0.4523809523809524,
+      "grad_norm": 0.043361663818359375,
+      "learning_rate": 9.929037600764186e-05,
+      "loss": 0.923095703125,
+      "step": 19
+    },
+    {
+      "epoch": 0.47619047619047616,
+      "grad_norm": 0.04190001264214516,
+      "learning_rate": 9.912446267696358e-05,
+      "loss": 0.9501953125,
+      "step": 20
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.040796488523483276,
+      "learning_rate": 9.894132388750662e-05,
+      "loss": 1.07861328125,
+      "step": 21
+    },
+    {
+      "epoch": 0.5238095238095238,
+      "grad_norm": 0.03711692616343498,
+      "learning_rate": 9.874103113361849e-05,
+      "loss": 1.06396484375,
+      "step": 22
+    },
+    {
+      "epoch": 0.5476190476190477,
+      "grad_norm": 0.03595205768942833,
+      "learning_rate": 9.852366260627007e-05,
+      "loss": 1.05419921875,
+      "step": 23
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.03659743070602417,
+      "learning_rate": 9.828930316253112e-05,
+      "loss": 1.03515625,
+      "step": 24
+    },
+    {
+      "epoch": 0.5952380952380952,
+      "grad_norm": 0.034952323883771896,
+      "learning_rate": 9.80380442924435e-05,
+      "loss": 0.8470458984375,
+      "step": 25
+    },
+    {
+      "epoch": 0.6190476190476191,
+      "grad_norm": 0.03435644879937172,
+      "learning_rate": 9.7769984083305e-05,
+      "loss": 0.97412109375,
+      "step": 26
+    },
+    {
+      "epoch": 0.6428571428571429,
+      "grad_norm": 0.03677888587117195,
+      "learning_rate": 9.74852271813776e-05,
+      "loss": 1.0244140625,
+      "step": 27
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.037179429084062576,
+      "learning_rate": 9.718388475103545e-05,
+      "loss": 1.0361328125,
+      "step": 28
+    },
+    {
+      "epoch": 0.6904761904761905,
+      "grad_norm": 0.03537730127573013,
+      "learning_rate": 9.686607443136796e-05,
+      "loss": 1.07275390625,
+      "step": 29
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 0.033934418112039566,
+      "learning_rate": 9.653192029025561e-05,
+      "loss": 1.0636672973632812,
+      "step": 30
+    },
+    {
+      "epoch": 0.7380952380952381,
+      "grad_norm": 0.0342508926987648,
+      "learning_rate": 9.618155277593583e-05,
+      "loss": 0.9755859375,
+      "step": 31
+    },
+    {
+      "epoch": 0.7619047619047619,
+      "grad_norm": 0.03621754050254822,
+      "learning_rate": 9.581510866607817e-05,
+      "loss": 1.02587890625,
+      "step": 32
+    },
+    {
+      "epoch": 0.7857142857142857,
+      "grad_norm": 0.029987191781401634,
+      "learning_rate": 9.543273101438859e-05,
+      "loss": 0.9814453125,
+      "step": 33
+    },
+    {
+      "epoch": 0.8095238095238095,
+      "grad_norm": 0.031167544424533844,
+      "learning_rate": 9.503456909476364e-05,
+      "loss": 0.8631591796875,
+      "step": 34
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.03571572154760361,
+      "learning_rate": 9.46207783430164e-05,
+      "loss": 0.89501953125,
+      "step": 35
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 0.03855983912944794,
+      "learning_rate": 9.419152029619696e-05,
+      "loss": 1.06396484375,
+      "step": 36
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "eval_loss": 0.95556640625,
+      "eval_runtime": 6.3853,
+      "eval_samples_per_second": 0.783,
+      "eval_steps_per_second": 0.157,
+      "step": 36
+    },
+    {
+      "epoch": 0.8809523809523809,
+      "grad_norm": 0.028229564428329468,
+      "learning_rate": 9.374696252953101e-05,
+      "loss": 0.88671875,
+      "step": 37
+    },
+    {
+      "epoch": 0.9047619047619048,
+      "grad_norm": 0.03608233481645584,
+      "learning_rate": 9.328727859100121e-05,
+      "loss": 1.0419921875,
+      "step": 38
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 0.035341136157512665,
+      "learning_rate": 9.2812647933597e-05,
+      "loss": 0.90478515625,
+      "step": 39
+    },
+    {
+      "epoch": 0.9523809523809523,
+      "grad_norm": 0.03361688181757927,
+      "learning_rate": 9.23232558452591e-05,
+      "loss": 0.99609375,
+      "step": 40
+    },
+    {
+      "epoch": 0.9761904761904762,
+      "grad_norm": 0.0339219830930233,
+      "learning_rate": 9.181929337654612e-05,
+      "loss": 0.91748046875,
+      "step": 41
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.05091368034482002,
+      "learning_rate": 9.130095726605169e-05,
+      "loss": 0.79888916015625,
+      "step": 42
+    },
+    {
+      "epoch": 1.0238095238095237,
+      "grad_norm": 0.03272996097803116,
+      "learning_rate": 9.07684498636009e-05,
+      "loss": 0.95849609375,
+      "step": 43
+    },
+    {
+      "epoch": 1.0476190476190477,
+      "grad_norm": 0.039782822132110596,
+      "learning_rate": 9.02219790512563e-05,
+      "loss": 1.04638671875,
+      "step": 44
+    },
+    {
+      "epoch": 1.0714285714285714,
+      "grad_norm": 0.03554113209247589,
+      "learning_rate": 8.96617581621642e-05,
+      "loss": 0.9638671875,
+      "step": 45
+    },
+    {
+      "epoch": 1.0952380952380953,
+      "grad_norm": 0.03178030997514725,
+      "learning_rate": 8.90880058972729e-05,
+      "loss": 0.939453125,
+      "step": 46
+    },
+    {
+      "epoch": 1.119047619047619,
+      "grad_norm": 0.036923594772815704,
+      "learning_rate": 8.850094623995557e-05,
+      "loss": 0.95703125,
+      "step": 47
+    },
+    {
+      "epoch": 1.1428571428571428,
+      "grad_norm": 0.03367718309164047,
+      "learning_rate": 8.790080836857073e-05,
+      "loss": 0.8343505859375,
+      "step": 48
+    },
+    {
+      "epoch": 1.1666666666666667,
+      "grad_norm": 0.03425205498933792,
+      "learning_rate": 8.72878265669949e-05,
+      "loss": 1.0361328125,
+      "step": 49
+    },
+    {
+      "epoch": 1.1904761904761905,
+      "grad_norm": 0.031229691579937935,
+      "learning_rate": 8.666224013316203e-05,
+      "loss": 0.87744140625,
+      "step": 50
+    },
+    {
+      "epoch": 1.2142857142857142,
+      "grad_norm": 0.03565659001469612,
+      "learning_rate": 8.602429328564554e-05,
+      "loss": 0.984375,
+      "step": 51
+    },
+    {
+      "epoch": 1.2380952380952381,
+      "grad_norm": 0.03452285751700401,
+      "learning_rate": 8.537423506831948e-05,
+      "loss": 0.96875,
+      "step": 52
+    },
+    {
+      "epoch": 1.2619047619047619,
+      "grad_norm": 0.03380265459418297,
+      "learning_rate": 8.471231925313589e-05,
+      "loss": 0.84814453125,
+      "step": 53
+    },
+    {
+      "epoch": 1.2857142857142856,
+      "grad_norm": 0.032453760504722595,
+      "learning_rate": 8.403880424105644e-05,
+      "loss": 0.86328125,
+      "step": 54
+    },
+    {
+      "epoch": 1.3095238095238095,
+      "grad_norm": 0.03952884301543236,
+      "learning_rate": 8.335395296117701e-05,
+      "loss": 0.6719970703125,
+      "step": 55
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 0.034161731600761414,
+      "learning_rate": 8.265803276808449e-05,
+      "loss": 0.95947265625,
+      "step": 56
+    },
+    {
+      "epoch": 1.3571428571428572,
+      "grad_norm": 0.033962540328502655,
+      "learning_rate": 8.195131533748601e-05,
+      "loss": 0.89990234375,
+      "step": 57
+    },
+    {
+      "epoch": 1.380952380952381,
+      "grad_norm": 0.03533850982785225,
+      "learning_rate": 8.123407656015125e-05,
+      "loss": 0.974609375,
+      "step": 58
+    },
+    {
+      "epoch": 1.4047619047619047,
+      "grad_norm": 0.03641538694500923,
+      "learning_rate": 8.050659643420921e-05,
+      "loss": 1.0,
+      "step": 59
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 0.03433677926659584,
+      "learning_rate": 7.976915895584163e-05,
+      "loss": 0.9296875,
+      "step": 60
+    },
+    {
+      "epoch": 1.4523809523809523,
+      "grad_norm": 0.03667641431093216,
+      "learning_rate": 7.902205200841559e-05,
+      "loss": 0.778076171875,
+      "step": 61
+    },
+    {
+      "epoch": 1.4761904761904763,
+      "grad_norm": 0.034833427518606186,
+      "learning_rate": 7.826556725009857e-05,
+      "loss": 0.8173828125,
+      "step": 62
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.0373813733458519,
+      "learning_rate": 7.75e-05,
+      "loss": 0.9296875,
+      "step": 63
+    },
+    {
+      "epoch": 1.5238095238095237,
+      "grad_norm": 0.039146050810813904,
+      "learning_rate": 7.672564912288352e-05,
+      "loss": 0.9169921875,
+      "step": 64
+    },
+    {
+      "epoch": 1.5476190476190477,
+      "grad_norm": 0.03721168264746666,
+      "learning_rate": 7.594281691249517e-05,
+      "loss": 0.9189453125,
+      "step": 65
+    },
+    {
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.04111553728580475,
+      "learning_rate": 7.515180897355285e-05,
+      "loss": 0.8896484375,
+      "step": 66
+    },
+    {
+      "epoch": 1.5952380952380953,
+      "grad_norm": 0.033117227256298065,
+      "learning_rate": 7.435293410244342e-05,
+      "loss": 0.73291015625,
+      "step": 67
+    },
+    {
+      "epoch": 1.619047619047619,
+      "grad_norm": 0.03749477490782738,
+      "learning_rate": 7.354650416667365e-05,
+      "loss": 0.841796875,
+      "step": 68
+    },
+    {
+      "epoch": 1.6428571428571428,
+      "grad_norm": 0.044476382434368134,
+      "learning_rate": 7.273283398312226e-05,
+      "loss": 0.89013671875,
+      "step": 69
+    },
+    {
+      "epoch": 1.6428571428571428,
+      "eval_loss": 0.89794921875,
+      "eval_runtime": 6.3986,
+      "eval_samples_per_second": 0.781,
+      "eval_steps_per_second": 0.156,
+      "step": 69
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 0.04343419894576073,
+      "learning_rate": 7.191224119514072e-05,
+      "loss": 0.88916015625,
+      "step": 70
+    },
+    {
+      "epoch": 1.6904761904761905,
+      "grad_norm": 0.04387887194752693,
+      "learning_rate": 7.108504614855052e-05,
+      "loss": 0.92529296875,
+      "step": 71
+    },
+    {
+      "epoch": 1.7142857142857144,
+      "grad_norm": 0.047499287873506546,
+      "learning_rate": 7.025157176658543e-05,
+      "loss": 0.917877197265625,
+      "step": 72
+    },
+    {
+      "epoch": 1.7380952380952381,
+      "grad_norm": 0.04214579984545708,
+      "learning_rate": 6.94121434238277e-05,
+      "loss": 0.8505859375,
+      "step": 73
+    },
+    {
+      "epoch": 1.7619047619047619,
+      "grad_norm": 0.04342899098992348,
+      "learning_rate": 6.856708881918714e-05,
+      "loss": 0.892578125,
+      "step": 74
+    },
+    {
+      "epoch": 1.7857142857142856,
+      "grad_norm": 0.04087640345096588,
+      "learning_rate": 6.771673784797296e-05,
+      "loss": 0.85693359375,
+      "step": 75
+    },
+    {
+      "epoch": 1.8095238095238095,
+      "grad_norm": 0.03890484198927879,
+      "learning_rate": 6.686142247310805e-05,
+      "loss": 0.7464599609375,
+      "step": 76
+    },
+    {
+      "epoch": 1.8333333333333335,
+      "grad_norm": 0.038060594350099564,
+      "learning_rate": 6.600147659553622e-05,
+      "loss": 0.7783203125,
+      "step": 77
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.04899071902036667,
+      "learning_rate": 6.513723592387277e-05,
+      "loss": 0.9326171875,
+      "step": 78
+    },
+    {
+      "epoch": 1.880952380952381,
+      "grad_norm": 0.03934891149401665,
+      "learning_rate": 6.426903784334943e-05,
+      "loss": 0.77880859375,
+      "step": 79
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "grad_norm": 0.04572214558720589,
+      "learning_rate": 6.33972212841047e-05,
+      "loss": 0.9130859375,
+      "step": 80
+    },
+    {
+      "epoch": 1.9285714285714286,
+      "grad_norm": 0.04346858337521553,
+      "learning_rate": 6.252212658887124e-05,
+      "loss": 0.7880859375,
+      "step": 81
+    },
+    {
+      "epoch": 1.9523809523809523,
+      "grad_norm": 0.04769371077418327,
+      "learning_rate": 6.164409538011167e-05,
+      "loss": 0.87255859375,
+      "step": 82
+    },
+    {
+      "epoch": 1.9761904761904763,
+      "grad_norm": 0.05800820142030716,
+      "learning_rate": 6.076347042665471e-05,
+      "loss": 0.8046875,
+      "step": 83
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.03807287663221359,
+      "learning_rate": 5.988059550988415e-05,
+      "loss": 0.681884765625,
+      "step": 84
+    },
+    {
+      "epoch": 2.0238095238095237,
+      "grad_norm": 0.04672769457101822,
+      "learning_rate": 5.899581528953207e-05,
+      "loss": 0.8271484375,
+      "step": 85
+    },
+    {
+      "epoch": 2.0476190476190474,
+      "grad_norm": 0.051381245255470276,
+      "learning_rate": 5.810947516912951e-05,
+      "loss": 0.90966796875,
+      "step": 86
+    },
+    {
+      "epoch": 2.0714285714285716,
+      "grad_norm": 0.05037863925099373,
+      "learning_rate": 5.722192116116664e-05,
+      "loss": 0.8447265625,
+      "step": 87
+    },
+    {
+      "epoch": 2.0952380952380953,
+      "grad_norm": 0.047110676765441895,
+      "learning_rate": 5.633349975201519e-05,
+      "loss": 0.81982421875,
+      "step": 88
+    },
+    {
+      "epoch": 2.119047619047619,
+      "grad_norm": 0.051410332322120667,
+      "learning_rate": 5.544455776666596e-05,
+      "loss": 0.8359375,
+      "step": 89
+    },
+    {
+      "epoch": 2.142857142857143,
+      "grad_norm": 0.0461672879755497,
+      "learning_rate": 5.4555442233334054e-05,
+      "loss": 0.7286376953125,
+      "step": 90
+    },
+    {
+      "epoch": 2.1666666666666665,
+      "grad_norm": 0.06829432398080826,
+      "learning_rate": 5.366650024798482e-05,
+      "loss": 0.90234375,
+      "step": 91
+    },
+    {
+      "epoch": 2.1904761904761907,
+      "grad_norm": 0.04619131237268448,
+      "learning_rate": 5.2778078838833365e-05,
+      "loss": 0.766357421875,
+      "step": 92
+    },
+    {
+      "epoch": 2.2142857142857144,
+      "grad_norm": 0.057096630334854126,
+      "learning_rate": 5.18905248308705e-05,
+      "loss": 0.8525390625,
+      "step": 93
+    },
+    {
+      "epoch": 2.238095238095238,
+      "grad_norm": 0.050893694162368774,
+      "learning_rate": 5.100418471046794e-05,
+      "loss": 0.8447265625,
+      "step": 94
+    },
+    {
+      "epoch": 2.261904761904762,
+      "grad_norm": 0.04828200489282608,
+      "learning_rate": 5.011940449011586e-05,
+      "loss": 0.741455078125,
+      "step": 95
+    },
+    {
+      "epoch": 2.2857142857142856,
+      "grad_norm": 0.048259492963552475,
+      "learning_rate": 4.92365295733453e-05,
+      "loss": 0.75634765625,
+      "step": 96
+    },
+    {
+      "epoch": 2.3095238095238093,
+      "grad_norm": 0.04044073820114136,
+      "learning_rate": 4.835590461988836e-05,
+      "loss": 0.5877685546875,
+      "step": 97
+    },
+    {
+      "epoch": 2.3333333333333335,
+      "grad_norm": 0.051675036549568176,
+      "learning_rate": 4.747787341112876e-05,
+      "loss": 0.83984375,
+      "step": 98
+    },
+    {
+      "epoch": 2.357142857142857,
+      "grad_norm": 0.05046414956450462,
+      "learning_rate": 4.66027787158953e-05,
+      "loss": 0.78515625,
+      "step": 99
+    },
+    {
+      "epoch": 2.380952380952381,
+      "grad_norm": 0.05342910438776016,
+      "learning_rate": 4.573096215665058e-05,
+      "loss": 0.849609375,
+      "step": 100
+    },
+    {
+      "epoch": 2.4047619047619047,
+      "grad_norm": 0.056085772812366486,
+      "learning_rate": 4.486276407612723e-05,
+      "loss": 0.87158203125,
+      "step": 101
+    },
+    {
+      "epoch": 2.4285714285714284,
+      "grad_norm": 0.053744085133075714,
+      "learning_rate": 4.399852340446379e-05,
+      "loss": 0.81005859375,
+      "step": 102
+    },
+    {
+      "epoch": 2.4285714285714284,
+      "eval_loss": 0.89111328125,
+      "eval_runtime": 6.389,
+      "eval_samples_per_second": 0.783,
+      "eval_steps_per_second": 0.157,
+      "step": 102
+    },
+    {
+      "epoch": 2.4523809523809526,
+      "grad_norm": 0.05157238617539406,
+      "learning_rate": 4.3138577526891964e-05,
+      "loss": 0.680908203125,
+      "step": 103
+    },
+    {
+      "epoch": 2.4761904761904763,
+      "grad_norm": 0.053052742034196854,
+      "learning_rate": 4.228326215202706e-05,
+      "loss": 0.712890625,
+      "step": 104
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.05984127148985863,
+      "learning_rate": 4.1432911180812875e-05,
+      "loss": 0.81103515625,
+      "step": 105
+    },
+    {
+      "epoch": 2.5238095238095237,
+      "grad_norm": 0.061421554535627365,
+      "learning_rate": 4.058785657617232e-05,
+      "loss": 0.7939453125,
+      "step": 106
+    },
+    {
+      "epoch": 2.5476190476190474,
+      "grad_norm": 0.0586903840303421,
+      "learning_rate": 3.974842823341458e-05,
+      "loss": 0.8076171875,
+      "step": 107
+    },
+    {
+      "epoch": 2.571428571428571,
+      "grad_norm": 0.05761508643627167,
+      "learning_rate": 3.89149538514495e-05,
+      "loss": 0.775390625,
+      "step": 108
+    },
+    {
+      "epoch": 2.5952380952380953,
+      "grad_norm": 0.08362194895744324,
+      "learning_rate": 3.8087758804859295e-05,
+      "loss": 0.643157958984375,
+      "step": 109
+    },
+    {
+      "epoch": 2.619047619047619,
+      "grad_norm": 0.05520815774798393,
+      "learning_rate": 3.726716601687775e-05,
+      "loss": 0.7333984375,
+      "step": 110
+    },
+    {
+      "epoch": 2.642857142857143,
+      "grad_norm": 0.060749031603336334,
+      "learning_rate": 3.645349583332637e-05,
+      "loss": 0.7763671875,
+      "step": 111
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 0.06587613373994827,
+      "learning_rate": 3.5647065897556596e-05,
+      "loss": 0.76416015625,
+      "step": 112
+    },
+    {
+      "epoch": 2.6904761904761907,
+      "grad_norm": 0.06273584812879562,
+      "learning_rate": 3.484819102644717e-05,
+      "loss": 0.798828125,
+      "step": 113
+    },
+    {
+      "epoch": 2.7142857142857144,
+      "grad_norm": 0.06525491923093796,
+      "learning_rate": 3.405718308750485e-05,
+      "loss": 0.782867431640625,
+      "step": 114
+    },
+    {
+      "epoch": 2.738095238095238,
+      "grad_norm": 0.062464285641908646,
+      "learning_rate": 3.327435087711649e-05,
+      "loss": 0.7421875,
+      "step": 115
+    },
+    {
+      "epoch": 2.761904761904762,
+      "grad_norm": 0.0668715164065361,
+      "learning_rate": 3.250000000000001e-05,
+      "loss": 0.77587890625,
+      "step": 116
+    },
+    {
+      "epoch": 2.7857142857142856,
+      "grad_norm": 0.06833118200302124,
+      "learning_rate": 3.173443274990145e-05,
+      "loss": 0.74267578125,
+      "step": 117
+    },
+    {
+      "epoch": 2.8095238095238093,
+      "grad_norm": 0.059672702103853226,
+      "learning_rate": 3.097794799158444e-05,
+      "loss": 0.6473388671875,
+      "step": 118
+    },
+    {
+      "epoch": 2.8333333333333335,
+      "grad_norm": 0.0572911761701107,
+      "learning_rate": 3.023084104415838e-05,
+      "loss": 0.681396484375,
+      "step": 119
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 0.06925146281719208,
+      "learning_rate": 2.949340356579081e-05,
+      "loss": 0.81201171875,
+      "step": 120
+    },
+    {
+      "epoch": 2.880952380952381,
+      "grad_norm": 0.058548275381326675,
+      "learning_rate": 2.8765923439848764e-05,
+      "loss": 0.681884765625,
+      "step": 121
+    },
+    {
+      "epoch": 2.9047619047619047,
+      "grad_norm": 0.06802289932966232,
+      "learning_rate": 2.8048684662514003e-05,
+      "loss": 0.7958984375,
+      "step": 122
+    },
+    {
+      "epoch": 2.928571428571429,
+      "grad_norm": 0.06336236745119095,
+      "learning_rate": 2.7341967231915534e-05,
+      "loss": 0.685546875,
+      "step": 123
+    },
+    {
+      "epoch": 2.9523809523809526,
+      "grad_norm": 0.0702662467956543,
+      "learning_rate": 2.6646047038823007e-05,
+      "loss": 0.7607421875,
+      "step": 124
+    },
+    {
+      "epoch": 2.9761904761904763,
+      "grad_norm": 0.06782855093479156,
+      "learning_rate": 2.596119575894358e-05,
+      "loss": 0.71142578125,
+      "step": 125
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.06282392889261246,
+      "learning_rate": 2.5287680746864136e-05,
+      "loss": 0.5767822265625,
+      "step": 126
+    },
+    {
+      "epoch": 3.0238095238095237,
+      "grad_norm": 0.06877307593822479,
+      "learning_rate": 2.4625764931680536e-05,
+      "loss": 0.701904296875,
+      "step": 127
+    },
+    {
+      "epoch": 3.0476190476190474,
+      "grad_norm": 0.075661800801754,
+      "learning_rate": 2.397570671435448e-05,
+      "loss": 0.78076171875,
+      "step": 128
+    },
+    {
+      "epoch": 3.0714285714285716,
+      "grad_norm": 0.06782848387956619,
+      "learning_rate": 2.3337759866837984e-05,
+      "loss": 0.7373046875,
+      "step": 129
+    },
+    {
+      "epoch": 3.0952380952380953,
+      "grad_norm": 0.08836882561445236,
+      "learning_rate": 2.271217343300509e-05,
+      "loss": 0.71240234375,
+      "step": 130
+    },
+    {
+      "epoch": 3.119047619047619,
+      "grad_norm": 0.07202401012182236,
+      "learning_rate": 2.209919163142926e-05,
+      "loss": 0.72607421875,
+      "step": 131
+    },
+    {
+      "epoch": 3.142857142857143,
+      "grad_norm": 0.0658789649605751,
+      "learning_rate": 2.1499053760044426e-05,
+      "loss": 0.6387939453125,
+      "step": 132
+    },
+    {
+      "epoch": 3.1666666666666665,
+      "grad_norm": 0.07658467441797256,
+      "learning_rate": 2.0911994102727095e-05,
+      "loss": 0.78076171875,
+      "step": 133
+    },
+    {
+      "epoch": 3.1904761904761907,
+      "grad_norm": 0.06584449857473373,
+      "learning_rate": 2.0338241837835813e-05,
+      "loss": 0.669921875,
+      "step": 134
+    },
+    {
+      "epoch": 3.2142857142857144,
+      "grad_norm": 0.07634307444095612,
+      "learning_rate": 1.977802094874371e-05,
+      "loss": 0.7353515625,
+      "step": 135
+    },
+    {
+      "epoch": 3.2142857142857144,
+      "eval_loss": 0.9072265625,
+      "eval_runtime": 6.3793,
+      "eval_samples_per_second": 0.784,
+      "eval_steps_per_second": 0.157,
+      "step": 135
+    },
+    {
+      "epoch": 3.238095238095238,
+      "grad_norm": 0.07586608082056046,
+      "learning_rate": 1.9231550136399102e-05,
+      "loss": 0.7333984375,
+      "step": 136
+    },
+    {
+      "epoch": 3.261904761904762,
+      "grad_norm": 0.06973817199468613,
+      "learning_rate": 1.8699042733948316e-05,
+      "loss": 0.647705078125,
+      "step": 137
+    },
+    {
+      "epoch": 3.2857142857142856,
+      "grad_norm": 0.07253584265708923,
+      "learning_rate": 1.818070662345389e-05,
+      "loss": 0.6630859375,
+      "step": 138
+    },
+    {
+      "epoch": 3.3095238095238093,
+      "grad_norm": 0.06525201350450516,
+      "learning_rate": 1.7676744154740922e-05,
+      "loss": 0.5201416015625,
+      "step": 139
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 0.0821850597858429,
+      "learning_rate": 1.7187352066403012e-05,
+      "loss": 0.744140625,
+      "step": 140
+    },
+    {
+      "epoch": 3.357142857142857,
+      "grad_norm": 0.07470913976430893,
+      "learning_rate": 1.6712721408998797e-05,
+      "loss": 0.69384765625,
+      "step": 141
+    },
+    {
+      "epoch": 3.380952380952381,
+      "grad_norm": 0.07955524325370789,
+      "learning_rate": 1.6253037470469e-05,
+      "loss": 0.74267578125,
+      "step": 142
+    },
+    {
+      "epoch": 3.4047619047619047,
+      "grad_norm": 0.07748151570558548,
+      "learning_rate": 1.5808479703803046e-05,
+      "loss": 0.763671875,
+      "step": 143
+    },
+    {
+      "epoch": 3.4285714285714284,
+      "grad_norm": 0.07890678197145462,
+      "learning_rate": 1.5379221656983617e-05,
+      "loss": 0.7119140625,
+      "step": 144
+    },
+    {
+      "epoch": 3.4523809523809526,
+      "grad_norm": 0.06560467183589935,
+      "learning_rate": 1.4965430905236377e-05,
+      "loss": 0.60595703125,
+      "step": 145
+    },
+    {
+      "epoch": 3.4761904761904763,
+      "grad_norm": 0.07202399522066116,
+      "learning_rate": 1.456726898561141e-05,
+      "loss": 0.6318359375,
+      "step": 146
+    },
+    {
+      "epoch": 3.5,
+      "grad_norm": 0.08173156529664993,
+      "learning_rate": 1.4184891333921824e-05,
+      "loss": 0.71630859375,
+      "step": 147
+    },
+    {
+      "epoch": 3.5238095238095237,
+      "grad_norm": 0.08595408499240875,
+      "learning_rate": 1.381844722406417e-05,
+      "loss": 0.701171875,
+      "step": 148
+    },
+    {
+      "epoch": 3.5476190476190474,
+      "grad_norm": 0.07890654355287552,
+      "learning_rate": 1.3468079709744394e-05,
+      "loss": 0.72412109375,
+      "step": 149
+    },
+    {
+      "epoch": 3.571428571428571,
+      "grad_norm": 0.0794968381524086,
+      "learning_rate": 1.313392556863205e-05,
+      "loss": 0.689453125,
+      "step": 150
+    },
+    {
+      "epoch": 3.5952380952380953,
+      "grad_norm": 0.07299920916557312,
+      "learning_rate": 1.281611524896456e-05,
+      "loss": 0.5780029296875,
+      "step": 151
+    },
+    {
+      "epoch": 3.619047619047619,
+      "grad_norm": 0.07846228778362274,
+      "learning_rate": 1.2514772818622392e-05,
+      "loss": 0.654296875,
+      "step": 152
+    },
+    {
+      "epoch": 3.642857142857143,
+      "grad_norm": 0.0770033448934555,
+      "learning_rate": 1.2230015916695003e-05,
+      "loss": 0.68896484375,
+      "step": 153
+    },
+    {
+      "epoch": 3.6666666666666665,
+      "grad_norm": 0.08408365398645401,
+      "learning_rate": 1.1961955707556493e-05,
+      "loss": 0.6728515625,
+      "step": 154
+    },
+    {
+      "epoch": 3.6904761904761907,
+      "grad_norm": 0.08708649128675461,
+      "learning_rate": 1.171069683746888e-05,
+      "loss": 0.705078125,
+      "step": 155
+    },
+    {
+      "epoch": 3.7142857142857144,
+      "grad_norm": 0.09406232833862305,
+      "learning_rate": 1.1476337393729933e-05,
+      "loss": 0.6865119934082031,
+      "step": 156
+    },
+    {
+      "epoch": 3.738095238095238,
+      "grad_norm": 0.0820961520075798,
+      "learning_rate": 1.1258968866381517e-05,
+      "loss": 0.6630859375,
+      "step": 157
+    },
+    {
+      "epoch": 3.761904761904762,
+      "grad_norm": 0.08721650391817093,
+      "learning_rate": 1.1058676112493394e-05,
+      "loss": 0.6923828125,
+      "step": 158
+    },
+    {
+      "epoch": 3.7857142857142856,
+      "grad_norm": 0.0904761329293251,
+      "learning_rate": 1.0875537323036434e-05,
+      "loss": 0.6640625,
+      "step": 159
+    },
+    {
+      "epoch": 3.8095238095238093,
+      "grad_norm": 0.08843201398849487,
+      "learning_rate": 1.0709623992358144e-05,
+      "loss": 0.5806884765625,
+      "step": 160
+    },
+    {
+      "epoch": 3.8333333333333335,
+      "grad_norm": 0.08065912872552872,
+      "learning_rate": 1.0561000890272495e-05,
+      "loss": 0.617431640625,
+      "step": 161
+    },
+    {
+      "epoch": 3.857142857142857,
+      "grad_norm": 0.08902918547391891,
+      "learning_rate": 1.04297260367748e-05,
+      "loss": 0.7333984375,
+      "step": 162
+    },
+    {
+      "epoch": 3.880952380952381,
+      "grad_norm": 0.07814877480268478,
+      "learning_rate": 1.031585067939174e-05,
+      "loss": 0.6190185546875,
+      "step": 163
+    },
+    {
+      "epoch": 3.9047619047619047,
+      "grad_norm": 0.08857692778110504,
+      "learning_rate": 1.0219419273175115e-05,
+      "loss": 0.72265625,
+      "step": 164
+    },
+    {
+      "epoch": 3.928571428571429,
+      "grad_norm": 0.07772155106067657,
+      "learning_rate": 1.014046946334738e-05,
+      "loss": 0.623291015625,
+      "step": 165
+    },
+    {
+      "epoch": 3.9523809523809526,
+      "grad_norm": 0.09102645516395569,
+      "learning_rate": 1.007903207060554e-05,
+      "loss": 0.693359375,
+      "step": 166
+    },
+    {
+      "epoch": 3.9761904761904763,
+      "grad_norm": 0.07549668103456497,
+      "learning_rate": 1.0035131079089268e-05,
+      "loss": 0.65283203125,
+      "step": 167
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.07577605545520782,
+      "learning_rate": 1.0008783627017913e-05,
+      "loss": 0.515533447265625,
+      "step": 168
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.9208984375,
+      "eval_runtime": 6.3687,
+      "eval_samples_per_second": 0.785,
+      "eval_steps_per_second": 0.157,
+      "step": 168
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 168,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3897071954197742e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}