lora only

README.md

@@ -1,3 +1,140 @@
 ---
 license: apache-2.0
+library_name: peft
+tags:
+- generated_from_trainer
+base_model: HuggingFaceTB/cosmo-1b
+model-index:
+- name: rag-lora-out
+  results: []
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.0`
+```yaml
+base_model: HuggingFaceTB/cosmo-1b
+model_type: LlamaForCausalLM
+tokenizer_type: LlamaTokenizer
+
+load_in_8bit: true
+load_in_4bit: false
+strict: false
+
+datasets:
+  - path: neural-bridge/rag-dataset-12000
+    type: context_qa.load_v2
+  - path: neural-bridge/rag-hallucination-dataset-1000
+    type: context_qa.load_v2
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./rag-lora-out
+
+sequence_len: 2048
+sample_packing: true
+pad_to_sequence_len: true
+
+adapter: lora
+lora_model_dir:
+lora_r: 32
+lora_alpha: 32
+lora_dropout: 0.1
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+wandb_project: Cosmo-1b-RAG-v0.1
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+gradient_accumulation_steps: 4
+micro_batch_size: 8
+eval_batch_size: 8
+num_epochs: 3
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_steps: 10
+evals_per_epoch: 4
+saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+special_tokens:
+
+```
+
+</details><br>
+
+# rag-lora-out
+
+This model is a fine-tuned version of [HuggingFaceTB/cosmo-1b](https://huggingface.co/HuggingFaceTB/cosmo-1b) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6086
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 10
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.5873        | 1.02  | 148  | 0.6392          |
+| 0.4513        | 2.02  | 296  | 0.6006          |
+| 0.422         | 2.95  | 435  | 0.6086          |
+
+
+### Framework versions
+
+- PEFT 0.9.1.dev0
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.1+cu121
+- Datasets 2.17.1
+- Tokenizers 0.15.0

adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/cosmo-1b",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24bd5232d3bbaf5cb3924dc87739532ef34b4a256997033cea388c17750e57ca
+size 144824970

checkpoint-145/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: HuggingFaceTB/cosmo-1b
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.1.dev0

checkpoint-145/adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/cosmo-1b",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-145/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e15aca1ee7543e55af667d28321a0b59a74e90f2f5e2b15430aec7bdc747d62c
+size 144748392

checkpoint-145/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bc4f1a539ae93886c989ceba123acdd796887ea67fd2510d7c7a04d2fb463ba
+size 72886996

checkpoint-145/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7d682e16e2d1eff31639821332e33972e3bbb8ddc928d8042f7015d3f47be01
+size 14244

checkpoint-145/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadc53b42f15a836e5c06cb606100a044276c39771c5eca3bb88745ea4ea20dd
+size 1064

checkpoint-145/trainer_state.json

@@ -0,0 +1,1036 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9965635738831615,
+  "eval_steps": 500,
+  "global_step": 145,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.37809523940086365,
+      "learning_rate": 2e-05,
+      "loss": 1.7814,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.366623193025589,
+      "learning_rate": 4e-05,
+      "loss": 1.69,
+      "step": 2
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.3546508848667145,
+      "learning_rate": 6e-05,
+      "loss": 1.7151,
+      "step": 3
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.36381417512893677,
+      "learning_rate": 8e-05,
+      "loss": 1.6609,
+      "step": 4
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.4049898684024811,
+      "learning_rate": 0.0001,
+      "loss": 1.752,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.4318874478340149,
+      "learning_rate": 0.00012,
+      "loss": 1.747,
+      "step": 6
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.39768990874290466,
+      "learning_rate": 0.00014,
+      "loss": 1.4432,
+      "step": 7
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.5056629180908203,
+      "learning_rate": 0.00016,
+      "loss": 1.7657,
+      "step": 8
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.4276069104671478,
+      "learning_rate": 0.00018,
+      "loss": 1.4221,
+      "step": 9
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 0.433552622795105,
+      "learning_rate": 0.0002,
+      "loss": 1.3117,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.42256900668144226,
+      "learning_rate": 0.0001999972679420179,
+      "loss": 1.2687,
+      "step": 11
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.3066452145576477,
+      "learning_rate": 0.00019998907191735434,
+      "loss": 1.0398,
+      "step": 12
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 0.3393724858760834,
+      "learning_rate": 0.00019997541237384966,
+      "loss": 1.0063,
+      "step": 13
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.49836328625679016,
+      "learning_rate": 0.00019995629005787713,
+      "loss": 1.2251,
+      "step": 14
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.3539387285709381,
+      "learning_rate": 0.0001999317060143023,
+      "loss": 0.9454,
+      "step": 15
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 0.35584160685539246,
+      "learning_rate": 0.0001999016615864258,
+      "loss": 1.0105,
+      "step": 16
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.33191052079200745,
+      "learning_rate": 0.00019986615841591002,
+      "loss": 1.1322,
+      "step": 17
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.3499913513660431,
+      "learning_rate": 0.00019982519844268933,
+      "loss": 0.8495,
+      "step": 18
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 0.2582085132598877,
+      "learning_rate": 0.0001997787839048642,
+      "loss": 0.9725,
+      "step": 19
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.23511624336242676,
+      "learning_rate": 0.00019972691733857883,
+      "loss": 0.9233,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.273019939661026,
+      "learning_rate": 0.00019966960157788248,
+      "loss": 1.0366,
+      "step": 21
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.283527135848999,
+      "learning_rate": 0.0001996068397545748,
+      "loss": 0.9944,
+      "step": 22
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24311360716819763,
+      "learning_rate": 0.00019953863529803466,
+      "loss": 1.0308,
+      "step": 23
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24368223547935486,
+      "learning_rate": 0.00019946499193503262,
+      "loss": 1.0051,
+      "step": 24
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 0.2269805669784546,
+      "learning_rate": 0.0001993859136895274,
+      "loss": 0.835,
+      "step": 25
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.21915565431118011,
+      "learning_rate": 0.00019930140488244602,
+      "loss": 0.9997,
+      "step": 26
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.21885964274406433,
+      "learning_rate": 0.0001992114701314478,
+      "loss": 1.0013,
+      "step": 27
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.22426557540893555,
+      "learning_rate": 0.00019911611435067172,
+      "loss": 0.8633,
+      "step": 28
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.21587449312210083,
+      "learning_rate": 0.0001990153427504683,
+      "loss": 0.8479,
+      "step": 29
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.24985891580581665,
+      "learning_rate": 0.0001989091608371146,
+      "loss": 0.8936,
+      "step": 30
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.19733141362667084,
+      "learning_rate": 0.0001987975744125135,
+      "loss": 0.7743,
+      "step": 31
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.1997266411781311,
+      "learning_rate": 0.00019868058957387663,
+      "loss": 0.9078,
+      "step": 32
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.24114945530891418,
+      "learning_rate": 0.00019855821271339125,
+      "loss": 1.0413,
+      "step": 33
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.2427120953798294,
+      "learning_rate": 0.00019843045051787096,
+      "loss": 0.8308,
+      "step": 34
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.1968858540058136,
+      "learning_rate": 0.0001982973099683902,
+      "loss": 0.818,
+      "step": 35
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2783421576023102,
+      "learning_rate": 0.00019815879833990304,
+      "loss": 0.8504,
+      "step": 36
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.23332269489765167,
+      "learning_rate": 0.00019801492320084546,
+      "loss": 0.9826,
+      "step": 37
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.22864645719528198,
+      "learning_rate": 0.00019786569241272197,
+      "loss": 0.8705,
+      "step": 38
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20620585978031158,
+      "learning_rate": 0.00019771111412967583,
+      "loss": 0.8264,
+      "step": 39
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20590244233608246,
+      "learning_rate": 0.00019755119679804367,
+      "loss": 0.8455,
+      "step": 40
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.20340456068515778,
+      "learning_rate": 0.00019738594915589397,
+      "loss": 0.871,
+      "step": 41
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 0.2383277714252472,
+      "learning_rate": 0.0001972153802325495,
+      "loss": 0.6833,
+      "step": 42
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.21479888260364532,
+      "learning_rate": 0.00019703949934809408,
+      "loss": 0.8355,
+      "step": 43
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.23538872599601746,
+      "learning_rate": 0.0001968583161128631,
+      "loss": 0.8479,
+      "step": 44
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 0.2260449230670929,
+      "learning_rate": 0.00019667184042691875,
+      "loss": 0.779,
+      "step": 45
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.20232993364334106,
+      "learning_rate": 0.0001964800824795087,
+      "loss": 0.7293,
+      "step": 46
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.19780154526233673,
+      "learning_rate": 0.00019628305274850956,
+      "loss": 0.7709,
+      "step": 47
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 0.24822202324867249,
+      "learning_rate": 0.00019608076199985433,
+      "loss": 0.8885,
+      "step": 48
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.20940056443214417,
+      "learning_rate": 0.0001958732212869441,
+      "loss": 0.6916,
+      "step": 49
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.18944858014583588,
+      "learning_rate": 0.0001956604419500441,
+      "loss": 0.7326,
+      "step": 50
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.201398104429245,
+      "learning_rate": 0.00019544243561566403,
+      "loss": 0.6542,
+      "step": 51
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.22385160624980927,
+      "learning_rate": 0.00019521921419592283,
+      "loss": 0.8036,
+      "step": 52
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.19502398371696472,
+      "learning_rate": 0.0001949907898878977,
+      "loss": 0.7579,
+      "step": 53
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 0.21633867919445038,
+      "learning_rate": 0.00019475717517295778,
+      "loss": 0.7803,
+      "step": 54
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.24109330773353577,
+      "learning_rate": 0.00019451838281608197,
+      "loss": 0.7745,
+      "step": 55
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.20285621285438538,
+      "learning_rate": 0.00019427442586516155,
+      "loss": 0.7306,
+      "step": 56
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 0.23838144540786743,
+      "learning_rate": 0.00019402531765028722,
+      "loss": 0.7606,
+      "step": 57
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.25280559062957764,
+      "learning_rate": 0.00019377107178302074,
+      "loss": 0.7928,
+      "step": 58
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.24614648520946503,
+      "learning_rate": 0.00019351170215565114,
+      "loss": 0.739,
+      "step": 59
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.20391589403152466,
+      "learning_rate": 0.00019324722294043558,
+      "loss": 0.7636,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.23704086244106293,
+      "learning_rate": 0.00019297764858882514,
+      "loss": 0.6919,
+      "step": 61
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22057901322841644,
+      "learning_rate": 0.00019270299383067498,
+      "loss": 0.6035,
+      "step": 62
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22797846794128418,
+      "learning_rate": 0.0001924232736734396,
+      "loss": 0.7087,
+      "step": 63
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.22181542217731476,
+      "learning_rate": 0.00019213850340135276,
+      "loss": 0.8088,
+      "step": 64
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.2096872180700302,
+      "learning_rate": 0.00019184869857459232,
+      "loss": 0.7073,
+      "step": 65
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.192199245095253,
+      "learning_rate": 0.00019155387502843013,
+      "loss": 0.6605,
+      "step": 66
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.2242266982793808,
+      "learning_rate": 0.00019125404887236663,
+      "loss": 0.7287,
+      "step": 67
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.24139907956123352,
+      "learning_rate": 0.00019094923648925067,
+      "loss": 0.6611,
+      "step": 68
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.22859904170036316,
+      "learning_rate": 0.00019063945453438432,
+      "loss": 0.6189,
+      "step": 69
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.27671098709106445,
+      "learning_rate": 0.0001903247199346129,
+      "loss": 0.8595,
+      "step": 70
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.23342834413051605,
+      "learning_rate": 0.00019000504988739986,
+      "loss": 0.6718,
+      "step": 71
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.2517896890640259,
+      "learning_rate": 0.00018968046185988732,
+      "loss": 0.722,
+      "step": 72
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.25217679142951965,
+      "learning_rate": 0.00018935097358794144,
+      "loss": 0.824,
+      "step": 73
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 0.2682301998138428,
+      "learning_rate": 0.00018901660307518354,
+      "loss": 0.7837,
+      "step": 74
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.28926676511764526,
+      "learning_rate": 0.0001886773685920062,
+      "loss": 0.7364,
+      "step": 75
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.25249186158180237,
+      "learning_rate": 0.00018833328867457497,
+      "loss": 0.8566,
+      "step": 76
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 0.21752771735191345,
+      "learning_rate": 0.0001879843821238155,
+      "loss": 0.7077,
+      "step": 77
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.22346794605255127,
+      "learning_rate": 0.00018763066800438636,
+      "loss": 0.6522,
+      "step": 78
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.23172937333583832,
+      "learning_rate": 0.00018727216564363723,
+      "loss": 0.6099,
+      "step": 79
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.20895916223526,
+      "learning_rate": 0.00018690889463055283,
+      "loss": 0.7242,
+      "step": 80
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.26868510246276855,
+      "learning_rate": 0.0001865408748146826,
+      "loss": 0.705,
+      "step": 81
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.243666410446167,
+      "learning_rate": 0.00018616812630505597,
+      "loss": 0.8512,
+      "step": 82
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 0.21445880830287933,
+      "learning_rate": 0.00018579066946908384,
+      "loss": 0.7092,
+      "step": 83
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.22805824875831604,
+      "learning_rate": 0.00018540852493144545,
+      "loss": 0.6757,
+      "step": 84
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.25106164813041687,
+      "learning_rate": 0.00018502171357296144,
+      "loss": 0.7171,
+      "step": 85
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 0.2290162742137909,
+      "learning_rate": 0.000184630256529453,
+      "loss": 0.7053,
+      "step": 86
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.21118655800819397,
+      "learning_rate": 0.00018423417519058694,
+      "loss": 0.5827,
+      "step": 87
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.23761862516403198,
+      "learning_rate": 0.00018383349119870695,
+      "loss": 0.5827,
+      "step": 88
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 0.23689566552639008,
+      "learning_rate": 0.00018342822644765104,
+      "loss": 0.6401,
+      "step": 89
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.26510947942733765,
+      "learning_rate": 0.00018301840308155507,
+      "loss": 0.5963,
+      "step": 90
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.2142990678548813,
+      "learning_rate": 0.0001826040434936431,
+      "loss": 0.6258,
+      "step": 91
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.20709159970283508,
+      "learning_rate": 0.00018218517032500344,
+      "loss": 0.5685,
+      "step": 92
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.2485310137271881,
+      "learning_rate": 0.0001817618064633518,
+      "loss": 0.5901,
+      "step": 93
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2432468980550766,
+      "learning_rate": 0.00018133397504178057,
+      "loss": 0.6891,
+      "step": 94
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2318205088376999,
+      "learning_rate": 0.00018090169943749476,
+      "loss": 0.7339,
+      "step": 95
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.25346460938453674,
+      "learning_rate": 0.00018046500327053463,
+      "loss": 0.6606,
+      "step": 96
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.29421111941337585,
+      "learning_rate": 0.0001800239104024851,
+      "loss": 0.6698,
+      "step": 97
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.2356603443622589,
+      "learning_rate": 0.00017957844493517213,
+      "loss": 0.6225,
+      "step": 98
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.2331649512052536,
+      "learning_rate": 0.00017912863120934534,
+      "loss": 0.6838,
+      "step": 99
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.23008978366851807,
+      "learning_rate": 0.00017867449380334834,
+      "loss": 0.6303,
+      "step": 100
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.21604081988334656,
+      "learning_rate": 0.00017821605753177562,
+      "loss": 0.4923,
+      "step": 101
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.2283111810684204,
+      "learning_rate": 0.00017775334744411678,
+      "loss": 0.6344,
+      "step": 102
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.2448730617761612,
+      "learning_rate": 0.00017728638882338746,
+      "loss": 0.6789,
+      "step": 103
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.22479577362537384,
+      "learning_rate": 0.00017681520718474823,
+      "loss": 0.5841,
+      "step": 104
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.21999216079711914,
+      "learning_rate": 0.00017633982827411032,
+      "loss": 0.6402,
+      "step": 105
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 0.2296195775270462,
+      "learning_rate": 0.00017586027806672857,
+      "loss": 0.6148,
+      "step": 106
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.22455835342407227,
+      "learning_rate": 0.00017537658276578247,
+      "loss": 0.6418,
+      "step": 107
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.23929741978645325,
+      "learning_rate": 0.00017488876880094413,
+      "loss": 0.5459,
+      "step": 108
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.229934424161911,
+      "learning_rate": 0.00017439686282693436,
+      "loss": 0.6049,
+      "step": 109
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.24801373481750488,
+      "learning_rate": 0.00017390089172206592,
+      "loss": 0.7663,
+      "step": 110
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.2523267865180969,
+      "learning_rate": 0.00017340088258677522,
+      "loss": 0.6642,
+      "step": 111
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 0.22252008318901062,
+      "learning_rate": 0.00017289686274214118,
+      "loss": 0.6325,
+      "step": 112
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.27619731426239014,
+      "learning_rate": 0.0001723888597283926,
+      "loss": 0.655,
+      "step": 113
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.25090277194976807,
+      "learning_rate": 0.00017187690130340328,
+      "loss": 0.6251,
+      "step": 114
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 0.2236175239086151,
+      "learning_rate": 0.00017136101544117525,
+      "loss": 0.5959,
+      "step": 115
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.21940240263938904,
+      "learning_rate": 0.00017084123033031024,
+      "loss": 0.6028,
+      "step": 116
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.2741597294807434,
+      "learning_rate": 0.00017031757437246947,
+      "loss": 0.4947,
+      "step": 117
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 0.23343035578727722,
+      "learning_rate": 0.00016979007618082175,
+      "loss": 0.4415,
+      "step": 118
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.2116251140832901,
+      "learning_rate": 0.00016925876457848,
+      "loss": 0.5283,
+      "step": 119
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.25621068477630615,
+      "learning_rate": 0.00016872366859692627,
+      "loss": 0.7334,
+      "step": 120
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 0.22588759660720825,
+      "learning_rate": 0.00016818481747442554,
+      "loss": 0.6514,
+      "step": 121
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.24995574355125427,
+      "learning_rate": 0.00016764224065442796,
+      "loss": 0.5632,
+      "step": 122
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.2382134348154068,
+      "learning_rate": 0.00016709596778396026,
+      "loss": 0.5954,
+      "step": 123
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.25166618824005127,
+      "learning_rate": 0.00016654602871200546,
+      "loss": 0.6092,
+      "step": 124
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.23028074204921722,
+      "learning_rate": 0.0001659924534878723,
+      "loss": 0.576,
+      "step": 125
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.2527604103088379,
+      "learning_rate": 0.00016543527235955282,
+      "loss": 0.5985,
+      "step": 126
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.25870686769485474,
+      "learning_rate": 0.00016487451577207018,
+      "loss": 0.5767,
+      "step": 127
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.24936434626579285,
+      "learning_rate": 0.0001643102143658145,
+      "loss": 0.7494,
+      "step": 128
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.25158020853996277,
+      "learning_rate": 0.000163742398974869,
+      "loss": 0.578,
+      "step": 129
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.22290346026420593,
+      "learning_rate": 0.0001631711006253251,
+      "loss": 0.6489,
+      "step": 130
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.26421919465065,
+      "learning_rate": 0.00016259635053358717,
+      "loss": 0.5669,
+      "step": 131
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.2573990225791931,
+      "learning_rate": 0.0001620181801046667,
+      "loss": 0.6405,
+      "step": 132
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.19883453845977783,
+      "learning_rate": 0.00016143662093046638,
+      "loss": 0.4731,
+      "step": 133
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.24181459844112396,
+      "learning_rate": 0.00016085170478805395,
+      "loss": 0.6819,
+      "step": 134
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2319963425397873,
+      "learning_rate": 0.00016026346363792567,
+      "loss": 0.6373,
+      "step": 135
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2552095055580139,
+      "learning_rate": 0.0001596719296222601,
+      "loss": 0.7151,
+      "step": 136
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.27191445231437683,
+      "learning_rate": 0.00015907713506316192,
+      "loss": 0.611,
+      "step": 137
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.26981061697006226,
+      "learning_rate": 0.0001584791124608955,
+      "loss": 0.5923,
+      "step": 138
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23907916247844696,
+      "learning_rate": 0.00015787789449210938,
+      "loss": 0.5655,
+      "step": 139
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23364509642124176,
+      "learning_rate": 0.00015727351400805052,
+      "loss": 0.6019,
+      "step": 140
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 0.23096178472042084,
+      "learning_rate": 0.0001566660040327695,
+      "loss": 0.602,
+      "step": 141
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.2656322717666626,
+      "learning_rate": 0.0001560553977613158,
+      "loss": 0.6195,
+      "step": 142
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.23051652312278748,
+      "learning_rate": 0.00015544172855792423,
+      "loss": 0.6132,
+      "step": 143
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 0.2443581074476242,
+      "learning_rate": 0.00015482502995419167,
+      "loss": 0.6144,
+      "step": 144
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.2075706273317337,
+      "learning_rate": 0.00015420533564724495,
+      "loss": 0.5556,
+      "step": 145
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 435,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 145,
+  "total_flos": 9.76361694953472e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-145/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f4e74f968ef592cbe5e646087e575fbeada43b9dc36d459d99d6a9ebc528b7
+size 5560

checkpoint-290/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: HuggingFaceTB/cosmo-1b
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.1.dev0

checkpoint-290/adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/cosmo-1b",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-290/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3c3beef805b9b3a73aa826333b30c1f8b651be54657b72a6274b31ac5be49be
+size 144748392

checkpoint-290/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f46f8a2c6fccf8486194b3dca83a0baaec044d0dba76b81f549546e0c11df636
+size 72887380

checkpoint-290/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8b7fb51585bedb5cfc596fed320770581a502ad993a98decb3f0b397ca635e8
+size 14244

checkpoint-290/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a977aa284b2579061f2f683a13b0da8d3cef821bd533e41970eab743abe4618
+size 1064

checkpoint-290/trainer_state.json

@@ -0,0 +1,2059 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9742268041237114,
+  "eval_steps": 500,
+  "global_step": 290,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.37809523940086365,
+      "learning_rate": 2e-05,
+      "loss": 1.7814,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.366623193025589,
+      "learning_rate": 4e-05,
+      "loss": 1.69,
+      "step": 2
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.3546508848667145,
+      "learning_rate": 6e-05,
+      "loss": 1.7151,
+      "step": 3
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.36381417512893677,
+      "learning_rate": 8e-05,
+      "loss": 1.6609,
+      "step": 4
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.4049898684024811,
+      "learning_rate": 0.0001,
+      "loss": 1.752,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.4318874478340149,
+      "learning_rate": 0.00012,
+      "loss": 1.747,
+      "step": 6
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.39768990874290466,
+      "learning_rate": 0.00014,
+      "loss": 1.4432,
+      "step": 7
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.5056629180908203,
+      "learning_rate": 0.00016,
+      "loss": 1.7657,
+      "step": 8
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.4276069104671478,
+      "learning_rate": 0.00018,
+      "loss": 1.4221,
+      "step": 9
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 0.433552622795105,
+      "learning_rate": 0.0002,
+      "loss": 1.3117,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.42256900668144226,
+      "learning_rate": 0.0001999972679420179,
+      "loss": 1.2687,
+      "step": 11
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.3066452145576477,
+      "learning_rate": 0.00019998907191735434,
+      "loss": 1.0398,
+      "step": 12
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 0.3393724858760834,
+      "learning_rate": 0.00019997541237384966,
+      "loss": 1.0063,
+      "step": 13
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.49836328625679016,
+      "learning_rate": 0.00019995629005787713,
+      "loss": 1.2251,
+      "step": 14
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.3539387285709381,
+      "learning_rate": 0.0001999317060143023,
+      "loss": 0.9454,
+      "step": 15
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 0.35584160685539246,
+      "learning_rate": 0.0001999016615864258,
+      "loss": 1.0105,
+      "step": 16
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.33191052079200745,
+      "learning_rate": 0.00019986615841591002,
+      "loss": 1.1322,
+      "step": 17
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.3499913513660431,
+      "learning_rate": 0.00019982519844268933,
+      "loss": 0.8495,
+      "step": 18
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 0.2582085132598877,
+      "learning_rate": 0.0001997787839048642,
+      "loss": 0.9725,
+      "step": 19
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.23511624336242676,
+      "learning_rate": 0.00019972691733857883,
+      "loss": 0.9233,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.273019939661026,
+      "learning_rate": 0.00019966960157788248,
+      "loss": 1.0366,
+      "step": 21
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.283527135848999,
+      "learning_rate": 0.0001996068397545748,
+      "loss": 0.9944,
+      "step": 22
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24311360716819763,
+      "learning_rate": 0.00019953863529803466,
+      "loss": 1.0308,
+      "step": 23
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24368223547935486,
+      "learning_rate": 0.00019946499193503262,
+      "loss": 1.0051,
+      "step": 24
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 0.2269805669784546,
+      "learning_rate": 0.0001993859136895274,
+      "loss": 0.835,
+      "step": 25
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.21915565431118011,
+      "learning_rate": 0.00019930140488244602,
+      "loss": 0.9997,
+      "step": 26
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.21885964274406433,
+      "learning_rate": 0.0001992114701314478,
+      "loss": 1.0013,
+      "step": 27
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.22426557540893555,
+      "learning_rate": 0.00019911611435067172,
+      "loss": 0.8633,
+      "step": 28
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.21587449312210083,
+      "learning_rate": 0.0001990153427504683,
+      "loss": 0.8479,
+      "step": 29
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.24985891580581665,
+      "learning_rate": 0.0001989091608371146,
+      "loss": 0.8936,
+      "step": 30
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.19733141362667084,
+      "learning_rate": 0.0001987975744125135,
+      "loss": 0.7743,
+      "step": 31
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.1997266411781311,
+      "learning_rate": 0.00019868058957387663,
+      "loss": 0.9078,
+      "step": 32
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.24114945530891418,
+      "learning_rate": 0.00019855821271339125,
+      "loss": 1.0413,
+      "step": 33
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.2427120953798294,
+      "learning_rate": 0.00019843045051787096,
+      "loss": 0.8308,
+      "step": 34
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.1968858540058136,
+      "learning_rate": 0.0001982973099683902,
+      "loss": 0.818,
+      "step": 35
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2783421576023102,
+      "learning_rate": 0.00019815879833990304,
+      "loss": 0.8504,
+      "step": 36
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.23332269489765167,
+      "learning_rate": 0.00019801492320084546,
+      "loss": 0.9826,
+      "step": 37
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.22864645719528198,
+      "learning_rate": 0.00019786569241272197,
+      "loss": 0.8705,
+      "step": 38
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20620585978031158,
+      "learning_rate": 0.00019771111412967583,
+      "loss": 0.8264,
+      "step": 39
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20590244233608246,
+      "learning_rate": 0.00019755119679804367,
+      "loss": 0.8455,
+      "step": 40
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.20340456068515778,
+      "learning_rate": 0.00019738594915589397,
+      "loss": 0.871,
+      "step": 41
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 0.2383277714252472,
+      "learning_rate": 0.0001972153802325495,
+      "loss": 0.6833,
+      "step": 42
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.21479888260364532,
+      "learning_rate": 0.00019703949934809408,
+      "loss": 0.8355,
+      "step": 43
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.23538872599601746,
+      "learning_rate": 0.0001968583161128631,
+      "loss": 0.8479,
+      "step": 44
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 0.2260449230670929,
+      "learning_rate": 0.00019667184042691875,
+      "loss": 0.779,
+      "step": 45
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.20232993364334106,
+      "learning_rate": 0.0001964800824795087,
+      "loss": 0.7293,
+      "step": 46
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.19780154526233673,
+      "learning_rate": 0.00019628305274850956,
+      "loss": 0.7709,
+      "step": 47
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 0.24822202324867249,
+      "learning_rate": 0.00019608076199985433,
+      "loss": 0.8885,
+      "step": 48
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.20940056443214417,
+      "learning_rate": 0.0001958732212869441,
+      "loss": 0.6916,
+      "step": 49
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.18944858014583588,
+      "learning_rate": 0.0001956604419500441,
+      "loss": 0.7326,
+      "step": 50
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.201398104429245,
+      "learning_rate": 0.00019544243561566403,
+      "loss": 0.6542,
+      "step": 51
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.22385160624980927,
+      "learning_rate": 0.00019521921419592283,
+      "loss": 0.8036,
+      "step": 52
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.19502398371696472,
+      "learning_rate": 0.0001949907898878977,
+      "loss": 0.7579,
+      "step": 53
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 0.21633867919445038,
+      "learning_rate": 0.00019475717517295778,
+      "loss": 0.7803,
+      "step": 54
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.24109330773353577,
+      "learning_rate": 0.00019451838281608197,
+      "loss": 0.7745,
+      "step": 55
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.20285621285438538,
+      "learning_rate": 0.00019427442586516155,
+      "loss": 0.7306,
+      "step": 56
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 0.23838144540786743,
+      "learning_rate": 0.00019402531765028722,
+      "loss": 0.7606,
+      "step": 57
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.25280559062957764,
+      "learning_rate": 0.00019377107178302074,
+      "loss": 0.7928,
+      "step": 58
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.24614648520946503,
+      "learning_rate": 0.00019351170215565114,
+      "loss": 0.739,
+      "step": 59
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.20391589403152466,
+      "learning_rate": 0.00019324722294043558,
+      "loss": 0.7636,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.23704086244106293,
+      "learning_rate": 0.00019297764858882514,
+      "loss": 0.6919,
+      "step": 61
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22057901322841644,
+      "learning_rate": 0.00019270299383067498,
+      "loss": 0.6035,
+      "step": 62
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22797846794128418,
+      "learning_rate": 0.0001924232736734396,
+      "loss": 0.7087,
+      "step": 63
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.22181542217731476,
+      "learning_rate": 0.00019213850340135276,
+      "loss": 0.8088,
+      "step": 64
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.2096872180700302,
+      "learning_rate": 0.00019184869857459232,
+      "loss": 0.7073,
+      "step": 65
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.192199245095253,
+      "learning_rate": 0.00019155387502843013,
+      "loss": 0.6605,
+      "step": 66
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.2242266982793808,
+      "learning_rate": 0.00019125404887236663,
+      "loss": 0.7287,
+      "step": 67
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.24139907956123352,
+      "learning_rate": 0.00019094923648925067,
+      "loss": 0.6611,
+      "step": 68
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.22859904170036316,
+      "learning_rate": 0.00019063945453438432,
+      "loss": 0.6189,
+      "step": 69
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.27671098709106445,
+      "learning_rate": 0.0001903247199346129,
+      "loss": 0.8595,
+      "step": 70
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.23342834413051605,
+      "learning_rate": 0.00019000504988739986,
+      "loss": 0.6718,
+      "step": 71
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.2517896890640259,
+      "learning_rate": 0.00018968046185988732,
+      "loss": 0.722,
+      "step": 72
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.25217679142951965,
+      "learning_rate": 0.00018935097358794144,
+      "loss": 0.824,
+      "step": 73
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 0.2682301998138428,
+      "learning_rate": 0.00018901660307518354,
+      "loss": 0.7837,
+      "step": 74
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.28926676511764526,
+      "learning_rate": 0.0001886773685920062,
+      "loss": 0.7364,
+      "step": 75
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.25249186158180237,
+      "learning_rate": 0.00018833328867457497,
+      "loss": 0.8566,
+      "step": 76
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 0.21752771735191345,
+      "learning_rate": 0.0001879843821238155,
+      "loss": 0.7077,
+      "step": 77
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.22346794605255127,
+      "learning_rate": 0.00018763066800438636,
+      "loss": 0.6522,
+      "step": 78
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.23172937333583832,
+      "learning_rate": 0.00018727216564363723,
+      "loss": 0.6099,
+      "step": 79
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.20895916223526,
+      "learning_rate": 0.00018690889463055283,
+      "loss": 0.7242,
+      "step": 80
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.26868510246276855,
+      "learning_rate": 0.0001865408748146826,
+      "loss": 0.705,
+      "step": 81
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.243666410446167,
+      "learning_rate": 0.00018616812630505597,
+      "loss": 0.8512,
+      "step": 82
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 0.21445880830287933,
+      "learning_rate": 0.00018579066946908384,
+      "loss": 0.7092,
+      "step": 83
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.22805824875831604,
+      "learning_rate": 0.00018540852493144545,
+      "loss": 0.6757,
+      "step": 84
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.25106164813041687,
+      "learning_rate": 0.00018502171357296144,
+      "loss": 0.7171,
+      "step": 85
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 0.2290162742137909,
+      "learning_rate": 0.000184630256529453,
+      "loss": 0.7053,
+      "step": 86
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.21118655800819397,
+      "learning_rate": 0.00018423417519058694,
+      "loss": 0.5827,
+      "step": 87
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.23761862516403198,
+      "learning_rate": 0.00018383349119870695,
+      "loss": 0.5827,
+      "step": 88
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 0.23689566552639008,
+      "learning_rate": 0.00018342822644765104,
+      "loss": 0.6401,
+      "step": 89
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.26510947942733765,
+      "learning_rate": 0.00018301840308155507,
+      "loss": 0.5963,
+      "step": 90
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.2142990678548813,
+      "learning_rate": 0.0001826040434936431,
+      "loss": 0.6258,
+      "step": 91
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.20709159970283508,
+      "learning_rate": 0.00018218517032500344,
+      "loss": 0.5685,
+      "step": 92
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.2485310137271881,
+      "learning_rate": 0.0001817618064633518,
+      "loss": 0.5901,
+      "step": 93
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2432468980550766,
+      "learning_rate": 0.00018133397504178057,
+      "loss": 0.6891,
+      "step": 94
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2318205088376999,
+      "learning_rate": 0.00018090169943749476,
+      "loss": 0.7339,
+      "step": 95
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.25346460938453674,
+      "learning_rate": 0.00018046500327053463,
+      "loss": 0.6606,
+      "step": 96
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.29421111941337585,
+      "learning_rate": 0.0001800239104024851,
+      "loss": 0.6698,
+      "step": 97
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.2356603443622589,
+      "learning_rate": 0.00017957844493517213,
+      "loss": 0.6225,
+      "step": 98
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.2331649512052536,
+      "learning_rate": 0.00017912863120934534,
+      "loss": 0.6838,
+      "step": 99
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.23008978366851807,
+      "learning_rate": 0.00017867449380334834,
+      "loss": 0.6303,
+      "step": 100
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.21604081988334656,
+      "learning_rate": 0.00017821605753177562,
+      "loss": 0.4923,
+      "step": 101
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.2283111810684204,
+      "learning_rate": 0.00017775334744411678,
+      "loss": 0.6344,
+      "step": 102
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.2448730617761612,
+      "learning_rate": 0.00017728638882338746,
+      "loss": 0.6789,
+      "step": 103
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.22479577362537384,
+      "learning_rate": 0.00017681520718474823,
+      "loss": 0.5841,
+      "step": 104
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.21999216079711914,
+      "learning_rate": 0.00017633982827411032,
+      "loss": 0.6402,
+      "step": 105
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 0.2296195775270462,
+      "learning_rate": 0.00017586027806672857,
+      "loss": 0.6148,
+      "step": 106
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.22455835342407227,
+      "learning_rate": 0.00017537658276578247,
+      "loss": 0.6418,
+      "step": 107
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.23929741978645325,
+      "learning_rate": 0.00017488876880094413,
+      "loss": 0.5459,
+      "step": 108
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.229934424161911,
+      "learning_rate": 0.00017439686282693436,
+      "loss": 0.6049,
+      "step": 109
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.24801373481750488,
+      "learning_rate": 0.00017390089172206592,
+      "loss": 0.7663,
+      "step": 110
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.2523267865180969,
+      "learning_rate": 0.00017340088258677522,
+      "loss": 0.6642,
+      "step": 111
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 0.22252008318901062,
+      "learning_rate": 0.00017289686274214118,
+      "loss": 0.6325,
+      "step": 112
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.27619731426239014,
+      "learning_rate": 0.0001723888597283926,
+      "loss": 0.655,
+      "step": 113
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.25090277194976807,
+      "learning_rate": 0.00017187690130340328,
+      "loss": 0.6251,
+      "step": 114
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 0.2236175239086151,
+      "learning_rate": 0.00017136101544117525,
+      "loss": 0.5959,
+      "step": 115
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.21940240263938904,
+      "learning_rate": 0.00017084123033031024,
+      "loss": 0.6028,
+      "step": 116
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.2741597294807434,
+      "learning_rate": 0.00017031757437246947,
+      "loss": 0.4947,
+      "step": 117
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 0.23343035578727722,
+      "learning_rate": 0.00016979007618082175,
+      "loss": 0.4415,
+      "step": 118
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.2116251140832901,
+      "learning_rate": 0.00016925876457848,
+      "loss": 0.5283,
+      "step": 119
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.25621068477630615,
+      "learning_rate": 0.00016872366859692627,
+      "loss": 0.7334,
+      "step": 120
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 0.22588759660720825,
+      "learning_rate": 0.00016818481747442554,
+      "loss": 0.6514,
+      "step": 121
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.24995574355125427,
+      "learning_rate": 0.00016764224065442796,
+      "loss": 0.5632,
+      "step": 122
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.2382134348154068,
+      "learning_rate": 0.00016709596778396026,
+      "loss": 0.5954,
+      "step": 123
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.25166618824005127,
+      "learning_rate": 0.00016654602871200546,
+      "loss": 0.6092,
+      "step": 124
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.23028074204921722,
+      "learning_rate": 0.0001659924534878723,
+      "loss": 0.576,
+      "step": 125
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.2527604103088379,
+      "learning_rate": 0.00016543527235955282,
+      "loss": 0.5985,
+      "step": 126
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.25870686769485474,
+      "learning_rate": 0.00016487451577207018,
+      "loss": 0.5767,
+      "step": 127
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.24936434626579285,
+      "learning_rate": 0.0001643102143658145,
+      "loss": 0.7494,
+      "step": 128
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.25158020853996277,
+      "learning_rate": 0.000163742398974869,
+      "loss": 0.578,
+      "step": 129
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.22290346026420593,
+      "learning_rate": 0.0001631711006253251,
+      "loss": 0.6489,
+      "step": 130
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.26421919465065,
+      "learning_rate": 0.00016259635053358717,
+      "loss": 0.5669,
+      "step": 131
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.2573990225791931,
+      "learning_rate": 0.0001620181801046667,
+      "loss": 0.6405,
+      "step": 132
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.19883453845977783,
+      "learning_rate": 0.00016143662093046638,
+      "loss": 0.4731,
+      "step": 133
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.24181459844112396,
+      "learning_rate": 0.00016085170478805395,
+      "loss": 0.6819,
+      "step": 134
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2319963425397873,
+      "learning_rate": 0.00016026346363792567,
+      "loss": 0.6373,
+      "step": 135
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2552095055580139,
+      "learning_rate": 0.0001596719296222601,
+      "loss": 0.7151,
+      "step": 136
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.27191445231437683,
+      "learning_rate": 0.00015907713506316192,
+      "loss": 0.611,
+      "step": 137
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.26981061697006226,
+      "learning_rate": 0.0001584791124608955,
+      "loss": 0.5923,
+      "step": 138
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23907916247844696,
+      "learning_rate": 0.00015787789449210938,
+      "loss": 0.5655,
+      "step": 139
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23364509642124176,
+      "learning_rate": 0.00015727351400805052,
+      "loss": 0.6019,
+      "step": 140
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 0.23096178472042084,
+      "learning_rate": 0.0001566660040327695,
+      "loss": 0.602,
+      "step": 141
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.2656322717666626,
+      "learning_rate": 0.0001560553977613158,
+      "loss": 0.6195,
+      "step": 142
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.23051652312278748,
+      "learning_rate": 0.00015544172855792423,
+      "loss": 0.6132,
+      "step": 143
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 0.2443581074476242,
+      "learning_rate": 0.00015482502995419167,
+      "loss": 0.6144,
+      "step": 144
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.2075706273317337,
+      "learning_rate": 0.00015420533564724495,
+      "loss": 0.5556,
+      "step": 145
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.23596633970737457,
+      "learning_rate": 0.00015358267949789966,
+      "loss": 0.5926,
+      "step": 146
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.25716009736061096,
+      "learning_rate": 0.0001529570955288099,
+      "loss": 0.6569,
+      "step": 147
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.21337899565696716,
+      "learning_rate": 0.0001523286179226091,
+      "loss": 0.5873,
+      "step": 148
+    },
+    {
+      "epoch": 1.02,
+      "eval_loss": 0.6392110586166382,
+      "eval_runtime": 57.9435,
+      "eval_samples_per_second": 8.974,
+      "eval_steps_per_second": 1.122,
+      "step": 148
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.20899337530136108,
+      "learning_rate": 0.00015169728102004256,
+      "loss": 0.5699,
+      "step": 149
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.2038021683692932,
+      "learning_rate": 0.0001510631193180907,
+      "loss": 0.6119,
+      "step": 150
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.19867417216300964,
+      "learning_rate": 0.00015042616746808435,
+      "loss": 0.5437,
+      "step": 151
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 0.22234468162059784,
+      "learning_rate": 0.00014978646027381123,
+      "loss": 0.461,
+      "step": 152
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 0.2240789234638214,
+      "learning_rate": 0.00014914403268961426,
+      "loss": 0.5873,
+      "step": 153
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.1992528736591339,
+      "learning_rate": 0.00014849891981848158,
+      "loss": 0.4627,
+      "step": 154
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 0.23049698770046234,
+      "learning_rate": 0.00014785115691012864,
+      "loss": 0.439,
+      "step": 155
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 0.22458627820014954,
+      "learning_rate": 0.00014720077935907196,
+      "loss": 0.5893,
+      "step": 156
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.21547600626945496,
+      "learning_rate": 0.00014654782270269497,
+      "loss": 0.6106,
+      "step": 157
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 0.21248772740364075,
+      "learning_rate": 0.00014589232261930674,
+      "loss": 0.5599,
+      "step": 158
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 0.22486011683940887,
+      "learning_rate": 0.0001452343149261919,
+      "loss": 0.5421,
+      "step": 159
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.2963809072971344,
+      "learning_rate": 0.00014457383557765386,
+      "loss": 0.5321,
+      "step": 160
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 0.2344168871641159,
+      "learning_rate": 0.0001439109206630501,
+      "loss": 0.5582,
+      "step": 161
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 0.21608802676200867,
+      "learning_rate": 0.0001432456064048204,
+      "loss": 0.5249,
+      "step": 162
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.3204934895038605,
+      "learning_rate": 0.00014257792915650728,
+      "loss": 0.5718,
+      "step": 163
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 0.2391967922449112,
+      "learning_rate": 0.00014190792540076986,
+      "loss": 0.4687,
+      "step": 164
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.20837116241455078,
+      "learning_rate": 0.00014123563174739037,
+      "loss": 0.437,
+      "step": 165
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.25774210691452026,
+      "learning_rate": 0.0001405610849312736,
+      "loss": 0.6387,
+      "step": 166
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 0.23517344892024994,
+      "learning_rate": 0.00013988432181043982,
+      "loss": 0.5234,
+      "step": 167
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 0.24470289051532745,
+      "learning_rate": 0.00013920537936401077,
+      "loss": 0.5389,
+      "step": 168
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 0.289318323135376,
+      "learning_rate": 0.000138524294690189,
+      "loss": 0.5869,
+      "step": 169
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 0.2550831437110901,
+      "learning_rate": 0.00013784110500423104,
+      "loss": 0.5825,
+      "step": 170
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.24041080474853516,
+      "learning_rate": 0.00013715584763641345,
+      "loss": 0.6241,
+      "step": 171
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.25661879777908325,
+      "learning_rate": 0.00013646856002999354,
+      "loss": 0.5429,
+      "step": 172
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 0.26754602789878845,
+      "learning_rate": 0.00013577927973916306,
+      "loss": 0.6131,
+      "step": 173
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.22897249460220337,
+      "learning_rate": 0.00013508804442699648,
+      "loss": 0.4834,
+      "step": 174
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.22126823663711548,
+      "learning_rate": 0.00013439489186339282,
+      "loss": 0.4735,
+      "step": 175
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 0.2490013688802719,
+      "learning_rate": 0.00013369985992301198,
+      "loss": 0.5248,
+      "step": 176
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.29270628094673157,
+      "learning_rate": 0.00013300298658320517,
+      "loss": 0.643,
+      "step": 177
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.21294419467449188,
+      "learning_rate": 0.00013230430992193973,
+      "loss": 0.4513,
+      "step": 178
+    },
+    {
+      "epoch": 1.21,
+      "grad_norm": 0.20743757486343384,
+      "learning_rate": 0.00013160386811571876,
+      "loss": 0.5091,
+      "step": 179
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.2378052920103073,
+      "learning_rate": 0.00013090169943749476,
+      "loss": 0.5357,
+      "step": 180
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 0.24781295657157898,
+      "learning_rate": 0.00013019784225457855,
+      "loss": 0.5801,
+      "step": 181
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 0.23349957168102264,
+      "learning_rate": 0.00012949233502654284,
+      "loss": 0.5168,
+      "step": 182
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.2096189707517624,
+      "learning_rate": 0.00012878521630312078,
+      "loss": 0.4809,
+      "step": 183
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.21930260956287384,
+      "learning_rate": 0.0001280765247220993,
+      "loss": 0.45,
+      "step": 184
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.28573113679885864,
+      "learning_rate": 0.0001273662990072083,
+      "loss": 0.6648,
+      "step": 185
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.23740755021572113,
+      "learning_rate": 0.00012665457796600443,
+      "loss": 0.5403,
+      "step": 186
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 0.23442810773849487,
+      "learning_rate": 0.0001259414004877507,
+      "loss": 0.5332,
+      "step": 187
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 0.22515830397605896,
+      "learning_rate": 0.00012522680554129156,
+      "loss": 0.4567,
+      "step": 188
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.23571448028087616,
+      "learning_rate": 0.00012451083217292357,
+      "loss": 0.4068,
+      "step": 189
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 0.1914808303117752,
+      "learning_rate": 0.00012379351950426187,
+      "loss": 0.4618,
+      "step": 190
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 0.2481367439031601,
+      "learning_rate": 0.0001230749067301025,
+      "loss": 0.4055,
+      "step": 191
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.22704686224460602,
+      "learning_rate": 0.00012235503311628073,
+      "loss": 0.4984,
+      "step": 192
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 0.2515145242214203,
+      "learning_rate": 0.00012163393799752565,
+      "loss": 0.5463,
+      "step": 193
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 0.23563461005687714,
+      "learning_rate": 0.00012091166077531075,
+      "loss": 0.5143,
+      "step": 194
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.22308993339538574,
+      "learning_rate": 0.00012018824091570103,
+      "loss": 0.5242,
+      "step": 195
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 0.2774996757507324,
+      "learning_rate": 0.00011946371794719656,
+      "loss": 0.5629,
+      "step": 196
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.20966477692127228,
+      "learning_rate": 0.00011873813145857249,
+      "loss": 0.4812,
+      "step": 197
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.2643747925758362,
+      "learning_rate": 0.00011801152109671595,
+      "loss": 0.4194,
+      "step": 198
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 0.23725077509880066,
+      "learning_rate": 0.00011728392656445981,
+      "loss": 0.5517,
+      "step": 199
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.25400522351264954,
+      "learning_rate": 0.000116555387618413,
+      "loss": 0.5144,
+      "step": 200
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.24393105506896973,
+      "learning_rate": 0.00011582594406678839,
+      "loss": 0.5923,
+      "step": 201
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 0.25045904517173767,
+      "learning_rate": 0.00011509563576722753,
+      "loss": 0.5662,
+      "step": 202
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.20616577565670013,
+      "learning_rate": 0.00011436450262462284,
+      "loss": 0.4704,
+      "step": 203
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.2518722712993622,
+      "learning_rate": 0.00011363258458893699,
+      "loss": 0.5764,
+      "step": 204
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 0.2739958167076111,
+      "learning_rate": 0.00011289992165302035,
+      "loss": 0.5376,
+      "step": 205
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.2688806354999542,
+      "learning_rate": 0.00011216655385042525,
+      "loss": 0.5916,
+      "step": 206
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.22472062706947327,
+      "learning_rate": 0.00011143252125321892,
+      "loss": 0.4668,
+      "step": 207
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 0.2582922577857971,
+      "learning_rate": 0.00011069786396979367,
+      "loss": 0.6268,
+      "step": 208
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.2291630357503891,
+      "learning_rate": 0.0001099626221426754,
+      "loss": 0.498,
+      "step": 209
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.24377630650997162,
+      "learning_rate": 0.00010922683594633021,
+      "loss": 0.5157,
+      "step": 210
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 0.2518634498119354,
+      "learning_rate": 0.00010849054558496905,
+      "loss": 0.4786,
+      "step": 211
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.23311232030391693,
+      "learning_rate": 0.00010775379129035116,
+      "loss": 0.4653,
+      "step": 212
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.3206501007080078,
+      "learning_rate": 0.00010701661331958553,
+      "loss": 0.4863,
+      "step": 213
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.23678985238075256,
+      "learning_rate": 0.00010627905195293135,
+      "loss": 0.5854,
+      "step": 214
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 0.2651495933532715,
+      "learning_rate": 0.000105541147491597,
+      "loss": 0.4881,
+      "step": 215
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 0.2638210952281952,
+      "learning_rate": 0.00010480294025553798,
+      "loss": 0.4984,
+      "step": 216
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 0.28642264008522034,
+      "learning_rate": 0.00010406447058125368,
+      "loss": 0.6304,
+      "step": 217
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.29784247279167175,
+      "learning_rate": 0.0001033257788195835,
+      "loss": 0.6045,
+      "step": 218
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 0.28990456461906433,
+      "learning_rate": 0.0001025869053335019,
+      "loss": 0.6472,
+      "step": 219
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 0.2643013894557953,
+      "learning_rate": 0.00010184789049591299,
+      "loss": 0.5676,
+      "step": 220
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.23880311846733093,
+      "learning_rate": 0.0001011087746874445,
+      "loss": 0.4708,
+      "step": 221
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 0.2558128833770752,
+      "learning_rate": 0.00010036959829424131,
+      "loss": 0.5367,
+      "step": 222
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 0.2300223559141159,
+      "learning_rate": 9.96304017057587e-05,
+      "loss": 0.4626,
+      "step": 223
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.2416905015707016,
+      "learning_rate": 9.889122531255552e-05,
+      "loss": 0.4464,
+      "step": 224
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 0.26283279061317444,
+      "learning_rate": 9.815210950408704e-05,
+      "loss": 0.6352,
+      "step": 225
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 0.24686123430728912,
+      "learning_rate": 9.741309466649813e-05,
+      "loss": 0.5295,
+      "step": 226
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 0.2653113305568695,
+      "learning_rate": 9.667422118041651e-05,
+      "loss": 0.4808,
+      "step": 227
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 0.26245051622390747,
+      "learning_rate": 9.593552941874635e-05,
+      "loss": 0.542,
+      "step": 228
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 0.29881128668785095,
+      "learning_rate": 9.519705974446207e-05,
+      "loss": 0.6013,
+      "step": 229
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.2243887037038803,
+      "learning_rate": 9.4458852508403e-05,
+      "loss": 0.474,
+      "step": 230
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 0.2504364252090454,
+      "learning_rate": 9.372094804706867e-05,
+      "loss": 0.472,
+      "step": 231
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.27919840812683105,
+      "learning_rate": 9.298338668041451e-05,
+      "loss": 0.4604,
+      "step": 232
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.23338647186756134,
+      "learning_rate": 9.224620870964886e-05,
+      "loss": 0.4342,
+      "step": 233
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 0.2783346474170685,
+      "learning_rate": 9.150945441503093e-05,
+      "loss": 0.5666,
+      "step": 234
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.2518472373485565,
+      "learning_rate": 9.077316405366981e-05,
+      "loss": 0.4972,
+      "step": 235
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.2644898295402527,
+      "learning_rate": 9.00373778573246e-05,
+      "loss": 0.4328,
+      "step": 236
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 0.34149104356765747,
+      "learning_rate": 8.930213603020638e-05,
+      "loss": 0.6514,
+      "step": 237
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.2584168612957001,
+      "learning_rate": 8.85674787467811e-05,
+      "loss": 0.5692,
+      "step": 238
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.22137361764907837,
+      "learning_rate": 8.783344614957477e-05,
+      "loss": 0.5334,
+      "step": 239
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 0.26125484704971313,
+      "learning_rate": 8.710007834697969e-05,
+      "loss": 0.5633,
+      "step": 240
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.263741672039032,
+      "learning_rate": 8.636741541106299e-05,
+      "loss": 0.525,
+      "step": 241
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.2840401232242584,
+      "learning_rate": 8.563549737537719e-05,
+      "loss": 0.5634,
+      "step": 242
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 0.25339609384536743,
+      "learning_rate": 8.490436423277248e-05,
+      "loss": 0.5634,
+      "step": 243
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.26044854521751404,
+      "learning_rate": 8.417405593321163e-05,
+      "loss": 0.5488,
+      "step": 244
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.23519276082515717,
+      "learning_rate": 8.344461238158699e-05,
+      "loss": 0.4145,
+      "step": 245
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 0.2189461588859558,
+      "learning_rate": 8.271607343554021e-05,
+      "loss": 0.518,
+      "step": 246
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 0.2877447009086609,
+      "learning_rate": 8.198847890328406e-05,
+      "loss": 0.4556,
+      "step": 247
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 0.2689398229122162,
+      "learning_rate": 8.126186854142752e-05,
+      "loss": 0.4426,
+      "step": 248
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 0.25999996066093445,
+      "learning_rate": 8.053628205280347e-05,
+      "loss": 0.4472,
+      "step": 249
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.2660202980041504,
+      "learning_rate": 7.9811759084299e-05,
+      "loss": 0.4789,
+      "step": 250
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 0.2570638060569763,
+      "learning_rate": 7.908833922468927e-05,
+      "loss": 0.4245,
+      "step": 251
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 0.22138474881649017,
+      "learning_rate": 7.836606200247436e-05,
+      "loss": 0.4377,
+      "step": 252
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.2643532454967499,
+      "learning_rate": 7.764496688371929e-05,
+      "loss": 0.5689,
+      "step": 253
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 0.3166361153125763,
+      "learning_rate": 7.692509326989753e-05,
+      "loss": 0.559,
+      "step": 254
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 0.26772719621658325,
+      "learning_rate": 7.620648049573815e-05,
+      "loss": 0.5051,
+      "step": 255
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 0.31207817792892456,
+      "learning_rate": 7.548916782707642e-05,
+      "loss": 0.5906,
+      "step": 256
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.3109002113342285,
+      "learning_rate": 7.477319445870845e-05,
+      "loss": 0.5926,
+      "step": 257
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.24690654873847961,
+      "learning_rate": 7.405859951224933e-05,
+      "loss": 0.5132,
+      "step": 258
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.2552764117717743,
+      "learning_rate": 7.33454220339956e-05,
+      "loss": 0.6071,
+      "step": 259
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 0.22326481342315674,
+      "learning_rate": 7.263370099279172e-05,
+      "loss": 0.5135,
+      "step": 260
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 0.22871443629264832,
+      "learning_rate": 7.192347527790073e-05,
+      "loss": 0.4972,
+      "step": 261
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 0.25344693660736084,
+      "learning_rate": 7.121478369687926e-05,
+      "loss": 0.4668,
+      "step": 262
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 0.24425604939460754,
+      "learning_rate": 7.050766497345714e-05,
+      "loss": 0.5499,
+      "step": 263
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.24569104611873627,
+      "learning_rate": 6.980215774542147e-05,
+      "loss": 0.5668,
+      "step": 264
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.20475324988365173,
+      "learning_rate": 6.909830056250527e-05,
+      "loss": 0.5205,
+      "step": 265
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 0.2739812731742859,
+      "learning_rate": 6.839613188428126e-05,
+      "loss": 0.4883,
+      "step": 266
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.24493646621704102,
+      "learning_rate": 6.769569007806027e-05,
+      "loss": 0.5187,
+      "step": 267
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.25336140394210815,
+      "learning_rate": 6.699701341679488e-05,
+      "loss": 0.5467,
+      "step": 268
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 0.3002632260322571,
+      "learning_rate": 6.630014007698807e-05,
+      "loss": 0.613,
+      "step": 269
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.23223187029361725,
+      "learning_rate": 6.560510813660719e-05,
+      "loss": 0.4893,
+      "step": 270
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.22384615242481232,
+      "learning_rate": 6.491195557300353e-05,
+      "loss": 0.4206,
+      "step": 271
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 0.2999289333820343,
+      "learning_rate": 6.422072026083697e-05,
+      "loss": 0.5848,
+      "step": 272
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.230120450258255,
+      "learning_rate": 6.35314399700065e-05,
+      "loss": 0.4872,
+      "step": 273
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.2404165416955948,
+      "learning_rate": 6.284415236358653e-05,
+      "loss": 0.5125,
+      "step": 274
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 0.2641153037548065,
+      "learning_rate": 6.215889499576898e-05,
+      "loss": 0.5132,
+      "step": 275
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.2617990970611572,
+      "learning_rate": 6.147570530981099e-05,
+      "loss": 0.4281,
+      "step": 276
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.2755200266838074,
+      "learning_rate": 6.0794620635989244e-05,
+      "loss": 0.4031,
+      "step": 277
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 0.32721757888793945,
+      "learning_rate": 6.011567818956021e-05,
+      "loss": 0.4956,
+      "step": 278
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.3732713758945465,
+      "learning_rate": 5.943891506872645e-05,
+      "loss": 0.5509,
+      "step": 279
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 0.27332013845443726,
+      "learning_rate": 5.876436825260967e-05,
+      "loss": 0.5633,
+      "step": 280
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 0.23139792680740356,
+      "learning_rate": 5.809207459923016e-05,
+      "loss": 0.3776,
+      "step": 281
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.29461902379989624,
+      "learning_rate": 5.7422070843492734e-05,
+      "loss": 0.438,
+      "step": 282
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 0.27846580743789673,
+      "learning_rate": 5.675439359517962e-05,
+      "loss": 0.4784,
+      "step": 283
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 0.2910459637641907,
+      "learning_rate": 5.608907933694994e-05,
+      "loss": 0.5903,
+      "step": 284
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.2746853828430176,
+      "learning_rate": 5.542616442234618e-05,
+      "loss": 0.4604,
+      "step": 285
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 0.2670218348503113,
+      "learning_rate": 5.476568507380815e-05,
+      "loss": 0.4851,
+      "step": 286
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 0.289640873670578,
+      "learning_rate": 5.410767738069328e-05,
+      "loss": 0.4749,
+      "step": 287
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.2754352390766144,
+      "learning_rate": 5.345217729730501e-05,
+      "loss": 0.5822,
+      "step": 288
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 0.29798761010169983,
+      "learning_rate": 5.279922064092808e-05,
+      "loss": 0.5244,
+      "step": 289
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 0.2856166660785675,
+      "learning_rate": 5.214884308987136e-05,
+      "loss": 0.5525,
+      "step": 290
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 435,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 145,
+  "total_flos": 1.9525129671278592e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-290/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f4e74f968ef592cbe5e646087e575fbeada43b9dc36d459d99d6a9ebc528b7
+size 5560

checkpoint-435/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: HuggingFaceTB/cosmo-1b
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.1.dev0

checkpoint-435/adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/cosmo-1b",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-435/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:359774f6b53f026ff6ce012c633ef131eb058dac10ad79492ade359d7303cf4c
+size 144748392

checkpoint-435/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a01a74a078b74a1a44b1e41ecc2c3e3ea0287ce5a785afe95be90b285e8b7fb4
+size 72887380

checkpoint-435/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a63f3d4ca1eb6cd7c0cdbb9b50e83d2608862f542c93e95054dd1a0d4d70f4
+size 14244

checkpoint-435/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a301035e0f353aca3f770342289bdb6e15ca0abb2f650431eca332188b0898af
+size 1064

checkpoint-435/trainer_state.json

@@ -0,0 +1,3082 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9518900343642613,
+  "eval_steps": 500,
+  "global_step": 435,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.37809523940086365,
+      "learning_rate": 2e-05,
+      "loss": 1.7814,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.366623193025589,
+      "learning_rate": 4e-05,
+      "loss": 1.69,
+      "step": 2
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.3546508848667145,
+      "learning_rate": 6e-05,
+      "loss": 1.7151,
+      "step": 3
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.36381417512893677,
+      "learning_rate": 8e-05,
+      "loss": 1.6609,
+      "step": 4
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.4049898684024811,
+      "learning_rate": 0.0001,
+      "loss": 1.752,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.4318874478340149,
+      "learning_rate": 0.00012,
+      "loss": 1.747,
+      "step": 6
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.39768990874290466,
+      "learning_rate": 0.00014,
+      "loss": 1.4432,
+      "step": 7
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.5056629180908203,
+      "learning_rate": 0.00016,
+      "loss": 1.7657,
+      "step": 8
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.4276069104671478,
+      "learning_rate": 0.00018,
+      "loss": 1.4221,
+      "step": 9
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 0.433552622795105,
+      "learning_rate": 0.0002,
+      "loss": 1.3117,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.42256900668144226,
+      "learning_rate": 0.0001999972679420179,
+      "loss": 1.2687,
+      "step": 11
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.3066452145576477,
+      "learning_rate": 0.00019998907191735434,
+      "loss": 1.0398,
+      "step": 12
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 0.3393724858760834,
+      "learning_rate": 0.00019997541237384966,
+      "loss": 1.0063,
+      "step": 13
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.49836328625679016,
+      "learning_rate": 0.00019995629005787713,
+      "loss": 1.2251,
+      "step": 14
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.3539387285709381,
+      "learning_rate": 0.0001999317060143023,
+      "loss": 0.9454,
+      "step": 15
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 0.35584160685539246,
+      "learning_rate": 0.0001999016615864258,
+      "loss": 1.0105,
+      "step": 16
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.33191052079200745,
+      "learning_rate": 0.00019986615841591002,
+      "loss": 1.1322,
+      "step": 17
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.3499913513660431,
+      "learning_rate": 0.00019982519844268933,
+      "loss": 0.8495,
+      "step": 18
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 0.2582085132598877,
+      "learning_rate": 0.0001997787839048642,
+      "loss": 0.9725,
+      "step": 19
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.23511624336242676,
+      "learning_rate": 0.00019972691733857883,
+      "loss": 0.9233,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.273019939661026,
+      "learning_rate": 0.00019966960157788248,
+      "loss": 1.0366,
+      "step": 21
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.283527135848999,
+      "learning_rate": 0.0001996068397545748,
+      "loss": 0.9944,
+      "step": 22
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24311360716819763,
+      "learning_rate": 0.00019953863529803466,
+      "loss": 1.0308,
+      "step": 23
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.24368223547935486,
+      "learning_rate": 0.00019946499193503262,
+      "loss": 1.0051,
+      "step": 24
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 0.2269805669784546,
+      "learning_rate": 0.0001993859136895274,
+      "loss": 0.835,
+      "step": 25
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.21915565431118011,
+      "learning_rate": 0.00019930140488244602,
+      "loss": 0.9997,
+      "step": 26
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.21885964274406433,
+      "learning_rate": 0.0001992114701314478,
+      "loss": 1.0013,
+      "step": 27
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.22426557540893555,
+      "learning_rate": 0.00019911611435067172,
+      "loss": 0.8633,
+      "step": 28
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.21587449312210083,
+      "learning_rate": 0.0001990153427504683,
+      "loss": 0.8479,
+      "step": 29
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.24985891580581665,
+      "learning_rate": 0.0001989091608371146,
+      "loss": 0.8936,
+      "step": 30
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.19733141362667084,
+      "learning_rate": 0.0001987975744125135,
+      "loss": 0.7743,
+      "step": 31
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.1997266411781311,
+      "learning_rate": 0.00019868058957387663,
+      "loss": 0.9078,
+      "step": 32
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.24114945530891418,
+      "learning_rate": 0.00019855821271339125,
+      "loss": 1.0413,
+      "step": 33
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.2427120953798294,
+      "learning_rate": 0.00019843045051787096,
+      "loss": 0.8308,
+      "step": 34
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.1968858540058136,
+      "learning_rate": 0.0001982973099683902,
+      "loss": 0.818,
+      "step": 35
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2783421576023102,
+      "learning_rate": 0.00019815879833990304,
+      "loss": 0.8504,
+      "step": 36
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.23332269489765167,
+      "learning_rate": 0.00019801492320084546,
+      "loss": 0.9826,
+      "step": 37
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.22864645719528198,
+      "learning_rate": 0.00019786569241272197,
+      "loss": 0.8705,
+      "step": 38
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20620585978031158,
+      "learning_rate": 0.00019771111412967583,
+      "loss": 0.8264,
+      "step": 39
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.20590244233608246,
+      "learning_rate": 0.00019755119679804367,
+      "loss": 0.8455,
+      "step": 40
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.20340456068515778,
+      "learning_rate": 0.00019738594915589397,
+      "loss": 0.871,
+      "step": 41
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 0.2383277714252472,
+      "learning_rate": 0.0001972153802325495,
+      "loss": 0.6833,
+      "step": 42
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.21479888260364532,
+      "learning_rate": 0.00019703949934809408,
+      "loss": 0.8355,
+      "step": 43
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.23538872599601746,
+      "learning_rate": 0.0001968583161128631,
+      "loss": 0.8479,
+      "step": 44
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 0.2260449230670929,
+      "learning_rate": 0.00019667184042691875,
+      "loss": 0.779,
+      "step": 45
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.20232993364334106,
+      "learning_rate": 0.0001964800824795087,
+      "loss": 0.7293,
+      "step": 46
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.19780154526233673,
+      "learning_rate": 0.00019628305274850956,
+      "loss": 0.7709,
+      "step": 47
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 0.24822202324867249,
+      "learning_rate": 0.00019608076199985433,
+      "loss": 0.8885,
+      "step": 48
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.20940056443214417,
+      "learning_rate": 0.0001958732212869441,
+      "loss": 0.6916,
+      "step": 49
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.18944858014583588,
+      "learning_rate": 0.0001956604419500441,
+      "loss": 0.7326,
+      "step": 50
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.201398104429245,
+      "learning_rate": 0.00019544243561566403,
+      "loss": 0.6542,
+      "step": 51
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.22385160624980927,
+      "learning_rate": 0.00019521921419592283,
+      "loss": 0.8036,
+      "step": 52
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.19502398371696472,
+      "learning_rate": 0.0001949907898878977,
+      "loss": 0.7579,
+      "step": 53
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 0.21633867919445038,
+      "learning_rate": 0.00019475717517295778,
+      "loss": 0.7803,
+      "step": 54
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.24109330773353577,
+      "learning_rate": 0.00019451838281608197,
+      "loss": 0.7745,
+      "step": 55
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.20285621285438538,
+      "learning_rate": 0.00019427442586516155,
+      "loss": 0.7306,
+      "step": 56
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 0.23838144540786743,
+      "learning_rate": 0.00019402531765028722,
+      "loss": 0.7606,
+      "step": 57
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.25280559062957764,
+      "learning_rate": 0.00019377107178302074,
+      "loss": 0.7928,
+      "step": 58
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.24614648520946503,
+      "learning_rate": 0.00019351170215565114,
+      "loss": 0.739,
+      "step": 59
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.20391589403152466,
+      "learning_rate": 0.00019324722294043558,
+      "loss": 0.7636,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.23704086244106293,
+      "learning_rate": 0.00019297764858882514,
+      "loss": 0.6919,
+      "step": 61
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22057901322841644,
+      "learning_rate": 0.00019270299383067498,
+      "loss": 0.6035,
+      "step": 62
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 0.22797846794128418,
+      "learning_rate": 0.0001924232736734396,
+      "loss": 0.7087,
+      "step": 63
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.22181542217731476,
+      "learning_rate": 0.00019213850340135276,
+      "loss": 0.8088,
+      "step": 64
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.2096872180700302,
+      "learning_rate": 0.00019184869857459232,
+      "loss": 0.7073,
+      "step": 65
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.192199245095253,
+      "learning_rate": 0.00019155387502843013,
+      "loss": 0.6605,
+      "step": 66
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.2242266982793808,
+      "learning_rate": 0.00019125404887236663,
+      "loss": 0.7287,
+      "step": 67
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.24139907956123352,
+      "learning_rate": 0.00019094923648925067,
+      "loss": 0.6611,
+      "step": 68
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 0.22859904170036316,
+      "learning_rate": 0.00019063945453438432,
+      "loss": 0.6189,
+      "step": 69
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.27671098709106445,
+      "learning_rate": 0.0001903247199346129,
+      "loss": 0.8595,
+      "step": 70
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.23342834413051605,
+      "learning_rate": 0.00019000504988739986,
+      "loss": 0.6718,
+      "step": 71
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 0.2517896890640259,
+      "learning_rate": 0.00018968046185988732,
+      "loss": 0.722,
+      "step": 72
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.25217679142951965,
+      "learning_rate": 0.00018935097358794144,
+      "loss": 0.824,
+      "step": 73
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 0.2682301998138428,
+      "learning_rate": 0.00018901660307518354,
+      "loss": 0.7837,
+      "step": 74
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.28926676511764526,
+      "learning_rate": 0.0001886773685920062,
+      "loss": 0.7364,
+      "step": 75
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.25249186158180237,
+      "learning_rate": 0.00018833328867457497,
+      "loss": 0.8566,
+      "step": 76
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 0.21752771735191345,
+      "learning_rate": 0.0001879843821238155,
+      "loss": 0.7077,
+      "step": 77
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.22346794605255127,
+      "learning_rate": 0.00018763066800438636,
+      "loss": 0.6522,
+      "step": 78
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.23172937333583832,
+      "learning_rate": 0.00018727216564363723,
+      "loss": 0.6099,
+      "step": 79
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.20895916223526,
+      "learning_rate": 0.00018690889463055283,
+      "loss": 0.7242,
+      "step": 80
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.26868510246276855,
+      "learning_rate": 0.0001865408748146826,
+      "loss": 0.705,
+      "step": 81
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.243666410446167,
+      "learning_rate": 0.00018616812630505597,
+      "loss": 0.8512,
+      "step": 82
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 0.21445880830287933,
+      "learning_rate": 0.00018579066946908384,
+      "loss": 0.7092,
+      "step": 83
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.22805824875831604,
+      "learning_rate": 0.00018540852493144545,
+      "loss": 0.6757,
+      "step": 84
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.25106164813041687,
+      "learning_rate": 0.00018502171357296144,
+      "loss": 0.7171,
+      "step": 85
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 0.2290162742137909,
+      "learning_rate": 0.000184630256529453,
+      "loss": 0.7053,
+      "step": 86
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.21118655800819397,
+      "learning_rate": 0.00018423417519058694,
+      "loss": 0.5827,
+      "step": 87
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.23761862516403198,
+      "learning_rate": 0.00018383349119870695,
+      "loss": 0.5827,
+      "step": 88
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 0.23689566552639008,
+      "learning_rate": 0.00018342822644765104,
+      "loss": 0.6401,
+      "step": 89
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.26510947942733765,
+      "learning_rate": 0.00018301840308155507,
+      "loss": 0.5963,
+      "step": 90
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.2142990678548813,
+      "learning_rate": 0.0001826040434936431,
+      "loss": 0.6258,
+      "step": 91
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 0.20709159970283508,
+      "learning_rate": 0.00018218517032500344,
+      "loss": 0.5685,
+      "step": 92
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.2485310137271881,
+      "learning_rate": 0.0001817618064633518,
+      "loss": 0.5901,
+      "step": 93
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2432468980550766,
+      "learning_rate": 0.00018133397504178057,
+      "loss": 0.6891,
+      "step": 94
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.2318205088376999,
+      "learning_rate": 0.00018090169943749476,
+      "loss": 0.7339,
+      "step": 95
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.25346460938453674,
+      "learning_rate": 0.00018046500327053463,
+      "loss": 0.6606,
+      "step": 96
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.29421111941337585,
+      "learning_rate": 0.0001800239104024851,
+      "loss": 0.6698,
+      "step": 97
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 0.2356603443622589,
+      "learning_rate": 0.00017957844493517213,
+      "loss": 0.6225,
+      "step": 98
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.2331649512052536,
+      "learning_rate": 0.00017912863120934534,
+      "loss": 0.6838,
+      "step": 99
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.23008978366851807,
+      "learning_rate": 0.00017867449380334834,
+      "loss": 0.6303,
+      "step": 100
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.21604081988334656,
+      "learning_rate": 0.00017821605753177562,
+      "loss": 0.4923,
+      "step": 101
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.2283111810684204,
+      "learning_rate": 0.00017775334744411678,
+      "loss": 0.6344,
+      "step": 102
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.2448730617761612,
+      "learning_rate": 0.00017728638882338746,
+      "loss": 0.6789,
+      "step": 103
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.22479577362537384,
+      "learning_rate": 0.00017681520718474823,
+      "loss": 0.5841,
+      "step": 104
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.21999216079711914,
+      "learning_rate": 0.00017633982827411032,
+      "loss": 0.6402,
+      "step": 105
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 0.2296195775270462,
+      "learning_rate": 0.00017586027806672857,
+      "loss": 0.6148,
+      "step": 106
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.22455835342407227,
+      "learning_rate": 0.00017537658276578247,
+      "loss": 0.6418,
+      "step": 107
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.23929741978645325,
+      "learning_rate": 0.00017488876880094413,
+      "loss": 0.5459,
+      "step": 108
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.229934424161911,
+      "learning_rate": 0.00017439686282693436,
+      "loss": 0.6049,
+      "step": 109
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.24801373481750488,
+      "learning_rate": 0.00017390089172206592,
+      "loss": 0.7663,
+      "step": 110
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.2523267865180969,
+      "learning_rate": 0.00017340088258677522,
+      "loss": 0.6642,
+      "step": 111
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 0.22252008318901062,
+      "learning_rate": 0.00017289686274214118,
+      "loss": 0.6325,
+      "step": 112
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.27619731426239014,
+      "learning_rate": 0.0001723888597283926,
+      "loss": 0.655,
+      "step": 113
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.25090277194976807,
+      "learning_rate": 0.00017187690130340328,
+      "loss": 0.6251,
+      "step": 114
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 0.2236175239086151,
+      "learning_rate": 0.00017136101544117525,
+      "loss": 0.5959,
+      "step": 115
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.21940240263938904,
+      "learning_rate": 0.00017084123033031024,
+      "loss": 0.6028,
+      "step": 116
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.2741597294807434,
+      "learning_rate": 0.00017031757437246947,
+      "loss": 0.4947,
+      "step": 117
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 0.23343035578727722,
+      "learning_rate": 0.00016979007618082175,
+      "loss": 0.4415,
+      "step": 118
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.2116251140832901,
+      "learning_rate": 0.00016925876457848,
+      "loss": 0.5283,
+      "step": 119
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.25621068477630615,
+      "learning_rate": 0.00016872366859692627,
+      "loss": 0.7334,
+      "step": 120
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 0.22588759660720825,
+      "learning_rate": 0.00016818481747442554,
+      "loss": 0.6514,
+      "step": 121
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.24995574355125427,
+      "learning_rate": 0.00016764224065442796,
+      "loss": 0.5632,
+      "step": 122
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.2382134348154068,
+      "learning_rate": 0.00016709596778396026,
+      "loss": 0.5954,
+      "step": 123
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.25166618824005127,
+      "learning_rate": 0.00016654602871200546,
+      "loss": 0.6092,
+      "step": 124
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.23028074204921722,
+      "learning_rate": 0.0001659924534878723,
+      "loss": 0.576,
+      "step": 125
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.2527604103088379,
+      "learning_rate": 0.00016543527235955282,
+      "loss": 0.5985,
+      "step": 126
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 0.25870686769485474,
+      "learning_rate": 0.00016487451577207018,
+      "loss": 0.5767,
+      "step": 127
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.24936434626579285,
+      "learning_rate": 0.0001643102143658145,
+      "loss": 0.7494,
+      "step": 128
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.25158020853996277,
+      "learning_rate": 0.000163742398974869,
+      "loss": 0.578,
+      "step": 129
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.22290346026420593,
+      "learning_rate": 0.0001631711006253251,
+      "loss": 0.6489,
+      "step": 130
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.26421919465065,
+      "learning_rate": 0.00016259635053358717,
+      "loss": 0.5669,
+      "step": 131
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.2573990225791931,
+      "learning_rate": 0.0001620181801046667,
+      "loss": 0.6405,
+      "step": 132
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.19883453845977783,
+      "learning_rate": 0.00016143662093046638,
+      "loss": 0.4731,
+      "step": 133
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.24181459844112396,
+      "learning_rate": 0.00016085170478805395,
+      "loss": 0.6819,
+      "step": 134
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2319963425397873,
+      "learning_rate": 0.00016026346363792567,
+      "loss": 0.6373,
+      "step": 135
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.2552095055580139,
+      "learning_rate": 0.0001596719296222601,
+      "loss": 0.7151,
+      "step": 136
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.27191445231437683,
+      "learning_rate": 0.00015907713506316192,
+      "loss": 0.611,
+      "step": 137
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.26981061697006226,
+      "learning_rate": 0.0001584791124608955,
+      "loss": 0.5923,
+      "step": 138
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23907916247844696,
+      "learning_rate": 0.00015787789449210938,
+      "loss": 0.5655,
+      "step": 139
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.23364509642124176,
+      "learning_rate": 0.00015727351400805052,
+      "loss": 0.6019,
+      "step": 140
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 0.23096178472042084,
+      "learning_rate": 0.0001566660040327695,
+      "loss": 0.602,
+      "step": 141
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.2656322717666626,
+      "learning_rate": 0.0001560553977613158,
+      "loss": 0.6195,
+      "step": 142
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.23051652312278748,
+      "learning_rate": 0.00015544172855792423,
+      "loss": 0.6132,
+      "step": 143
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 0.2443581074476242,
+      "learning_rate": 0.00015482502995419167,
+      "loss": 0.6144,
+      "step": 144
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.2075706273317337,
+      "learning_rate": 0.00015420533564724495,
+      "loss": 0.5556,
+      "step": 145
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.23596633970737457,
+      "learning_rate": 0.00015358267949789966,
+      "loss": 0.5926,
+      "step": 146
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.25716009736061096,
+      "learning_rate": 0.0001529570955288099,
+      "loss": 0.6569,
+      "step": 147
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.21337899565696716,
+      "learning_rate": 0.0001523286179226091,
+      "loss": 0.5873,
+      "step": 148
+    },
+    {
+      "epoch": 1.02,
+      "eval_loss": 0.6392110586166382,
+      "eval_runtime": 57.9435,
+      "eval_samples_per_second": 8.974,
+      "eval_steps_per_second": 1.122,
+      "step": 148
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.20899337530136108,
+      "learning_rate": 0.00015169728102004256,
+      "loss": 0.5699,
+      "step": 149
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 0.2038021683692932,
+      "learning_rate": 0.0001510631193180907,
+      "loss": 0.6119,
+      "step": 150
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.19867417216300964,
+      "learning_rate": 0.00015042616746808435,
+      "loss": 0.5437,
+      "step": 151
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 0.22234468162059784,
+      "learning_rate": 0.00014978646027381123,
+      "loss": 0.461,
+      "step": 152
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 0.2240789234638214,
+      "learning_rate": 0.00014914403268961426,
+      "loss": 0.5873,
+      "step": 153
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.1992528736591339,
+      "learning_rate": 0.00014849891981848158,
+      "loss": 0.4627,
+      "step": 154
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 0.23049698770046234,
+      "learning_rate": 0.00014785115691012864,
+      "loss": 0.439,
+      "step": 155
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 0.22458627820014954,
+      "learning_rate": 0.00014720077935907196,
+      "loss": 0.5893,
+      "step": 156
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.21547600626945496,
+      "learning_rate": 0.00014654782270269497,
+      "loss": 0.6106,
+      "step": 157
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 0.21248772740364075,
+      "learning_rate": 0.00014589232261930674,
+      "loss": 0.5599,
+      "step": 158
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 0.22486011683940887,
+      "learning_rate": 0.0001452343149261919,
+      "loss": 0.5421,
+      "step": 159
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.2963809072971344,
+      "learning_rate": 0.00014457383557765386,
+      "loss": 0.5321,
+      "step": 160
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 0.2344168871641159,
+      "learning_rate": 0.0001439109206630501,
+      "loss": 0.5582,
+      "step": 161
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 0.21608802676200867,
+      "learning_rate": 0.0001432456064048204,
+      "loss": 0.5249,
+      "step": 162
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.3204934895038605,
+      "learning_rate": 0.00014257792915650728,
+      "loss": 0.5718,
+      "step": 163
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 0.2391967922449112,
+      "learning_rate": 0.00014190792540076986,
+      "loss": 0.4687,
+      "step": 164
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.20837116241455078,
+      "learning_rate": 0.00014123563174739037,
+      "loss": 0.437,
+      "step": 165
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.25774210691452026,
+      "learning_rate": 0.0001405610849312736,
+      "loss": 0.6387,
+      "step": 166
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 0.23517344892024994,
+      "learning_rate": 0.00013988432181043982,
+      "loss": 0.5234,
+      "step": 167
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 0.24470289051532745,
+      "learning_rate": 0.00013920537936401077,
+      "loss": 0.5389,
+      "step": 168
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 0.289318323135376,
+      "learning_rate": 0.000138524294690189,
+      "loss": 0.5869,
+      "step": 169
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 0.2550831437110901,
+      "learning_rate": 0.00013784110500423104,
+      "loss": 0.5825,
+      "step": 170
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.24041080474853516,
+      "learning_rate": 0.00013715584763641345,
+      "loss": 0.6241,
+      "step": 171
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.25661879777908325,
+      "learning_rate": 0.00013646856002999354,
+      "loss": 0.5429,
+      "step": 172
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 0.26754602789878845,
+      "learning_rate": 0.00013577927973916306,
+      "loss": 0.6131,
+      "step": 173
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.22897249460220337,
+      "learning_rate": 0.00013508804442699648,
+      "loss": 0.4834,
+      "step": 174
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.22126823663711548,
+      "learning_rate": 0.00013439489186339282,
+      "loss": 0.4735,
+      "step": 175
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 0.2490013688802719,
+      "learning_rate": 0.00013369985992301198,
+      "loss": 0.5248,
+      "step": 176
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.29270628094673157,
+      "learning_rate": 0.00013300298658320517,
+      "loss": 0.643,
+      "step": 177
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.21294419467449188,
+      "learning_rate": 0.00013230430992193973,
+      "loss": 0.4513,
+      "step": 178
+    },
+    {
+      "epoch": 1.21,
+      "grad_norm": 0.20743757486343384,
+      "learning_rate": 0.00013160386811571876,
+      "loss": 0.5091,
+      "step": 179
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.2378052920103073,
+      "learning_rate": 0.00013090169943749476,
+      "loss": 0.5357,
+      "step": 180
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 0.24781295657157898,
+      "learning_rate": 0.00013019784225457855,
+      "loss": 0.5801,
+      "step": 181
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 0.23349957168102264,
+      "learning_rate": 0.00012949233502654284,
+      "loss": 0.5168,
+      "step": 182
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.2096189707517624,
+      "learning_rate": 0.00012878521630312078,
+      "loss": 0.4809,
+      "step": 183
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.21930260956287384,
+      "learning_rate": 0.0001280765247220993,
+      "loss": 0.45,
+      "step": 184
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.28573113679885864,
+      "learning_rate": 0.0001273662990072083,
+      "loss": 0.6648,
+      "step": 185
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.23740755021572113,
+      "learning_rate": 0.00012665457796600443,
+      "loss": 0.5403,
+      "step": 186
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 0.23442810773849487,
+      "learning_rate": 0.0001259414004877507,
+      "loss": 0.5332,
+      "step": 187
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 0.22515830397605896,
+      "learning_rate": 0.00012522680554129156,
+      "loss": 0.4567,
+      "step": 188
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.23571448028087616,
+      "learning_rate": 0.00012451083217292357,
+      "loss": 0.4068,
+      "step": 189
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 0.1914808303117752,
+      "learning_rate": 0.00012379351950426187,
+      "loss": 0.4618,
+      "step": 190
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 0.2481367439031601,
+      "learning_rate": 0.0001230749067301025,
+      "loss": 0.4055,
+      "step": 191
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.22704686224460602,
+      "learning_rate": 0.00012235503311628073,
+      "loss": 0.4984,
+      "step": 192
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 0.2515145242214203,
+      "learning_rate": 0.00012163393799752565,
+      "loss": 0.5463,
+      "step": 193
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 0.23563461005687714,
+      "learning_rate": 0.00012091166077531075,
+      "loss": 0.5143,
+      "step": 194
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.22308993339538574,
+      "learning_rate": 0.00012018824091570103,
+      "loss": 0.5242,
+      "step": 195
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 0.2774996757507324,
+      "learning_rate": 0.00011946371794719656,
+      "loss": 0.5629,
+      "step": 196
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.20966477692127228,
+      "learning_rate": 0.00011873813145857249,
+      "loss": 0.4812,
+      "step": 197
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.2643747925758362,
+      "learning_rate": 0.00011801152109671595,
+      "loss": 0.4194,
+      "step": 198
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 0.23725077509880066,
+      "learning_rate": 0.00011728392656445981,
+      "loss": 0.5517,
+      "step": 199
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.25400522351264954,
+      "learning_rate": 0.000116555387618413,
+      "loss": 0.5144,
+      "step": 200
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.24393105506896973,
+      "learning_rate": 0.00011582594406678839,
+      "loss": 0.5923,
+      "step": 201
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 0.25045904517173767,
+      "learning_rate": 0.00011509563576722753,
+      "loss": 0.5662,
+      "step": 202
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.20616577565670013,
+      "learning_rate": 0.00011436450262462284,
+      "loss": 0.4704,
+      "step": 203
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.2518722712993622,
+      "learning_rate": 0.00011363258458893699,
+      "loss": 0.5764,
+      "step": 204
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 0.2739958167076111,
+      "learning_rate": 0.00011289992165302035,
+      "loss": 0.5376,
+      "step": 205
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.2688806354999542,
+      "learning_rate": 0.00011216655385042525,
+      "loss": 0.5916,
+      "step": 206
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.22472062706947327,
+      "learning_rate": 0.00011143252125321892,
+      "loss": 0.4668,
+      "step": 207
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 0.2582922577857971,
+      "learning_rate": 0.00011069786396979367,
+      "loss": 0.6268,
+      "step": 208
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.2291630357503891,
+      "learning_rate": 0.0001099626221426754,
+      "loss": 0.498,
+      "step": 209
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.24377630650997162,
+      "learning_rate": 0.00010922683594633021,
+      "loss": 0.5157,
+      "step": 210
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 0.2518634498119354,
+      "learning_rate": 0.00010849054558496905,
+      "loss": 0.4786,
+      "step": 211
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.23311232030391693,
+      "learning_rate": 0.00010775379129035116,
+      "loss": 0.4653,
+      "step": 212
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.3206501007080078,
+      "learning_rate": 0.00010701661331958553,
+      "loss": 0.4863,
+      "step": 213
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.23678985238075256,
+      "learning_rate": 0.00010627905195293135,
+      "loss": 0.5854,
+      "step": 214
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 0.2651495933532715,
+      "learning_rate": 0.000105541147491597,
+      "loss": 0.4881,
+      "step": 215
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 0.2638210952281952,
+      "learning_rate": 0.00010480294025553798,
+      "loss": 0.4984,
+      "step": 216
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 0.28642264008522034,
+      "learning_rate": 0.00010406447058125368,
+      "loss": 0.6304,
+      "step": 217
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.29784247279167175,
+      "learning_rate": 0.0001033257788195835,
+      "loss": 0.6045,
+      "step": 218
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 0.28990456461906433,
+      "learning_rate": 0.0001025869053335019,
+      "loss": 0.6472,
+      "step": 219
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 0.2643013894557953,
+      "learning_rate": 0.00010184789049591299,
+      "loss": 0.5676,
+      "step": 220
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.23880311846733093,
+      "learning_rate": 0.0001011087746874445,
+      "loss": 0.4708,
+      "step": 221
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 0.2558128833770752,
+      "learning_rate": 0.00010036959829424131,
+      "loss": 0.5367,
+      "step": 222
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 0.2300223559141159,
+      "learning_rate": 9.96304017057587e-05,
+      "loss": 0.4626,
+      "step": 223
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.2416905015707016,
+      "learning_rate": 9.889122531255552e-05,
+      "loss": 0.4464,
+      "step": 224
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 0.26283279061317444,
+      "learning_rate": 9.815210950408704e-05,
+      "loss": 0.6352,
+      "step": 225
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 0.24686123430728912,
+      "learning_rate": 9.741309466649813e-05,
+      "loss": 0.5295,
+      "step": 226
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 0.2653113305568695,
+      "learning_rate": 9.667422118041651e-05,
+      "loss": 0.4808,
+      "step": 227
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 0.26245051622390747,
+      "learning_rate": 9.593552941874635e-05,
+      "loss": 0.542,
+      "step": 228
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 0.29881128668785095,
+      "learning_rate": 9.519705974446207e-05,
+      "loss": 0.6013,
+      "step": 229
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.2243887037038803,
+      "learning_rate": 9.4458852508403e-05,
+      "loss": 0.474,
+      "step": 230
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 0.2504364252090454,
+      "learning_rate": 9.372094804706867e-05,
+      "loss": 0.472,
+      "step": 231
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.27919840812683105,
+      "learning_rate": 9.298338668041451e-05,
+      "loss": 0.4604,
+      "step": 232
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.23338647186756134,
+      "learning_rate": 9.224620870964886e-05,
+      "loss": 0.4342,
+      "step": 233
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 0.2783346474170685,
+      "learning_rate": 9.150945441503093e-05,
+      "loss": 0.5666,
+      "step": 234
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.2518472373485565,
+      "learning_rate": 9.077316405366981e-05,
+      "loss": 0.4972,
+      "step": 235
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.2644898295402527,
+      "learning_rate": 9.00373778573246e-05,
+      "loss": 0.4328,
+      "step": 236
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 0.34149104356765747,
+      "learning_rate": 8.930213603020638e-05,
+      "loss": 0.6514,
+      "step": 237
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.2584168612957001,
+      "learning_rate": 8.85674787467811e-05,
+      "loss": 0.5692,
+      "step": 238
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.22137361764907837,
+      "learning_rate": 8.783344614957477e-05,
+      "loss": 0.5334,
+      "step": 239
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 0.26125484704971313,
+      "learning_rate": 8.710007834697969e-05,
+      "loss": 0.5633,
+      "step": 240
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.263741672039032,
+      "learning_rate": 8.636741541106299e-05,
+      "loss": 0.525,
+      "step": 241
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.2840401232242584,
+      "learning_rate": 8.563549737537719e-05,
+      "loss": 0.5634,
+      "step": 242
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 0.25339609384536743,
+      "learning_rate": 8.490436423277248e-05,
+      "loss": 0.5634,
+      "step": 243
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.26044854521751404,
+      "learning_rate": 8.417405593321163e-05,
+      "loss": 0.5488,
+      "step": 244
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.23519276082515717,
+      "learning_rate": 8.344461238158699e-05,
+      "loss": 0.4145,
+      "step": 245
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 0.2189461588859558,
+      "learning_rate": 8.271607343554021e-05,
+      "loss": 0.518,
+      "step": 246
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 0.2877447009086609,
+      "learning_rate": 8.198847890328406e-05,
+      "loss": 0.4556,
+      "step": 247
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 0.2689398229122162,
+      "learning_rate": 8.126186854142752e-05,
+      "loss": 0.4426,
+      "step": 248
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 0.25999996066093445,
+      "learning_rate": 8.053628205280347e-05,
+      "loss": 0.4472,
+      "step": 249
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.2660202980041504,
+      "learning_rate": 7.9811759084299e-05,
+      "loss": 0.4789,
+      "step": 250
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 0.2570638060569763,
+      "learning_rate": 7.908833922468927e-05,
+      "loss": 0.4245,
+      "step": 251
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 0.22138474881649017,
+      "learning_rate": 7.836606200247436e-05,
+      "loss": 0.4377,
+      "step": 252
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.2643532454967499,
+      "learning_rate": 7.764496688371929e-05,
+      "loss": 0.5689,
+      "step": 253
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 0.3166361153125763,
+      "learning_rate": 7.692509326989753e-05,
+      "loss": 0.559,
+      "step": 254
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 0.26772719621658325,
+      "learning_rate": 7.620648049573815e-05,
+      "loss": 0.5051,
+      "step": 255
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 0.31207817792892456,
+      "learning_rate": 7.548916782707642e-05,
+      "loss": 0.5906,
+      "step": 256
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.3109002113342285,
+      "learning_rate": 7.477319445870845e-05,
+      "loss": 0.5926,
+      "step": 257
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.24690654873847961,
+      "learning_rate": 7.405859951224933e-05,
+      "loss": 0.5132,
+      "step": 258
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.2552764117717743,
+      "learning_rate": 7.33454220339956e-05,
+      "loss": 0.6071,
+      "step": 259
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 0.22326481342315674,
+      "learning_rate": 7.263370099279172e-05,
+      "loss": 0.5135,
+      "step": 260
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 0.22871443629264832,
+      "learning_rate": 7.192347527790073e-05,
+      "loss": 0.4972,
+      "step": 261
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 0.25344693660736084,
+      "learning_rate": 7.121478369687926e-05,
+      "loss": 0.4668,
+      "step": 262
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 0.24425604939460754,
+      "learning_rate": 7.050766497345714e-05,
+      "loss": 0.5499,
+      "step": 263
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.24569104611873627,
+      "learning_rate": 6.980215774542147e-05,
+      "loss": 0.5668,
+      "step": 264
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.20475324988365173,
+      "learning_rate": 6.909830056250527e-05,
+      "loss": 0.5205,
+      "step": 265
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 0.2739812731742859,
+      "learning_rate": 6.839613188428126e-05,
+      "loss": 0.4883,
+      "step": 266
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.24493646621704102,
+      "learning_rate": 6.769569007806027e-05,
+      "loss": 0.5187,
+      "step": 267
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.25336140394210815,
+      "learning_rate": 6.699701341679488e-05,
+      "loss": 0.5467,
+      "step": 268
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 0.3002632260322571,
+      "learning_rate": 6.630014007698807e-05,
+      "loss": 0.613,
+      "step": 269
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.23223187029361725,
+      "learning_rate": 6.560510813660719e-05,
+      "loss": 0.4893,
+      "step": 270
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.22384615242481232,
+      "learning_rate": 6.491195557300353e-05,
+      "loss": 0.4206,
+      "step": 271
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 0.2999289333820343,
+      "learning_rate": 6.422072026083697e-05,
+      "loss": 0.5848,
+      "step": 272
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.230120450258255,
+      "learning_rate": 6.35314399700065e-05,
+      "loss": 0.4872,
+      "step": 273
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.2404165416955948,
+      "learning_rate": 6.284415236358653e-05,
+      "loss": 0.5125,
+      "step": 274
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 0.2641153037548065,
+      "learning_rate": 6.215889499576898e-05,
+      "loss": 0.5132,
+      "step": 275
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.2617990970611572,
+      "learning_rate": 6.147570530981099e-05,
+      "loss": 0.4281,
+      "step": 276
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.2755200266838074,
+      "learning_rate": 6.0794620635989244e-05,
+      "loss": 0.4031,
+      "step": 277
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 0.32721757888793945,
+      "learning_rate": 6.011567818956021e-05,
+      "loss": 0.4956,
+      "step": 278
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.3732713758945465,
+      "learning_rate": 5.943891506872645e-05,
+      "loss": 0.5509,
+      "step": 279
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 0.27332013845443726,
+      "learning_rate": 5.876436825260967e-05,
+      "loss": 0.5633,
+      "step": 280
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 0.23139792680740356,
+      "learning_rate": 5.809207459923016e-05,
+      "loss": 0.3776,
+      "step": 281
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.29461902379989624,
+      "learning_rate": 5.7422070843492734e-05,
+      "loss": 0.438,
+      "step": 282
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 0.27846580743789673,
+      "learning_rate": 5.675439359517962e-05,
+      "loss": 0.4784,
+      "step": 283
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 0.2910459637641907,
+      "learning_rate": 5.608907933694994e-05,
+      "loss": 0.5903,
+      "step": 284
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.2746853828430176,
+      "learning_rate": 5.542616442234618e-05,
+      "loss": 0.4604,
+      "step": 285
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 0.2670218348503113,
+      "learning_rate": 5.476568507380815e-05,
+      "loss": 0.4851,
+      "step": 286
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 0.289640873670578,
+      "learning_rate": 5.410767738069328e-05,
+      "loss": 0.4749,
+      "step": 287
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.2754352390766144,
+      "learning_rate": 5.345217729730501e-05,
+      "loss": 0.5822,
+      "step": 288
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 0.29798761010169983,
+      "learning_rate": 5.279922064092808e-05,
+      "loss": 0.5244,
+      "step": 289
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 0.2856166660785675,
+      "learning_rate": 5.214884308987136e-05,
+      "loss": 0.5525,
+      "step": 290
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 0.2690298557281494,
+      "learning_rate": 5.150108018151845e-05,
+      "loss": 0.5414,
+      "step": 291
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 0.29631486535072327,
+      "learning_rate": 5.0855967310385776e-05,
+      "loss": 0.5426,
+      "step": 292
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 0.2700301706790924,
+      "learning_rate": 5.021353972618877e-05,
+      "loss": 0.5618,
+      "step": 293
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.283239483833313,
+      "learning_rate": 4.957383253191567e-05,
+      "loss": 0.5658,
+      "step": 294
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 0.33372044563293457,
+      "learning_rate": 4.893688068190932e-05,
+      "loss": 0.6722,
+      "step": 295
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.28842851519584656,
+      "learning_rate": 4.8302718979957465e-05,
+      "loss": 0.4513,
+      "step": 296
+    },
+    {
+      "epoch": 2.02,
+      "eval_loss": 0.6006384491920471,
+      "eval_runtime": 57.9742,
+      "eval_samples_per_second": 8.97,
+      "eval_steps_per_second": 1.121,
+      "step": 296
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.2431992143392563,
+      "learning_rate": 4.7671382077390923e-05,
+      "loss": 0.4022,
+      "step": 297
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 0.20665951073169708,
+      "learning_rate": 4.704290447119013e-05,
+      "loss": 0.3693,
+      "step": 298
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.21755042672157288,
+      "learning_rate": 4.6417320502100316e-05,
+      "loss": 0.4507,
+      "step": 299
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.25068557262420654,
+      "learning_rate": 4.5794664352755055e-05,
+      "loss": 0.4027,
+      "step": 300
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 0.21365563571453094,
+      "learning_rate": 4.5174970045808373e-05,
+      "loss": 0.447,
+      "step": 301
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.20687268674373627,
+      "learning_rate": 4.4558271442075817e-05,
+      "loss": 0.3979,
+      "step": 302
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.24014657735824585,
+      "learning_rate": 4.394460223868422e-05,
+      "loss": 0.494,
+      "step": 303
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 0.20596379041671753,
+      "learning_rate": 4.333399596723054e-05,
+      "loss": 0.3708,
+      "step": 304
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 0.23077203333377838,
+      "learning_rate": 4.272648599194948e-05,
+      "loss": 0.4383,
+      "step": 305
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 0.2302219718694687,
+      "learning_rate": 4.212210550789066e-05,
+      "loss": 0.437,
+      "step": 306
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 0.23546743392944336,
+      "learning_rate": 4.1520887539104516e-05,
+      "loss": 0.5266,
+      "step": 307
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.20262156426906586,
+      "learning_rate": 4.092286493683812e-05,
+      "loss": 0.4039,
+      "step": 308
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 0.2310255765914917,
+      "learning_rate": 4.0328070377739936e-05,
+      "loss": 0.4723,
+      "step": 309
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 0.22382591664791107,
+      "learning_rate": 3.973653636207437e-05,
+      "loss": 0.4599,
+      "step": 310
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 0.2400328814983368,
+      "learning_rate": 3.914829521194606e-05,
+      "loss": 0.4837,
+      "step": 311
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 0.23558029532432556,
+      "learning_rate": 3.8563379069533626e-05,
+      "loss": 0.4706,
+      "step": 312
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 0.24073629081249237,
+      "learning_rate": 3.7981819895333336e-05,
+      "loss": 0.4662,
+      "step": 313
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.23906177282333374,
+      "learning_rate": 3.740364946641284e-05,
+      "loss": 0.4566,
+      "step": 314
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 0.23790515959262848,
+      "learning_rate": 3.682889937467493e-05,
+      "loss": 0.4365,
+      "step": 315
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 0.2540919780731201,
+      "learning_rate": 3.6257601025131026e-05,
+      "loss": 0.4546,
+      "step": 316
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 0.28762128949165344,
+      "learning_rate": 3.568978563418551e-05,
+      "loss": 0.4624,
+      "step": 317
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 0.24216574430465698,
+      "learning_rate": 3.512548422792983e-05,
+      "loss": 0.4387,
+      "step": 318
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 0.2672511637210846,
+      "learning_rate": 3.456472764044718e-05,
+      "loss": 0.503,
+      "step": 319
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 0.26805609464645386,
+      "learning_rate": 3.400754651212776e-05,
+      "loss": 0.5428,
+      "step": 320
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 0.2592112123966217,
+      "learning_rate": 3.3453971287994545e-05,
+      "loss": 0.4206,
+      "step": 321
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 0.25570839643478394,
+      "learning_rate": 3.290403221603976e-05,
+      "loss": 0.5237,
+      "step": 322
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 0.2656814157962799,
+      "learning_rate": 3.235775934557204e-05,
+      "loss": 0.4655,
+      "step": 323
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 0.2883979380130768,
+      "learning_rate": 3.1815182525574495e-05,
+      "loss": 0.5198,
+      "step": 324
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.35970887541770935,
+      "learning_rate": 3.1276331403073735e-05,
+      "loss": 0.4448,
+      "step": 325
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.25861579179763794,
+      "learning_rate": 3.074123542152001e-05,
+      "loss": 0.5068,
+      "step": 326
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 0.2832384407520294,
+      "learning_rate": 3.020992381917823e-05,
+      "loss": 0.4679,
+      "step": 327
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 0.2728095054626465,
+      "learning_rate": 2.968242562753051e-05,
+      "loss": 0.4211,
+      "step": 328
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 0.27320149540901184,
+      "learning_rate": 2.915876966968978e-05,
+      "loss": 0.5132,
+      "step": 329
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 0.24840450286865234,
+      "learning_rate": 2.8638984558824777e-05,
+      "loss": 0.4337,
+      "step": 330
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.24390266835689545,
+      "learning_rate": 2.812309869659675e-05,
+      "loss": 0.4411,
+      "step": 331
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.23822544515132904,
+      "learning_rate": 2.7611140271607417e-05,
+      "loss": 0.4178,
+      "step": 332
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 0.2783829867839813,
+      "learning_rate": 2.7103137257858868e-05,
+      "loss": 0.4428,
+      "step": 333
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 0.2625970244407654,
+      "learning_rate": 2.6599117413224817e-05,
+      "loss": 0.4076,
+      "step": 334
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 0.2514960765838623,
+      "learning_rate": 2.6099108277934103e-05,
+      "loss": 0.3783,
+      "step": 335
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 0.2869970202445984,
+      "learning_rate": 2.5603137173065674e-05,
+      "loss": 0.3544,
+      "step": 336
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 0.24134644865989685,
+      "learning_rate": 2.5111231199055896e-05,
+      "loss": 0.3664,
+      "step": 337
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 0.30167311429977417,
+      "learning_rate": 2.462341723421758e-05,
+      "loss": 0.4425,
+      "step": 338
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 0.24307532608509064,
+      "learning_rate": 2.4139721933271465e-05,
+      "loss": 0.4078,
+      "step": 339
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 0.2623879015445709,
+      "learning_rate": 2.36601717258897e-05,
+      "loss": 0.4477,
+      "step": 340
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 0.2870636582374573,
+      "learning_rate": 2.3184792815251766e-05,
+      "loss": 0.5101,
+      "step": 341
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 0.36139050126075745,
+      "learning_rate": 2.2713611176612582e-05,
+      "loss": 0.3755,
+      "step": 342
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 0.2712129056453705,
+      "learning_rate": 2.224665255588325e-05,
+      "loss": 0.4178,
+      "step": 343
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 0.2709867060184479,
+      "learning_rate": 2.1783942468224382e-05,
+      "loss": 0.4848,
+      "step": 344
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 0.24954509735107422,
+      "learning_rate": 2.132550619665168e-05,
+      "loss": 0.4352,
+      "step": 345
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 0.23785164952278137,
+      "learning_rate": 2.08713687906547e-05,
+      "loss": 0.4731,
+      "step": 346
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 0.22254765033721924,
+      "learning_rate": 2.0421555064827878e-05,
+      "loss": 0.3899,
+      "step": 347
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 0.2513158619403839,
+      "learning_rate": 1.9976089597514903e-05,
+      "loss": 0.3748,
+      "step": 348
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 0.26037031412124634,
+      "learning_rate": 1.9534996729465426e-05,
+      "loss": 0.4183,
+      "step": 349
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 0.2564360499382019,
+      "learning_rate": 1.9098300562505266e-05,
+      "loss": 0.4153,
+      "step": 350
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 0.2211247682571411,
+      "learning_rate": 1.8666024958219408e-05,
+      "loss": 0.4147,
+      "step": 351
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 0.265408992767334,
+      "learning_rate": 1.8238193536648195e-05,
+      "loss": 0.4655,
+      "step": 352
+    },
+    {
+      "epoch": 2.39,
+      "grad_norm": 0.24408482015132904,
+      "learning_rate": 1.7814829674996592e-05,
+      "loss": 0.4488,
+      "step": 353
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.2215915322303772,
+      "learning_rate": 1.7395956506356937e-05,
+      "loss": 0.4214,
+      "step": 354
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.2894541919231415,
+      "learning_rate": 1.6981596918444953e-05,
+      "loss": 0.4201,
+      "step": 355
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 0.24868462979793549,
+      "learning_rate": 1.6571773552349e-05,
+      "loss": 0.4273,
+      "step": 356
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 0.26563265919685364,
+      "learning_rate": 1.6166508801293013e-05,
+      "loss": 0.5204,
+      "step": 357
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 0.27713629603385925,
+      "learning_rate": 1.5765824809413056e-05,
+      "loss": 0.424,
+      "step": 358
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 0.290195107460022,
+      "learning_rate": 1.5369743470547027e-05,
+      "loss": 0.3887,
+      "step": 359
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.27845337986946106,
+      "learning_rate": 1.4978286427038601e-05,
+      "loss": 0.4918,
+      "step": 360
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.2511301040649414,
+      "learning_rate": 1.4591475068554572e-05,
+      "loss": 0.3305,
+      "step": 361
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 0.28970545530319214,
+      "learning_rate": 1.4209330530916165e-05,
+      "loss": 0.484,
+      "step": 362
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 0.27981922030448914,
+      "learning_rate": 1.3831873694944031e-05,
+      "loss": 0.487,
+      "step": 363
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 0.23398059606552124,
+      "learning_rate": 1.3459125185317434e-05,
+      "loss": 0.4202,
+      "step": 364
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 0.265634685754776,
+      "learning_rate": 1.3091105369447165e-05,
+      "loss": 0.4261,
+      "step": 365
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.25916439294815063,
+      "learning_rate": 1.2727834356362778e-05,
+      "loss": 0.4882,
+      "step": 366
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.29924753308296204,
+      "learning_rate": 1.2369331995613665e-05,
+      "loss": 0.4131,
+      "step": 367
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 0.2214874029159546,
+      "learning_rate": 1.2015617876184527e-05,
+      "loss": 0.3283,
+      "step": 368
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.24375221133232117,
+      "learning_rate": 1.166671132542505e-05,
+      "loss": 0.38,
+      "step": 369
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 0.25932660698890686,
+      "learning_rate": 1.1322631407993811e-05,
+      "loss": 0.4511,
+      "step": 370
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 0.24242925643920898,
+      "learning_rate": 1.098339692481648e-05,
+      "loss": 0.3851,
+      "step": 371
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 0.24607916176319122,
+      "learning_rate": 1.0649026412058583e-05,
+      "loss": 0.4515,
+      "step": 372
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 0.2514863610267639,
+      "learning_rate": 1.0319538140112728e-05,
+      "loss": 0.4129,
+      "step": 373
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 0.23908530175685883,
+      "learning_rate": 9.994950112600154e-06,
+      "loss": 0.3945,
+      "step": 374
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 0.27527761459350586,
+      "learning_rate": 9.675280065387116e-06,
+      "loss": 0.4138,
+      "step": 375
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 0.24933388829231262,
+      "learning_rate": 9.360545465615667e-06,
+      "loss": 0.494,
+      "step": 376
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 0.23585304617881775,
+      "learning_rate": 9.05076351074936e-06,
+      "loss": 0.4261,
+      "step": 377
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.26296553015708923,
+      "learning_rate": 8.745951127633411e-06,
+      "loss": 0.4748,
+      "step": 378
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 0.3045956790447235,
+      "learning_rate": 8.44612497156989e-06,
+      "loss": 0.4836,
+      "step": 379
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 0.2398134022951126,
+      "learning_rate": 8.151301425407699e-06,
+      "loss": 0.3921,
+      "step": 380
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 0.24965588748455048,
+      "learning_rate": 7.861496598647278e-06,
+      "loss": 0.3871,
+      "step": 381
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 0.24710305035114288,
+      "learning_rate": 7.576726326560424e-06,
+      "loss": 0.417,
+      "step": 382
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 0.26674723625183105,
+      "learning_rate": 7.2970061693250154e-06,
+      "loss": 0.371,
+      "step": 383
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 0.25980159640312195,
+      "learning_rate": 7.022351411174866e-06,
+      "loss": 0.4956,
+      "step": 384
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 0.2925579249858856,
+      "learning_rate": 6.75277705956443e-06,
+      "loss": 0.4367,
+      "step": 385
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.3107716739177704,
+      "learning_rate": 6.48829784434889e-06,
+      "loss": 0.5735,
+      "step": 386
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.32108625769615173,
+      "learning_rate": 6.228928216979257e-06,
+      "loss": 0.4012,
+      "step": 387
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 0.24750036001205444,
+      "learning_rate": 5.97468234971279e-06,
+      "loss": 0.3773,
+      "step": 388
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.30614009499549866,
+      "learning_rate": 5.725574134838474e-06,
+      "loss": 0.4044,
+      "step": 389
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.24482354521751404,
+      "learning_rate": 5.481617183918053e-06,
+      "loss": 0.4081,
+      "step": 390
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 0.3191269636154175,
+      "learning_rate": 5.242824827042237e-06,
+      "loss": 0.4647,
+      "step": 391
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 0.2543604373931885,
+      "learning_rate": 5.009210112102292e-06,
+      "loss": 0.3955,
+      "step": 392
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 0.2701224684715271,
+      "learning_rate": 4.7807858040771924e-06,
+      "loss": 0.4296,
+      "step": 393
+    },
+    {
+      "epoch": 2.67,
+      "grad_norm": 0.21666711568832397,
+      "learning_rate": 4.557564384335977e-06,
+      "loss": 0.3457,
+      "step": 394
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 0.2999736964702606,
+      "learning_rate": 4.339558049955927e-06,
+      "loss": 0.4869,
+      "step": 395
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 0.2725822925567627,
+      "learning_rate": 4.126778713055923e-06,
+      "loss": 0.4391,
+      "step": 396
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 0.2413054257631302,
+      "learning_rate": 3.919238000145687e-06,
+      "loss": 0.3792,
+      "step": 397
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 0.3631000518798828,
+      "learning_rate": 3.7169472514904544e-06,
+      "loss": 0.5391,
+      "step": 398
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 0.2513572573661804,
+      "learning_rate": 3.5199175204913117e-06,
+      "loss": 0.4417,
+      "step": 399
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 0.30949896574020386,
+      "learning_rate": 3.3281595730812575e-06,
+      "loss": 0.5317,
+      "step": 400
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 0.23445190489292145,
+      "learning_rate": 3.1416838871368924e-06,
+      "loss": 0.3915,
+      "step": 401
+    },
+    {
+      "epoch": 2.73,
+      "grad_norm": 0.2802431583404541,
+      "learning_rate": 2.9605006519059507e-06,
+      "loss": 0.5729,
+      "step": 402
+    },
+    {
+      "epoch": 2.73,
+      "grad_norm": 0.2695799171924591,
+      "learning_rate": 2.7846197674504913e-06,
+      "loss": 0.444,
+      "step": 403
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 0.31457579135894775,
+      "learning_rate": 2.614050844106042e-06,
+      "loss": 0.4572,
+      "step": 404
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.29997965693473816,
+      "learning_rate": 2.4488032019563402e-06,
+      "loss": 0.4318,
+      "step": 405
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.3099231719970703,
+      "learning_rate": 2.2888858703241866e-06,
+      "loss": 0.4432,
+      "step": 406
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 0.29187706112861633,
+      "learning_rate": 2.1343075872780396e-06,
+      "loss": 0.5138,
+      "step": 407
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 0.26461467146873474,
+      "learning_rate": 1.985076799154528e-06,
+      "loss": 0.423,
+      "step": 408
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 0.3078599274158478,
+      "learning_rate": 1.8412016600969695e-06,
+      "loss": 0.5092,
+      "step": 409
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 0.27981695532798767,
+      "learning_rate": 1.7026900316098215e-06,
+      "loss": 0.4187,
+      "step": 410
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 0.25746870040893555,
+      "learning_rate": 1.5695494821290735e-06,
+      "loss": 0.349,
+      "step": 411
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 0.3197019398212433,
+      "learning_rate": 1.4417872866087534e-06,
+      "loss": 0.4377,
+      "step": 412
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.26146501302719116,
+      "learning_rate": 1.3194104261233798e-06,
+      "loss": 0.4026,
+      "step": 413
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 0.2363249659538269,
+      "learning_rate": 1.2024255874865108e-06,
+      "loss": 0.4289,
+      "step": 414
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 0.24949558079242706,
+      "learning_rate": 1.0908391628854041e-06,
+      "loss": 0.4798,
+      "step": 415
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 0.24620747566223145,
+      "learning_rate": 9.846572495316952e-07,
+      "loss": 0.4676,
+      "step": 416
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 0.3077957034111023,
+      "learning_rate": 8.838856493282754e-07,
+      "loss": 0.392,
+      "step": 417
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 0.3206171691417694,
+      "learning_rate": 7.885298685522235e-07,
+      "loss": 0.512,
+      "step": 418
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 0.27576300501823425,
+      "learning_rate": 6.985951175539796e-07,
+      "loss": 0.4261,
+      "step": 419
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 0.32109883427619934,
+      "learning_rate": 6.140863104726391e-07,
+      "loss": 0.4245,
+      "step": 420
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 0.24693267047405243,
+      "learning_rate": 5.350080649674127e-07,
+      "loss": 0.455,
+      "step": 421
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 0.2874516248703003,
+      "learning_rate": 4.613647019653389e-07,
+      "loss": 0.4103,
+      "step": 422
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 0.24350453913211823,
+      "learning_rate": 3.931602454251837e-07,
+      "loss": 0.3772,
+      "step": 423
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.2467474788427353,
+      "learning_rate": 3.3039842211752824e-07,
+      "loss": 0.4136,
+      "step": 424
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.2721412479877472,
+      "learning_rate": 2.7308266142119785e-07,
+      "loss": 0.3993,
+      "step": 425
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 0.29503345489501953,
+      "learning_rate": 2.212160951358011e-07,
+      "loss": 0.4101,
+      "step": 426
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 0.23621775209903717,
+      "learning_rate": 1.748015573106887e-07,
+      "loss": 0.4014,
+      "step": 427
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 0.2449731081724167,
+      "learning_rate": 1.338415840900109e-07,
+      "loss": 0.3782,
+      "step": 428
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 0.26130935549736023,
+      "learning_rate": 9.833841357421714e-08,
+      "loss": 0.4822,
+      "step": 429
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.21016739308834076,
+      "learning_rate": 6.829398569770939e-08,
+      "loss": 0.3927,
+      "step": 430
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.27280792593955994,
+      "learning_rate": 4.370994212287149e-08,
+      "loss": 0.5188,
+      "step": 431
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 0.28947770595550537,
+      "learning_rate": 2.4587626150351926e-08,
+      "loss": 0.486,
+      "step": 432
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 0.2234964370727539,
+      "learning_rate": 1.0928082645667025e-08,
+      "loss": 0.3444,
+      "step": 433
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 0.2797339856624603,
+      "learning_rate": 2.732057982124392e-09,
+      "loss": 0.4264,
+      "step": 434
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 0.2575642168521881,
+      "learning_rate": 0.0,
+      "loss": 0.422,
+      "step": 435
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 435,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 145,
+  "total_flos": 2.9274017026277376e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-435/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f4e74f968ef592cbe5e646087e575fbeada43b9dc36d459d99d6a9ebc528b7
+size 5560

config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "HuggingFaceTB/cosmo-1b",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "_load_in_4bit": false,
+    "_load_in_8bit": true,
+    "bnb_4bit_compute_dtype": "float32",
+    "bnb_4bit_quant_type": "fp4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": false,
+    "load_in_8bit": true,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.39.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}