Upload 9 files

config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "CustomModelForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_custom.CustomConfig",
+    "AutoModel": "modeling_custom.CustomModel",
+    "AutoModelForCausalLM": "modeling_custom.CustomModelForCausalLM"
+  },
+  "dtype": "float32",
+  "engram_entries": 196,
+  "engram_ngram_orders": [
+    4,
+    8
+  ],
+  "head_dim": 8,
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "max_position_embeddings": 96,
+  "model_type": "custom_model",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 2,
+  "num_lanes": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 2500.0,
+  "swiglu_interval": 4,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.8.0.dev0",
+  "use_cache": false,
+  "use_engram": true,
+  "use_per_head_gating": false,
+  "use_xsa": false,
+  "vocab_size": 260
+}

configuration_custom.py

@@ -0,0 +1,60 @@
+from transformers.configuration_utils import PretrainedConfig
+from typing import Tuple, List, Optional
+
+class CustomConfig(PretrainedConfig):
+    model_type = "custom_model"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size: int = 2564,
+        hidden_size: int = 128,
+        num_hidden_layers: int = 21,
+        num_attention_heads: int = 4,
+        num_key_value_heads: int = 2,
+        intermediate_size: int = 345,
+        swiglu_interval: int = 3,
+        num_lanes: int = 4,
+        use_engram: bool = True,
+        engram_entries: int = 2400,
+        engram_ngram_orders: Tuple[int, ...] = (2, 3),
+        use_xsa: bool = False,
+        use_per_head_gating: bool = False,
+        max_position_embeddings: int = 2048,
+        rope_theta: float = 2500.0,
+        rms_norm_eps: float = 1e-5,
+        tie_word_embeddings: bool = True,
+        use_cache: bool = False,
+        initializer_range: float = 0.02,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.intermediate_size = intermediate_size
+        self.swiglu_interval = swiglu_interval
+        self.num_lanes = num_lanes
+        self.use_engram = use_engram
+        self.engram_entries = engram_entries
+        self.engram_ngram_orders = list(engram_ngram_orders)
+        self.use_xsa = use_xsa
+        self.use_per_head_gating = use_per_head_gating
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.rms_norm_eps = rms_norm_eps
+        self.initializer_range = initializer_range
+        self.head_dim = hidden_size // num_attention_heads
+        self.auto_map = {
+            "AutoConfig": "configuration_custom.CustomConfig",
+            "AutoModel": "modeling_custom.CustomModel",
+            "AutoModelForCausalLM": "modeling_custom.CustomModelForCausalLM",
+        }
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            use_cache=use_cache,
+            **kwargs,
+        )
+

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.8.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00a05ac6aa9f306e158b90ef2885def022f23a056fbb9bd584f6bb8ab8283f2c
+size 283192

modeling_custom.py

@@ -0,0 +1,459 @@
+import math
+import os
+from typing import Optional, Tuple, Union
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as cp
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.generation import GenerationMixin
+from safetensors.torch import load_file
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
+
+from .configuration_custom import CustomConfig
+
+@torch.no_grad()
+def get_hadamard_matrix(d: int, dtype=torch.float32) -> torch.Tensor:
+    eye = torch.eye(d, dtype=dtype)
+    h = 1
+    out = eye.clone()
+    while h < d:
+        out = out.view(-1, 2, h)
+        u = out[:, 0, :]
+        v = out[:, 1, :]
+        out = torch.cat((u + v, u - v), dim=-2)
+        out = out.view(d, d)
+        h *= 2
+    return (out * (1.0 / math.sqrt(d))).contiguous()
+
+class HadamardMLP(nn.Module):
+    def __init__(self, config: CustomConfig):
+        super().__init__()
+        self.dim = config.hidden_size
+        self.scale1 = nn.Parameter(torch.ones(self.dim))
+        self.scale2 = nn.Parameter(torch.ones(self.dim))
+        self.gate = nn.Parameter(torch.ones(self.dim))
+        self.bias = nn.Parameter(torch.zeros(self.dim))
+
+        hadamard_mat = get_hadamard_matrix(self.dim)
+        self.register_buffer("hadamard_mat", hadamard_mat, persistent=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        mat = self.hadamard_mat.type_as(x)
+        h = (x * self.scale1) @ mat
+        g = F.silu(x * self.gate)
+        out = ((h * g) @ mat) * self.scale2 + self.bias
+        return out
+
+class SwiGLUMLP(nn.Module):
+    def __init__(self, config: CustomConfig):
+        super().__init__()
+        self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.up_proj   = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+class EngramMemory(nn.Module):
+    def __init__(self, config: CustomConfig):
+        super().__init__()
+        self.dim = config.hidden_size
+        self.num_entries = config.engram_entries
+        self.n_gram_orders = config.engram_ngram_orders
+
+        self.tables = nn.ModuleList([
+            nn.Embedding(self.num_entries, self.dim) for _ in self.n_gram_orders
+        ])
+        self.gate_proj = nn.Linear(self.dim, self.dim * len(self.n_gram_orders), bias=False)
+        self.out_proj = nn.Linear(self.dim * len(self.n_gram_orders), self.dim, bias=False)
+
+    def _hash_ngram(self, tokens: torch.Tensor, order: int, table_idx: int) -> torch.Tensor:
+        bsz, seqlen = tokens.shape
+        padded = F.pad(tokens, (order - 1, 0), value=0)
+        primes = (10007, 10009, 10037, 10039, 10061, 10067)
+        p = primes[table_idx % len(primes)]
+
+        if order == 2:
+            return (padded[:, :seqlen] * p + padded[:, 1 : seqlen + 1]) % self.num_entries
+        elif order == 3:
+            h = (padded[:, :seqlen] * p + padded[:, 1 : seqlen + 1]) % self.num_entries
+            return (h * p + padded[:, 2 : seqlen + 2]) % self.num_entries
+        else:
+            hash_val = torch.zeros((bsz, seqlen), dtype=torch.int64, device=tokens.device)
+            for k in range(order):
+                tok = padded[:, k : k + seqlen]
+                hash_val = (hash_val * p + tok) % self.num_entries
+            return hash_val
+
+    def forward(self, x: torch.Tensor, tokens: torch.Tensor) -> torch.Tensor:
+        mem_lookups = [self.tables[i](self._hash_ngram(tokens, order, i)) for i, order in enumerate(self.n_gram_orders)]
+        concat_mem = torch.cat(mem_lookups, dim=-1)
+        gate = torch.sigmoid(self.gate_proj(x))
+        return self.out_proj(concat_mem * gate)
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        norm = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        return x * norm * self.weight
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim: int, max_position_embeddings: int = 2048, base: float = 10000.0):
+        super().__init__()
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.float32) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self._set_cos_sin_cache(max_position_embeddings)
+
+    def _set_cos_sin_cache(self, seq_len: int, device=None, dtype=torch.float32):
+        t = torch.arange(seq_len, device=device, dtype=torch.float32)
+        inv_freq = self.inv_freq.to(device=device, dtype=torch.float32)
+        freqs = torch.outer(t, inv_freq)
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype=dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype=dtype), persistent=False)
+
+    def forward(self, seq_len: int, device: torch.device, dtype: torch.dtype = torch.float32):
+        if not hasattr(self, "cos_cached") or seq_len > self.cos_cached.shape[0] or self.cos_cached.device != device:
+            self._set_cos_sin_cache(seq_len, device=device, dtype=dtype)
+        return (
+            self.cos_cached[:seq_len].to(device=device, dtype=dtype),
+            self.sin_cached[:seq_len].to(device=device, dtype=dtype),
+        )
+
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+def apply_rotary_pos_emb(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor):
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+class XSAGQAttention(nn.Module):
+    def __init__(self, config: CustomConfig):
+        super().__init__()
+        self.dim = config.hidden_size
+        self.n_heads = config.num_attention_heads
+        self.n_kv_heads = config.num_key_value_heads
+        self.head_dim = config.head_dim
+        self.num_kv_groups = self.n_heads // self.n_kv_heads
+        self.use_xsa = config.use_xsa
+        self.use_per_head_gating = config.use_per_head_gating
+
+        self.wq = nn.Linear(self.dim, self.n_heads * self.head_dim, bias=False)
+        self.wk = nn.Linear(self.dim, self.n_kv_heads * self.head_dim, bias=False)
+        self.wv = nn.Linear(self.dim, self.n_kv_heads * self.head_dim, bias=False)
+        self.wo = nn.Linear(self.n_heads * self.head_dim, self.dim, bias=False)
+
+        self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+
+        if self.use_per_head_gating:
+            self.head_gate = nn.Linear(self.dim, self.n_heads, bias=True)
+            nn.init.constant_(self.head_gate.bias, 1.0)
+            nn.init.zeros_(self.head_gate.weight)
+
+    def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+        bsz, seqlen, _ = x.shape
+
+        xq = self.wq(x).view(bsz, seqlen, self.n_heads, self.head_dim).transpose(1, 2)
+        xk = self.wk(x).view(bsz, seqlen, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        xv = self.wv(x).view(bsz, seqlen, self.n_kv_heads, self.head_dim).transpose(1, 2)
+
+        xq = self.q_norm(xq)
+        xk = self.k_norm(xk)
+
+        xq, xk = apply_rotary_pos_emb(xq, xk, cos, sin)
+
+        if self.num_kv_groups > 1:
+            xk = xk.repeat_interleave(self.num_kv_groups, dim=1)
+            xv_expanded = xv.repeat_interleave(self.num_kv_groups, dim=1)
+        else:
+            xv_expanded = xv
+
+        attn_out = F.scaled_dot_product_attention(xq, xk, xv_expanded, is_causal=True)
+
+        if self.use_xsa:
+            vn = F.normalize(xv_expanded, p=2, dim=-1, eps=1e-6)
+            proj = (attn_out * vn).sum(dim=-1, keepdim=True)
+            attn_out = attn_out - proj * vn
+
+        if self.use_per_head_gating:
+            gate = torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1)
+            attn_out = attn_out * gate
+
+        out = attn_out.transpose(1, 2).contiguous().view(bsz, seqlen, -1)
+        return self.wo(out)
+
+
+class MultiLaneBlock(nn.Module):
+    def __init__(self, config: CustomConfig, layer_idx: int):
+        super().__init__()
+        self.num_lanes = config.num_lanes
+        self.dim = config.hidden_size
+        self.layer_idx = layer_idx
+
+        self.attn_norm = RMSNorm(self.dim, eps=config.rms_norm_eps)
+        self.attn = XSAGQAttention(config)
+
+        self.mlp_norm = RMSNorm(self.dim, eps=config.rms_norm_eps)
+        if config.swiglu_interval == 0:
+            self.use_swiglu = False
+        elif config.swiglu_interval == 1:
+            self.use_swiglu = True
+        else:
+            self.use_swiglu = ((layer_idx + 1) % config.swiglu_interval == 0)
+
+        if self.use_swiglu:
+            self.mlp = SwiGLUMLP(config)
+        else:
+            self.mlp = HadamardMLP(config)
+
+        self.lane_mix_attn = nn.Parameter(torch.eye(self.num_lanes) + 0.05 * torch.randn(self.num_lanes, self.num_lanes))
+        self.lane_mix_mlp = nn.Parameter(torch.eye(self.num_lanes) + 0.05 * torch.randn(self.num_lanes, self.num_lanes))
+
+    def forward(self, lanes: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+        primary = lanes[0]
+        attn_update = self.attn(self.attn_norm(primary), cos, sin)
+
+        mixed = torch.matmul(self.lane_mix_attn, lanes.view(self.num_lanes, -1)).view_as(lanes)
+        lanes = torch.cat([(mixed[0] + attn_update).unsqueeze(0), mixed[1:]], dim=0)
+
+        mlp_update = self.mlp(self.mlp_norm(lanes[0]))
+        mixed = torch.matmul(self.lane_mix_mlp, lanes.view(self.num_lanes, -1)).view_as(lanes)
+        lanes = torch.cat([(mixed[0] + mlp_update).unsqueeze(0), mixed[1:]], dim=0)
+        return lanes
+
+class CustomPreTrainedModel(PreTrainedModel):
+    config_class = CustomConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["MultiLaneBlock"]
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if hasattr(module, "bias") and module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, RMSNorm):
+            module.weight.data.fill_(1.0)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        config = kwargs.pop("config", None)
+        kwargs.pop("trust_remote_code", None)
+        torch_dtype = kwargs.pop("torch_dtype", None)
+        kwargs.pop("device_map", None)
+        kwargs.pop("low_cpu_mem_usage", None)
+
+        if config is None:
+            config = CustomConfig.from_pretrained(pretrained_model_name_or_path)
+
+        model = cls(config, *model_args)
+
+        st_file = os.path.join(pretrained_model_name_or_path, "model.safetensors")
+        bin_file = os.path.join(pretrained_model_name_or_path, "pytorch_model.bin")
+
+        if os.path.exists(st_file):
+            state_dict = load_file(st_file)
+            model.load_state_dict(state_dict, strict=False)
+        elif os.path.exists(bin_file):
+            state_dict = torch.load(bin_file, map_location="cpu")
+            model.load_state_dict(state_dict, strict=False)
+        else:
+            return super().from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
+
+        if getattr(config, "tie_word_embeddings", True) and hasattr(model, "lm_head") and hasattr(model, "model"):
+            model.lm_head.weight = model.model.embed_tokens.weight
+
+        if torch_dtype is not None:
+            model.to(dtype=torch_dtype)
+
+        return model
+
+class CustomModel(CustomPreTrainedModel):
+    def __init__(self, config: CustomConfig, *args, **kwargs):
+        super().__init__(config)
+        self.config = config
+        self.num_lanes = config.num_lanes
+        self.gradient_checkpointing = False
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        if config.use_engram:
+            self.engram = EngramMemory(config)
+            self.engram_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        else:
+            self.engram = None
+            self.engram_norm = None
+
+        self.layers = nn.ModuleList([
+            MultiLaneBlock(config, layer_idx=i) for i in range(config.num_hidden_layers)
+        ])
+
+        # Enhanced Lane Pooling: Learned softmax combination of all 3 lanes before norm
+        self.lane_pool_weights = nn.Parameter(torch.tensor([1.0] + [0.1] * (config.num_lanes - 1)))
+        self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = RotaryEmbedding(config.head_dim, config.max_position_embeddings, config.rope_theta)
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ):
+        if input_ids is not None:
+            bsz, seqlen = input_ids.shape
+            h0 = self.embed_tokens(input_ids)
+            tokens_for_engram = input_ids
+        elif inputs_embeds is not None:
+            bsz, seqlen, _ = inputs_embeds.shape
+            h0 = inputs_embeds
+            tokens_for_engram = torch.zeros((bsz, seqlen), dtype=torch.long, device=inputs_embeds.device)
+        else:
+            raise ValueError("You must specify either input_ids or inputs_embeds")
+
+        if self.engram is not None:
+            engram_out = self.engram(self.engram_norm(h0), tokens_for_engram)
+            h0 = h0 + engram_out
+
+        lanes = h0.unsqueeze(0).repeat(self.num_lanes, 1, 1, 1)
+
+        cos, sin = self.rotary_emb(seqlen, device=h0.device, dtype=h0.dtype)
+        cos = cos.unsqueeze(0).unsqueeze(0)
+        sin = sin.unsqueeze(0).unsqueeze(0)
+
+        for layer in self.layers:
+            if self.gradient_checkpointing and self.training:
+                lanes = cp.checkpoint(layer, lanes, cos, sin, use_reentrant=False)
+            else:
+                lanes = layer(lanes, cos, sin)
+
+        # Weighted lane pooling for higher representation power
+        pool_weights = F.softmax(self.lane_pool_weights, dim=0).view(self.num_lanes, 1, 1, 1)
+        pooled = (lanes * pool_weights).sum(dim=0)
+        out = self.norm(pooled)
+        return out
+
+class CustomModelForCausalLM(CustomPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+    _keys_to_ignore_on_load_missing = ["lm_head.weight"]
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config: CustomConfig, *args, **kwargs):
+        super().__init__(config)
+        self.model = CustomModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        **kwargs,
+    ):
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update({
+            "attention_mask": attention_mask,
+            "use_cache": False,
+        })
+        return model_inputs
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        return_dict = return_dict if return_dict is not None else getattr(self.config, "return_dict", True)
+
+        hidden_states = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, self.config.vocab_size),
+                shift_labels.view(-1),
+                ignore_index=-100
+            )
+
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None,
+        )
+
+
+
+AutoConfig.register("custom_model", CustomConfig)
+AutoModel.register(CustomConfig, CustomModel)
+AutoModelForCausalLM.register(CustomConfig, CustomModelForCausalLM)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "unk_token": "<|unk|>",
+  "pad_token": "<|pad|>"
+}

tokenizer.json

@@ -0,0 +1,351 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 256,
+      "content": "<|bos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 257,
+      "content": "<|eos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 258,
+      "content": "<|unk|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 259,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": false
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "Ā": 0,
+      "ā": 1,
+      "Ă": 2,
+      "ă": 3,
+      "Ą": 4,
+      "ą": 5,
+      "Ć": 6,
+      "ć": 7,
+      "Ĉ": 8,
+      "ĉ": 9,
+      "Ċ": 10,
+      "ċ": 11,
+      "Č": 12,
+      "č": 13,
+      "Ď": 14,
+      "ď": 15,
+      "Đ": 16,
+      "đ": 17,
+      "Ē": 18,
+      "ē": 19,
+      "Ĕ": 20,
+      "ĕ": 21,
+      "Ė": 22,
+      "ė": 23,
+      "Ę": 24,
+      "ę": 25,
+      "Ě": 26,
+      "ě": 27,
+      "Ĝ": 28,
+      "ĝ": 29,
+      "Ğ": 30,
+      "ğ": 31,
+      "Ġ": 32,
+      "!": 33,
+      "\"": 34,
+      "#": 35,
+      "$": 36,
+      "%": 37,
+      "&": 38,
+      "'": 39,
+      "(": 40,
+      ")": 41,
+      "*": 42,
+      "+": 43,
+      ",": 44,
+      "-": 45,
+      ".": 46,
+      "/": 47,
+      "0": 48,
+      "1": 49,
+      "2": 50,
+      "3": 51,
+      "4": 52,
+      "5": 53,
+      "6": 54,
+      "7": 55,
+      "8": 56,
+      "9": 57,
+      ":": 58,
+      ";": 59,
+      "<": 60,
+      "=": 61,
+      ">": 62,
+      "?": 63,
+      "@": 64,
+      "A": 65,
+      "B": 66,
+      "C": 67,
+      "D": 68,
+      "E": 69,
+      "F": 70,
+      "G": 71,
+      "H": 72,
+      "I": 73,
+      "J": 74,
+      "K": 75,
+      "L": 76,
+      "M": 77,
+      "N": 78,
+      "O": 79,
+      "P": 80,
+      "Q": 81,
+      "R": 82,
+      "S": 83,
+      "T": 84,
+      "U": 85,
+      "V": 86,
+      "W": 87,
+      "X": 88,
+      "Y": 89,
+      "Z": 90,
+      "[": 91,
+      "\\": 92,
+      "]": 93,
+      "^": 94,
+      "_": 95,
+      "`": 96,
+      "a": 97,
+      "b": 98,
+      "c": 99,
+      "d": 100,
+      "e": 101,
+      "f": 102,
+      "g": 103,
+      "h": 104,
+      "i": 105,
+      "j": 106,
+      "k": 107,
+      "l": 108,
+      "m": 109,
+      "n": 110,
+      "o": 111,
+      "p": 112,
+      "q": 113,
+      "r": 114,
+      "s": 115,
+      "t": 116,
+      "u": 117,
+      "v": 118,
+      "w": 119,
+      "x": 120,
+      "y": 121,
+      "z": 122,
+      "{": 123,
+      "|": 124,
+      "}": 125,
+      "~": 126,
+      "ġ": 127,
+      "Ģ": 128,
+      "ģ": 129,
+      "Ĥ": 130,
+      "ĥ": 131,
+      "Ħ": 132,
+      "ħ": 133,
+      "Ĩ": 134,
+      "ĩ": 135,
+      "Ī": 136,
+      "ī": 137,
+      "Ĭ": 138,
+      "ĭ": 139,
+      "Į": 140,
+      "į": 141,
+      "İ": 142,
+      "ı": 143,
+      "IJ": 144,
+      "ij": 145,
+      "Ĵ": 146,
+      "ĵ": 147,
+      "Ķ": 148,
+      "ķ": 149,
+      "ĸ": 150,
+      "Ĺ": 151,
+      "ĺ": 152,
+      "Ļ": 153,
+      "ļ": 154,
+      "Ľ": 155,
+      "ľ": 156,
+      "Ŀ": 157,
+      "ŀ": 158,
+      "Ł": 159,
+      "ł": 160,
+      "¡": 161,
+      "¢": 162,
+      "£": 163,
+      "¤": 164,
+      "¥": 165,
+      "¦": 166,
+      "§": 167,
+      "¨": 168,
+      "©": 169,
+      "ª": 170,
+      "«": 171,
+      "¬": 172,
+      "Ń": 173,
+      "®": 174,
+      "¯": 175,
+      "°": 176,
+      "±": 177,
+      "²": 178,
+      "³": 179,
+      "´": 180,
+      "µ": 181,
+      "¶": 182,
+      "·": 183,
+      "¸": 184,
+      "¹": 185,
+      "º": 186,
+      "»": 187,
+      "¼": 188,
+      "½": 189,
+      "¾": 190,
+      "¿": 191,
+      "À": 192,
+      "Á": 193,
+      "Â": 194,
+      "Ã": 195,
+      "Ä": 196,
+      "Å": 197,
+      "Æ": 198,
+      "Ç": 199,
+      "È": 200,
+      "É": 201,
+      "Ê": 202,
+      "Ë": 203,
+      "Ì": 204,
+      "Í": 205,
+      "Î": 206,
+      "Ï": 207,
+      "Ð": 208,
+      "Ñ": 209,
+      "Ò": 210,
+      "Ó": 211,
+      "Ô": 212,
+      "Õ": 213,
+      "Ö": 214,
+      "×": 215,
+      "Ø": 216,
+      "Ù": 217,
+      "Ú": 218,
+      "Û": 219,
+      "Ü": 220,
+      "Ý": 221,
+      "Þ": 222,
+      "ß": 223,
+      "à": 224,
+      "á": 225,
+      "â": 226,
+      "ã": 227,
+      "ä": 228,
+      "å": 229,
+      "æ": 230,
+      "ç": 231,
+      "è": 232,
+      "é": 233,
+      "ê": 234,
+      "ë": 235,
+      "ì": 236,
+      "í": 237,
+      "î": 238,
+      "ï": 239,
+      "ð": 240,
+      "ñ": 241,
+      "ò": 242,
+      "ó": 243,
+      "ô": 244,
+      "õ": 245,
+      "ö": 246,
+      "÷": 247,
+      "ø": 248,
+      "ù": 249,
+      "ú": 250,
+      "û": 251,
+      "ü": 252,
+      "ý": 253,
+      "þ": 254,
+      "ÿ": 255
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

trainer_state.json

@@ -0,0 +1,1410 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.22103569035662726,
+  "eval_steps": 250,
+  "global_step": 8000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013814730647289204,
+      "grad_norm": 1.3092303276062012,
+      "learning_rate": 0.0009187500000000001,
+      "loss": 5.301650390625,
+      "step": 50
+    },
+    {
+      "epoch": 0.0027629461294578408,
+      "grad_norm": 0.2540936768054962,
+      "learning_rate": 0.00185625,
+      "loss": 3.7713958740234377,
+      "step": 100
+    },
+    {
+      "epoch": 0.004144419194186761,
+      "grad_norm": 0.8980741500854492,
+      "learning_rate": 0.0027937500000000002,
+      "loss": 3.084112548828125,
+      "step": 150
+    },
+    {
+      "epoch": 0.0055258922589156816,
+      "grad_norm": 0.513289213180542,
+      "learning_rate": 0.003,
+      "loss": 2.6681784057617186,
+      "step": 200
+    },
+    {
+      "epoch": 0.006907365323644602,
+      "grad_norm": 0.8646414875984192,
+      "learning_rate": 0.003,
+      "loss": 2.4488145446777345,
+      "step": 250
+    },
+    {
+      "epoch": 0.006907365323644602,
+      "eval_loss": 2.3864049911499023,
+      "eval_runtime": 5.0393,
+      "eval_samples_per_second": 404.423,
+      "eval_steps_per_second": 3.175,
+      "step": 250
+    },
+    {
+      "epoch": 0.008288838388373522,
+      "grad_norm": 1.049102783203125,
+      "learning_rate": 0.003,
+      "loss": 2.3174089050292968,
+      "step": 300
+    },
+    {
+      "epoch": 0.009670311453102444,
+      "grad_norm": 2.3267924785614014,
+      "learning_rate": 0.003,
+      "loss": 2.2231291198730467,
+      "step": 350
+    },
+    {
+      "epoch": 0.011051784517831363,
+      "grad_norm": 1.4895470142364502,
+      "learning_rate": 0.003,
+      "loss": 2.136985168457031,
+      "step": 400
+    },
+    {
+      "epoch": 0.012433257582560284,
+      "grad_norm": 0.8463179469108582,
+      "learning_rate": 0.003,
+      "loss": 2.075450897216797,
+      "step": 450
+    },
+    {
+      "epoch": 0.013814730647289204,
+      "grad_norm": 1.4170407056808472,
+      "learning_rate": 0.003,
+      "loss": 2.0253837585449217,
+      "step": 500
+    },
+    {
+      "epoch": 0.013814730647289204,
+      "eval_loss": 2.0032660961151123,
+      "eval_runtime": 4.8177,
+      "eval_samples_per_second": 423.027,
+      "eval_steps_per_second": 3.321,
+      "step": 500
+    },
+    {
+      "epoch": 0.015196203712018125,
+      "grad_norm": 0.9727197289466858,
+      "learning_rate": 0.003,
+      "loss": 1.985428924560547,
+      "step": 550
+    },
+    {
+      "epoch": 0.016577676776747045,
+      "grad_norm": 1.3294126987457275,
+      "learning_rate": 0.003,
+      "loss": 1.945387725830078,
+      "step": 600
+    },
+    {
+      "epoch": 0.017959149841475966,
+      "grad_norm": 1.1075317859649658,
+      "learning_rate": 0.003,
+      "loss": 1.909156494140625,
+      "step": 650
+    },
+    {
+      "epoch": 0.019340622906204887,
+      "grad_norm": 1.532372236251831,
+      "learning_rate": 0.003,
+      "loss": 1.8862774658203125,
+      "step": 700
+    },
+    {
+      "epoch": 0.02072209597093381,
+      "grad_norm": 1.4036468267440796,
+      "learning_rate": 0.003,
+      "loss": 1.8624417114257812,
+      "step": 750
+    },
+    {
+      "epoch": 0.02072209597093381,
+      "eval_loss": 1.8611562252044678,
+      "eval_runtime": 4.8927,
+      "eval_samples_per_second": 416.539,
+      "eval_steps_per_second": 3.27,
+      "step": 750
+    },
+    {
+      "epoch": 0.022103569035662726,
+      "grad_norm": 0.9797951579093933,
+      "learning_rate": 0.003,
+      "loss": 1.8470022583007812,
+      "step": 800
+    },
+    {
+      "epoch": 0.023485042100391647,
+      "grad_norm": 1.0192056894302368,
+      "learning_rate": 0.003,
+      "loss": 1.8303775024414062,
+      "step": 850
+    },
+    {
+      "epoch": 0.02486651516512057,
+      "grad_norm": 1.3044160604476929,
+      "learning_rate": 0.003,
+      "loss": 1.8083892822265626,
+      "step": 900
+    },
+    {
+      "epoch": 0.02624798822984949,
+      "grad_norm": 1.347043514251709,
+      "learning_rate": 0.003,
+      "loss": 1.7919261169433593,
+      "step": 950
+    },
+    {
+      "epoch": 0.027629461294578408,
+      "grad_norm": 1.4868067502975464,
+      "learning_rate": 0.003,
+      "loss": 1.7848297119140626,
+      "step": 1000
+    },
+    {
+      "epoch": 0.027629461294578408,
+      "eval_loss": 1.7806859016418457,
+      "eval_runtime": 4.9437,
+      "eval_samples_per_second": 412.245,
+      "eval_steps_per_second": 3.236,
+      "step": 1000
+    },
+    {
+      "epoch": 0.02901093435930733,
+      "grad_norm": 1.3199470043182373,
+      "learning_rate": 0.003,
+      "loss": 1.770255584716797,
+      "step": 1050
+    },
+    {
+      "epoch": 0.03039240742403625,
+      "grad_norm": 0.942984402179718,
+      "learning_rate": 0.003,
+      "loss": 1.764229278564453,
+      "step": 1100
+    },
+    {
+      "epoch": 0.03177388048876517,
+      "grad_norm": 1.001526951789856,
+      "learning_rate": 0.003,
+      "loss": 1.748619842529297,
+      "step": 1150
+    },
+    {
+      "epoch": 0.03315535355349409,
+      "grad_norm": 0.9165797233581543,
+      "learning_rate": 0.003,
+      "loss": 1.7377178955078125,
+      "step": 1200
+    },
+    {
+      "epoch": 0.034536826618223014,
+      "grad_norm": 1.5860354900360107,
+      "learning_rate": 0.003,
+      "loss": 1.7299589538574218,
+      "step": 1250
+    },
+    {
+      "epoch": 0.034536826618223014,
+      "eval_loss": 1.7355499267578125,
+      "eval_runtime": 4.8017,
+      "eval_samples_per_second": 424.429,
+      "eval_steps_per_second": 3.332,
+      "step": 1250
+    },
+    {
+      "epoch": 0.03591829968295193,
+      "grad_norm": 1.7614655494689941,
+      "learning_rate": 0.003,
+      "loss": 1.7231251525878906,
+      "step": 1300
+    },
+    {
+      "epoch": 0.03729977274768085,
+      "grad_norm": 0.8269591927528381,
+      "learning_rate": 0.003,
+      "loss": 1.7191682434082032,
+      "step": 1350
+    },
+    {
+      "epoch": 0.038681245812409774,
+      "grad_norm": 1.2936993837356567,
+      "learning_rate": 0.003,
+      "loss": 1.715775146484375,
+      "step": 1400
+    },
+    {
+      "epoch": 0.04006271887713869,
+      "grad_norm": 0.8943546414375305,
+      "learning_rate": 0.003,
+      "loss": 1.7027583312988281,
+      "step": 1450
+    },
+    {
+      "epoch": 0.04144419194186762,
+      "grad_norm": 0.8446581959724426,
+      "learning_rate": 0.003,
+      "loss": 1.7010877990722657,
+      "step": 1500
+    },
+    {
+      "epoch": 0.04144419194186762,
+      "eval_loss": 1.7066650390625,
+      "eval_runtime": 4.8419,
+      "eval_samples_per_second": 420.908,
+      "eval_steps_per_second": 3.304,
+      "step": 1500
+    },
+    {
+      "epoch": 0.042825665006596535,
+      "grad_norm": 1.1785316467285156,
+      "learning_rate": 0.003,
+      "loss": 1.6969337463378906,
+      "step": 1550
+    },
+    {
+      "epoch": 0.04420713807132545,
+      "grad_norm": 1.1146525144577026,
+      "learning_rate": 0.003,
+      "loss": 1.6898464965820312,
+      "step": 1600
+    },
+    {
+      "epoch": 0.04558861113605438,
+      "grad_norm": 1.050032377243042,
+      "learning_rate": 0.003,
+      "loss": 1.6903854370117188,
+      "step": 1650
+    },
+    {
+      "epoch": 0.046970084200783295,
+      "grad_norm": 0.817931056022644,
+      "learning_rate": 0.003,
+      "loss": 1.6805955505371093,
+      "step": 1700
+    },
+    {
+      "epoch": 0.04835155726551221,
+      "grad_norm": 0.7598580121994019,
+      "learning_rate": 0.003,
+      "loss": 1.6817108154296876,
+      "step": 1750
+    },
+    {
+      "epoch": 0.04835155726551221,
+      "eval_loss": 1.681281328201294,
+      "eval_runtime": 4.9443,
+      "eval_samples_per_second": 412.196,
+      "eval_steps_per_second": 3.236,
+      "step": 1750
+    },
+    {
+      "epoch": 0.04973303033024114,
+      "grad_norm": 1.0627769231796265,
+      "learning_rate": 0.003,
+      "loss": 1.6690780639648437,
+      "step": 1800
+    },
+    {
+      "epoch": 0.051114503394970055,
+      "grad_norm": 0.9967713952064514,
+      "learning_rate": 0.003,
+      "loss": 1.6677894592285156,
+      "step": 1850
+    },
+    {
+      "epoch": 0.05249597645969898,
+      "grad_norm": 0.8788776397705078,
+      "learning_rate": 0.003,
+      "loss": 1.671481170654297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.0538774495244279,
+      "grad_norm": 0.8975939750671387,
+      "learning_rate": 0.003,
+      "loss": 1.6636967468261719,
+      "step": 1950
+    },
+    {
+      "epoch": 0.055258922589156816,
+      "grad_norm": 0.8807463645935059,
+      "learning_rate": 0.003,
+      "loss": 1.662882080078125,
+      "step": 2000
+    },
+    {
+      "epoch": 0.055258922589156816,
+      "eval_loss": 1.659508228302002,
+      "eval_runtime": 4.8341,
+      "eval_samples_per_second": 421.589,
+      "eval_steps_per_second": 3.31,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05664039565388574,
+      "grad_norm": 0.8365132212638855,
+      "learning_rate": 0.003,
+      "loss": 1.6577224731445312,
+      "step": 2050
+    },
+    {
+      "epoch": 0.05802186871861466,
+      "grad_norm": 1.006463646888733,
+      "learning_rate": 0.003,
+      "loss": 1.651870574951172,
+      "step": 2100
+    },
+    {
+      "epoch": 0.059403341783343576,
+      "grad_norm": 1.0645591020584106,
+      "learning_rate": 0.003,
+      "loss": 1.655391845703125,
+      "step": 2150
+    },
+    {
+      "epoch": 0.0607848148480725,
+      "grad_norm": 0.7508670091629028,
+      "learning_rate": 0.003,
+      "loss": 1.641587677001953,
+      "step": 2200
+    },
+    {
+      "epoch": 0.06216628791280142,
+      "grad_norm": 0.9019992351531982,
+      "learning_rate": 0.003,
+      "loss": 1.6511729431152344,
+      "step": 2250
+    },
+    {
+      "epoch": 0.06216628791280142,
+      "eval_loss": 1.6495267152786255,
+      "eval_runtime": 4.9408,
+      "eval_samples_per_second": 412.485,
+      "eval_steps_per_second": 3.238,
+      "step": 2250
+    },
+    {
+      "epoch": 0.06354776097753034,
+      "grad_norm": 0.8229948282241821,
+      "learning_rate": 0.003,
+      "loss": 1.647374725341797,
+      "step": 2300
+    },
+    {
+      "epoch": 0.06492923404225927,
+      "grad_norm": 0.9779911041259766,
+      "learning_rate": 0.003,
+      "loss": 1.6417164611816406,
+      "step": 2350
+    },
+    {
+      "epoch": 0.06631070710698818,
+      "grad_norm": 0.8557340502738953,
+      "learning_rate": 0.003,
+      "loss": 1.6406777954101563,
+      "step": 2400
+    },
+    {
+      "epoch": 0.0676921801717171,
+      "grad_norm": 0.7618058919906616,
+      "learning_rate": 0.003,
+      "loss": 1.6387905883789062,
+      "step": 2450
+    },
+    {
+      "epoch": 0.06907365323644603,
+      "grad_norm": 0.9702732563018799,
+      "learning_rate": 0.003,
+      "loss": 1.6384074401855468,
+      "step": 2500
+    },
+    {
+      "epoch": 0.06907365323644603,
+      "eval_loss": 1.6367918252944946,
+      "eval_runtime": 5.186,
+      "eval_samples_per_second": 392.979,
+      "eval_steps_per_second": 3.085,
+      "step": 2500
+    },
+    {
+      "epoch": 0.07045512630117494,
+      "grad_norm": 0.8180472254753113,
+      "learning_rate": 0.003,
+      "loss": 1.6398277282714844,
+      "step": 2550
+    },
+    {
+      "epoch": 0.07183659936590386,
+      "grad_norm": 0.8394176959991455,
+      "learning_rate": 0.003,
+      "loss": 1.6309616088867187,
+      "step": 2600
+    },
+    {
+      "epoch": 0.07321807243063279,
+      "grad_norm": 1.1738929748535156,
+      "learning_rate": 0.003,
+      "loss": 1.6329539489746094,
+      "step": 2650
+    },
+    {
+      "epoch": 0.0745995454953617,
+      "grad_norm": 0.7382224798202515,
+      "learning_rate": 0.003,
+      "loss": 1.6252481079101562,
+      "step": 2700
+    },
+    {
+      "epoch": 0.07598101856009062,
+      "grad_norm": 0.7925794720649719,
+      "learning_rate": 0.003,
+      "loss": 1.628258056640625,
+      "step": 2750
+    },
+    {
+      "epoch": 0.07598101856009062,
+      "eval_loss": 1.6294986009597778,
+      "eval_runtime": 5.0001,
+      "eval_samples_per_second": 407.594,
+      "eval_steps_per_second": 3.2,
+      "step": 2750
+    },
+    {
+      "epoch": 0.07736249162481955,
+      "grad_norm": 1.0887517929077148,
+      "learning_rate": 0.003,
+      "loss": 1.6292604064941407,
+      "step": 2800
+    },
+    {
+      "epoch": 0.07874396468954846,
+      "grad_norm": 0.8471100926399231,
+      "learning_rate": 0.003,
+      "loss": 1.6208766174316407,
+      "step": 2850
+    },
+    {
+      "epoch": 0.08012543775427738,
+      "grad_norm": 0.8405469059944153,
+      "learning_rate": 0.003,
+      "loss": 1.6250108337402345,
+      "step": 2900
+    },
+    {
+      "epoch": 0.08150691081900631,
+      "grad_norm": 0.7940107583999634,
+      "learning_rate": 0.003,
+      "loss": 1.6223403930664062,
+      "step": 2950
+    },
+    {
+      "epoch": 0.08288838388373523,
+      "grad_norm": 0.9527040123939514,
+      "learning_rate": 0.003,
+      "loss": 1.62312744140625,
+      "step": 3000
+    },
+    {
+      "epoch": 0.08288838388373523,
+      "eval_loss": 1.622558832168579,
+      "eval_runtime": 4.9038,
+      "eval_samples_per_second": 415.594,
+      "eval_steps_per_second": 3.263,
+      "step": 3000
+    },
+    {
+      "epoch": 0.08426985694846414,
+      "grad_norm": 0.7105967402458191,
+      "learning_rate": 0.003,
+      "loss": 1.6175238037109374,
+      "step": 3050
+    },
+    {
+      "epoch": 0.08565133001319307,
+      "grad_norm": 1.089608907699585,
+      "learning_rate": 0.003,
+      "loss": 1.6170733642578126,
+      "step": 3100
+    },
+    {
+      "epoch": 0.087032803077922,
+      "grad_norm": 0.8875170350074768,
+      "learning_rate": 0.003,
+      "loss": 1.61830322265625,
+      "step": 3150
+    },
+    {
+      "epoch": 0.0884142761426509,
+      "grad_norm": 0.9304706454277039,
+      "learning_rate": 0.003,
+      "loss": 1.6177555847167968,
+      "step": 3200
+    },
+    {
+      "epoch": 0.08979574920737983,
+      "grad_norm": 0.8498280644416809,
+      "learning_rate": 0.0029992286819925424,
+      "loss": 1.6121067810058594,
+      "step": 3250
+    },
+    {
+      "epoch": 0.08979574920737983,
+      "eval_loss": 1.6201170682907104,
+      "eval_runtime": 4.8624,
+      "eval_samples_per_second": 419.135,
+      "eval_steps_per_second": 3.291,
+      "step": 3250
+    },
+    {
+      "epoch": 0.09117722227210875,
+      "grad_norm": 0.884667694568634,
+      "learning_rate": 0.0029968522736369967,
+      "loss": 1.6176124572753907,
+      "step": 3300
+    },
+    {
+      "epoch": 0.09255869533683767,
+      "grad_norm": 0.8628567457199097,
+      "learning_rate": 0.0029928729983747797,
+      "loss": 1.6118777465820313,
+      "step": 3350
+    },
+    {
+      "epoch": 0.09394016840156659,
+      "grad_norm": 0.7607647776603699,
+      "learning_rate": 0.002987295117313512,
+      "loss": 1.6121871948242188,
+      "step": 3400
+    },
+    {
+      "epoch": 0.09532164146629551,
+      "grad_norm": 0.6003952622413635,
+      "learning_rate": 0.002980124603387944,
+      "loss": 1.6035992431640624,
+      "step": 3450
+    },
+    {
+      "epoch": 0.09670311453102443,
+      "grad_norm": 0.668270468711853,
+      "learning_rate": 0.0029713691349639894,
+      "loss": 1.6169198608398438,
+      "step": 3500
+    },
+    {
+      "epoch": 0.09670311453102443,
+      "eval_loss": 1.61012864112854,
+      "eval_runtime": 4.8048,
+      "eval_samples_per_second": 424.158,
+      "eval_steps_per_second": 3.33,
+      "step": 3500
+    },
+    {
+      "epoch": 0.09808458759575335,
+      "grad_norm": 0.8415676355361938,
+      "learning_rate": 0.002961038087616538,
+      "loss": 1.6114456176757812,
+      "step": 3550
+    },
+    {
+      "epoch": 0.09946606066048228,
+      "grad_norm": 0.7975565195083618,
+      "learning_rate": 0.002949142524089853,
+      "loss": 1.6004766845703124,
+      "step": 3600
+    },
+    {
+      "epoch": 0.10084753372521119,
+      "grad_norm": 0.6167800426483154,
+      "learning_rate": 0.00293569518245131,
+      "loss": 1.60706298828125,
+      "step": 3650
+    },
+    {
+      "epoch": 0.10222900678994011,
+      "grad_norm": 0.6672324538230896,
+      "learning_rate": 0.0029207104624511555,
+      "loss": 1.6043368530273439,
+      "step": 3700
+    },
+    {
+      "epoch": 0.10361047985466904,
+      "grad_norm": 0.7184829115867615,
+      "learning_rate": 0.0029042044101028914,
+      "loss": 1.6044706726074218,
+      "step": 3750
+    },
+    {
+      "epoch": 0.10361047985466904,
+      "eval_loss": 1.608585238456726,
+      "eval_runtime": 4.872,
+      "eval_samples_per_second": 418.307,
+      "eval_steps_per_second": 3.284,
+      "step": 3750
+    },
+    {
+      "epoch": 0.10499195291939796,
+      "grad_norm": 0.6598625779151917,
+      "learning_rate": 0.002886194700500804,
+      "loss": 1.6044143676757812,
+      "step": 3800
+    },
+    {
+      "epoch": 0.10637342598412687,
+      "grad_norm": 0.7797738313674927,
+      "learning_rate": 0.002866700618893029,
+      "loss": 1.600813751220703,
+      "step": 3850
+    },
+    {
+      "epoch": 0.1077548990488558,
+      "grad_norm": 0.6865859031677246,
+      "learning_rate": 0.002845743040030426,
+      "loss": 1.6023855590820313,
+      "step": 3900
+    },
+    {
+      "epoch": 0.10913637211358472,
+      "grad_norm": 0.6436059474945068,
+      "learning_rate": 0.002823344405813371,
+      "loss": 1.6005615234375,
+      "step": 3950
+    },
+    {
+      "epoch": 0.11051784517831363,
+      "grad_norm": 0.6858559250831604,
+      "learning_rate": 0.0027995287012604077,
+      "loss": 1.5992294311523438,
+      "step": 4000
+    },
+    {
+      "epoch": 0.11051784517831363,
+      "eval_loss": 1.6017545461654663,
+      "eval_runtime": 4.9861,
+      "eval_samples_per_second": 408.736,
+      "eval_steps_per_second": 3.209,
+      "step": 4000
+    },
+    {
+      "epoch": 0.11189931824304256,
+      "grad_norm": 0.7086427807807922,
+      "learning_rate": 0.0027743214288244904,
+      "loss": 1.5969778442382812,
+      "step": 4050
+    },
+    {
+      "epoch": 0.11328079130777148,
+      "grad_norm": 0.6547996997833252,
+      "learning_rate": 0.002747749581084317,
+      "loss": 1.5946481323242188,
+      "step": 4100
+    },
+    {
+      "epoch": 0.11466226437250039,
+      "grad_norm": 0.6854317784309387,
+      "learning_rate": 0.0027198416118399986,
+      "loss": 1.594525146484375,
+      "step": 4150
+    },
+    {
+      "epoch": 0.11604373743722932,
+      "grad_norm": 0.8039860129356384,
+      "learning_rate": 0.0026906274056440215,
+      "loss": 1.6002627563476564,
+      "step": 4200
+    },
+    {
+      "epoch": 0.11742521050195824,
+      "grad_norm": 0.7050207257270813,
+      "learning_rate": 0.0026601382458001166,
+      "loss": 1.5937403869628906,
+      "step": 4250
+    },
+    {
+      "epoch": 0.11742521050195824,
+      "eval_loss": 1.5975710153579712,
+      "eval_runtime": 4.8014,
+      "eval_samples_per_second": 424.46,
+      "eval_steps_per_second": 3.332,
+      "step": 4250
+    },
+    {
+      "epoch": 0.11880668356668715,
+      "grad_norm": 0.6737009286880493,
+      "learning_rate": 0.0026284067808643144,
+      "loss": 1.5923365783691406,
+      "step": 4300
+    },
+    {
+      "epoch": 0.12018815663141608,
+      "grad_norm": 0.7063266634941101,
+      "learning_rate": 0.002595466989684055,
+      "loss": 1.5977363586425781,
+      "step": 4350
+    },
+    {
+      "epoch": 0.121569629696145,
+      "grad_norm": 0.6896716356277466,
+      "learning_rate": 0.0025613541450127857,
+      "loss": 1.5926210021972655,
+      "step": 4400
+    },
+    {
+      "epoch": 0.12295110276087393,
+      "grad_norm": 0.6772384643554688,
+      "learning_rate": 0.0025261047757390138,
+      "loss": 1.5924766540527344,
+      "step": 4450
+    },
+    {
+      "epoch": 0.12433257582560284,
+      "grad_norm": 0.6963664293289185,
+      "learning_rate": 0.002489756627770259,
+      "loss": 1.5876535034179688,
+      "step": 4500
+    },
+    {
+      "epoch": 0.12433257582560284,
+      "eval_loss": 1.5943443775177002,
+      "eval_runtime": 4.8657,
+      "eval_samples_per_second": 418.853,
+      "eval_steps_per_second": 3.288,
+      "step": 4500
+    },
+    {
+      "epoch": 0.12571404889033175,
+      "grad_norm": 0.7814108729362488,
+      "learning_rate": 0.002452348623613788,
+      "loss": 1.5891021728515624,
+      "step": 4550
+    },
+    {
+      "epoch": 0.1270955219550607,
+      "grad_norm": 0.6211883425712585,
+      "learning_rate": 0.002413920820697419,
+      "loss": 1.5855482482910157,
+      "step": 4600
+    },
+    {
+      "epoch": 0.1284769950197896,
+      "grad_norm": 0.6522833108901978,
+      "learning_rate": 0.0023745143684750264,
+      "loss": 1.5884809875488282,
+      "step": 4650
+    },
+    {
+      "epoch": 0.12985846808451854,
+      "grad_norm": 0.6828014254570007,
+      "learning_rate": 0.002334171464362675,
+      "loss": 1.5833425903320313,
+      "step": 4700
+    },
+    {
+      "epoch": 0.13123994114924745,
+      "grad_norm": 0.6603298783302307,
+      "learning_rate": 0.002292935308552567,
+      "loss": 1.583031005859375,
+      "step": 4750
+    },
+    {
+      "epoch": 0.13123994114924745,
+      "eval_loss": 1.5896168947219849,
+      "eval_runtime": 4.8861,
+      "eval_samples_per_second": 417.099,
+      "eval_steps_per_second": 3.275,
+      "step": 4750
+    },
+    {
+      "epoch": 0.13262141421397636,
+      "grad_norm": 0.6717214584350586,
+      "learning_rate": 0.002250850057753197,
+      "loss": 1.5836068725585937,
+      "step": 4800
+    },
+    {
+      "epoch": 0.1340028872787053,
+      "grad_norm": 0.6846370100975037,
+      "learning_rate": 0.002207960777905242,
+      "loss": 1.586727294921875,
+      "step": 4850
+    },
+    {
+      "epoch": 0.1353843603434342,
+      "grad_norm": 0.7362879514694214,
+      "learning_rate": 0.0021643133959238225,
+      "loss": 1.5847500610351561,
+      "step": 4900
+    },
+    {
+      "epoch": 0.13676583340816312,
+      "grad_norm": 0.7345172166824341,
+      "learning_rate": 0.0021199546505188105,
+      "loss": 1.5825830078125,
+      "step": 4950
+    },
+    {
+      "epoch": 0.13814730647289206,
+      "grad_norm": 0.5534242391586304,
+      "learning_rate": 0.0020749320421458535,
+      "loss": 1.5811091613769532,
+      "step": 5000
+    },
+    {
+      "epoch": 0.13814730647289206,
+      "eval_loss": 1.5833630561828613,
+      "eval_runtime": 4.9704,
+      "eval_samples_per_second": 410.03,
+      "eval_steps_per_second": 3.219,
+      "step": 5000
+    },
+    {
+      "epoch": 0.13952877953762097,
+      "grad_norm": 0.552155613899231,
+      "learning_rate": 0.002029293782141689,
+      "loss": 1.5763238525390626,
+      "step": 5050
+    },
+    {
+      "epoch": 0.14091025260234988,
+      "grad_norm": 0.7206612825393677,
+      "learning_rate": 0.001983088741098243,
+      "loss": 1.5772702026367187,
+      "step": 5100
+    },
+    {
+      "epoch": 0.14229172566707882,
+      "grad_norm": 0.6678832769393921,
+      "learning_rate": 0.001936366396530776,
+      "loss": 1.5752102661132812,
+      "step": 5150
+    },
+    {
+      "epoch": 0.14367319873180773,
+      "grad_norm": 0.6534181237220764,
+      "learning_rate": 0.0018891767798961177,
+      "loss": 1.5794488525390624,
+      "step": 5200
+    },
+    {
+      "epoch": 0.14505467179653664,
+      "grad_norm": 0.6262516379356384,
+      "learning_rate": 0.0018415704230177307,
+      "loss": 1.574145050048828,
+      "step": 5250
+    },
+    {
+      "epoch": 0.14505467179653664,
+      "eval_loss": 1.5795270204544067,
+      "eval_runtime": 4.8927,
+      "eval_samples_per_second": 416.542,
+      "eval_steps_per_second": 3.27,
+      "step": 5250
+    },
+    {
+      "epoch": 0.14643614486126558,
+      "grad_norm": 0.5793588757514954,
+      "learning_rate": 0.0017935983039749706,
+      "loss": 1.5746864318847655,
+      "step": 5300
+    },
+    {
+      "epoch": 0.1478176179259945,
+      "grad_norm": 0.6871621012687683,
+      "learning_rate": 0.0017453117925144783,
+      "loss": 1.5744833374023437,
+      "step": 5350
+    },
+    {
+      "epoch": 0.1491990909907234,
+      "grad_norm": 0.6331185102462769,
+      "learning_rate": 0.0016967625950421712,
+      "loss": 1.57496337890625,
+      "step": 5400
+    },
+    {
+      "epoch": 0.15058056405545234,
+      "grad_norm": 0.6986100673675537,
+      "learning_rate": 0.0016480026992547268,
+      "loss": 1.578210906982422,
+      "step": 5450
+    },
+    {
+      "epoch": 0.15196203712018125,
+      "grad_norm": 0.6834849715232849,
+      "learning_rate": 0.001599084318469858,
+      "loss": 1.5710919189453125,
+      "step": 5500
+    },
+    {
+      "epoch": 0.15196203712018125,
+      "eval_loss": 1.5749062299728394,
+      "eval_runtime": 4.8609,
+      "eval_samples_per_second": 419.261,
+      "eval_steps_per_second": 3.292,
+      "step": 5500
+    },
+    {
+      "epoch": 0.15334351018491016,
+      "grad_norm": 0.6833263039588928,
+      "learning_rate": 0.0015500598357149793,
+      "loss": 1.5740811157226562,
+      "step": 5550
+    },
+    {
+      "epoch": 0.1547249832496391,
+      "grad_norm": 0.8077707886695862,
+      "learning_rate": 0.001500981747634155,
+      "loss": 1.5700558471679686,
+      "step": 5600
+    },
+    {
+      "epoch": 0.156106456314368,
+      "grad_norm": 0.6584182977676392,
+      "learning_rate": 0.001451902608273374,
+      "loss": 1.5697401428222657,
+      "step": 5650
+    },
+    {
+      "epoch": 0.15748792937909692,
+      "grad_norm": 0.7573363184928894,
+      "learning_rate": 0.0014028749728043628,
+      "loss": 1.5668351745605469,
+      "step": 5700
+    },
+    {
+      "epoch": 0.15886940244382586,
+      "grad_norm": 0.6129063963890076,
+      "learning_rate": 0.0013539513412471889,
+      "loss": 1.570359344482422,
+      "step": 5750
+    },
+    {
+      "epoch": 0.15886940244382586,
+      "eval_loss": 1.5709928274154663,
+      "eval_runtime": 4.859,
+      "eval_samples_per_second": 419.431,
+      "eval_steps_per_second": 3.293,
+      "step": 5750
+    },
+    {
+      "epoch": 0.16025087550855477,
+      "grad_norm": 0.5028947591781616,
+      "learning_rate": 0.0013051841022519272,
+      "loss": 1.561181640625,
+      "step": 5800
+    },
+    {
+      "epoch": 0.16163234857328368,
+      "grad_norm": 0.7434332966804504,
+      "learning_rate": 0.0012566254769995806,
+      "loss": 1.5647598266601563,
+      "step": 5850
+    },
+    {
+      "epoch": 0.16301382163801262,
+      "grad_norm": 0.5033641457557678,
+      "learning_rate": 0.0012083274632823304,
+      "loss": 1.5645637512207031,
+      "step": 5900
+    },
+    {
+      "epoch": 0.16439529470274153,
+      "grad_norm": 0.5664800405502319,
+      "learning_rate": 0.0011603417798229966,
+      "loss": 1.567978515625,
+      "step": 5950
+    },
+    {
+      "epoch": 0.16577676776747047,
+      "grad_norm": 0.6270240545272827,
+      "learning_rate": 0.0011127198108933402,
+      "loss": 1.5611529541015625,
+      "step": 6000
+    },
+    {
+      "epoch": 0.16577676776747047,
+      "eval_loss": 1.5664807558059692,
+      "eval_runtime": 4.8532,
+      "eval_samples_per_second": 419.927,
+      "eval_steps_per_second": 3.297,
+      "step": 6000
+    },
+    {
+      "epoch": 0.16715824083219938,
+      "grad_norm": 0.5457693934440613,
+      "learning_rate": 0.0010655125512904898,
+      "loss": 1.5654832458496093,
+      "step": 6050
+    },
+    {
+      "epoch": 0.1685397138969283,
+      "grad_norm": 0.5332794785499573,
+      "learning_rate": 0.0010187705517304413,
+      "loss": 1.5635955810546875,
+      "step": 6100
+    },
+    {
+      "epoch": 0.16992118696165723,
+      "grad_norm": 0.5312942862510681,
+      "learning_rate": 0.0009725438647170831,
+      "loss": 1.5631723022460937,
+      "step": 6150
+    },
+    {
+      "epoch": 0.17130266002638614,
+      "grad_norm": 0.6611814498901367,
+      "learning_rate": 0.0009268819909447218,
+      "loss": 1.5617874145507813,
+      "step": 6200
+    },
+    {
+      "epoch": 0.17268413309111505,
+      "grad_norm": 0.5385921001434326,
+      "learning_rate": 0.000881833826291497,
+      "loss": 1.5621722412109376,
+      "step": 6250
+    },
+    {
+      "epoch": 0.17268413309111505,
+      "eval_loss": 1.5625797510147095,
+      "eval_runtime": 4.9424,
+      "eval_samples_per_second": 412.354,
+      "eval_steps_per_second": 3.237,
+      "step": 6250
+    },
+    {
+      "epoch": 0.174065606155844,
+      "grad_norm": 0.48100849986076355,
+      "learning_rate": 0.0008374476094604538,
+      "loss": 1.5620567321777343,
+      "step": 6300
+    },
+    {
+      "epoch": 0.1754470792205729,
+      "grad_norm": 0.5656850934028625,
+      "learning_rate": 0.0007937708703243286,
+      "loss": 1.5645301818847657,
+      "step": 6350
+    },
+    {
+      "epoch": 0.1768285522853018,
+      "grad_norm": 0.6062788367271423,
+      "learning_rate": 0.0007508503790293705,
+      "loss": 1.5537733459472656,
+      "step": 6400
+    },
+    {
+      "epoch": 0.17821002535003075,
+      "grad_norm": 0.5672012567520142,
+      "learning_rate": 0.0007087320959126999,
+      "loss": 1.559359130859375,
+      "step": 6450
+    },
+    {
+      "epoch": 0.17959149841475966,
+      "grad_norm": 0.5249131321907043,
+      "learning_rate": 0.0006674611222868254,
+      "loss": 1.5566461181640625,
+      "step": 6500
+    },
+    {
+      "epoch": 0.17959149841475966,
+      "eval_loss": 1.559727430343628,
+      "eval_runtime": 4.8874,
+      "eval_samples_per_second": 416.989,
+      "eval_steps_per_second": 3.274,
+      "step": 6500
+    },
+    {
+      "epoch": 0.18097297147948857,
+      "grad_norm": 0.5304768681526184,
+      "learning_rate": 0.000627081652144031,
+      "loss": 1.5591546630859374,
+      "step": 6550
+    },
+    {
+      "epoch": 0.1823544445442175,
+      "grad_norm": 0.5281299948692322,
+      "learning_rate": 0.0005876369248323417,
+      "loss": 1.5576934814453125,
+      "step": 6600
+    },
+    {
+      "epoch": 0.18373591760894642,
+      "grad_norm": 0.4922367334365845,
+      "learning_rate": 0.0005491691787537452,
+      "loss": 1.5613423156738282,
+      "step": 6650
+    },
+    {
+      "epoch": 0.18511739067367533,
+      "grad_norm": 0.6296632289886475,
+      "learning_rate": 0.000511719606134253,
+      "loss": 1.5597859191894532,
+      "step": 6700
+    },
+    {
+      "epoch": 0.18649886373840427,
+      "grad_norm": 0.49581730365753174,
+      "learning_rate": 0.00047532830891423806,
+      "loss": 1.5527120971679687,
+      "step": 6750
+    },
+    {
+      "epoch": 0.18649886373840427,
+      "eval_loss": 1.5563873052597046,
+      "eval_runtime": 4.8751,
+      "eval_samples_per_second": 418.047,
+      "eval_steps_per_second": 3.282,
+      "step": 6750
+    },
+    {
+      "epoch": 0.18788033680313318,
+      "grad_norm": 0.5099366307258606,
+      "learning_rate": 0.00044003425580626496,
+      "loss": 1.549066619873047,
+      "step": 6800
+    },
+    {
+      "epoch": 0.1892618098678621,
+      "grad_norm": 0.4683798551559448,
+      "learning_rate": 0.0004058752405664207,
+      "loss": 1.5529069519042968,
+      "step": 6850
+    },
+    {
+      "epoch": 0.19064328293259103,
+      "grad_norm": 0.5025731921195984,
+      "learning_rate": 0.0003728878415238149,
+      "loss": 1.5490866088867188,
+      "step": 6900
+    },
+    {
+      "epoch": 0.19202475599731994,
+      "grad_norm": 0.5808431506156921,
+      "learning_rate": 0.00034110738241158677,
+      "loss": 1.5539451599121095,
+      "step": 6950
+    },
+    {
+      "epoch": 0.19340622906204885,
+      "grad_norm": 0.5575580596923828,
+      "learning_rate": 0.0003105678945413668,
+      "loss": 1.5520620727539063,
+      "step": 7000
+    },
+    {
+      "epoch": 0.19340622906204885,
+      "eval_loss": 1.5543574094772339,
+      "eval_runtime": 4.8945,
+      "eval_samples_per_second": 416.387,
+      "eval_steps_per_second": 3.269,
+      "step": 7000
+    },
+    {
+      "epoch": 0.1947877021267778,
+      "grad_norm": 0.4817080497741699,
+      "learning_rate": 0.0002813020803616979,
+      "loss": 1.552979736328125,
+      "step": 7050
+    },
+    {
+      "epoch": 0.1961691751915067,
+      "grad_norm": 0.45744574069976807,
+      "learning_rate": 0.00025334127843943164,
+      "loss": 1.548302459716797,
+      "step": 7100
+    },
+    {
+      "epoch": 0.1975506482562356,
+      "grad_norm": 0.46788737177848816,
+      "learning_rate": 0.00022671542990160843,
+      "loss": 1.5542337036132812,
+      "step": 7150
+    },
+    {
+      "epoch": 0.19893212132096455,
+      "grad_norm": 0.44763970375061035,
+      "learning_rate": 0.00020145304637374757,
+      "loss": 1.5487118530273438,
+      "step": 7200
+    },
+    {
+      "epoch": 0.20031359438569346,
+      "grad_norm": 0.4314698576927185,
+      "learning_rate": 0.0001775811794488842,
+      "loss": 1.5508662414550782,
+      "step": 7250
+    },
+    {
+      "epoch": 0.20031359438569346,
+      "eval_loss": 1.551482915878296,
+      "eval_runtime": 4.8891,
+      "eval_samples_per_second": 416.845,
+      "eval_steps_per_second": 3.273,
+      "step": 7250
+    },
+    {
+      "epoch": 0.20169506745042237,
+      "grad_norm": 0.44681674242019653,
+      "learning_rate": 0.00015512539172004598,
+      "loss": 1.5449916076660157,
+      "step": 7300
+    },
+    {
+      "epoch": 0.2030765405151513,
+      "grad_norm": 0.4624103903770447,
+      "learning_rate": 0.00013410972940719012,
+      "loss": 1.5420948791503906,
+      "step": 7350
+    },
+    {
+      "epoch": 0.20445801357988022,
+      "grad_norm": 0.42456161975860596,
+      "learning_rate": 0.00011455669660790574,
+      "loss": 1.5426979064941406,
+      "step": 7400
+    },
+    {
+      "epoch": 0.20583948664460916,
+      "grad_norm": 0.43187159299850464,
+      "learning_rate": 9.648723119946411e-05,
+      "loss": 1.55113037109375,
+      "step": 7450
+    },
+    {
+      "epoch": 0.20722095970933807,
+      "grad_norm": 0.4058364927768707,
+      "learning_rate": 7.992068241801543e-05,
+      "loss": 1.5490814208984376,
+      "step": 7500
+    },
+    {
+      "epoch": 0.20722095970933807,
+      "eval_loss": 1.5499120950698853,
+      "eval_runtime": 4.6438,
+      "eval_samples_per_second": 438.865,
+      "eval_steps_per_second": 3.445,
+      "step": 7500
+    },
+    {
+      "epoch": 0.20860243277406698,
+      "grad_norm": 0.41115882992744446,
+      "learning_rate": 6.487479013894215e-05,
+      "loss": 1.53897705078125,
+      "step": 7550
+    },
+    {
+      "epoch": 0.20998390583879592,
+      "grad_norm": 0.40895798802375793,
+      "learning_rate": 5.1365665880554635e-05,
+      "loss": 1.5461245727539064,
+      "step": 7600
+    },
+    {
+      "epoch": 0.21136537890352483,
+      "grad_norm": 0.43710294365882874,
+      "learning_rate": 3.940777555147568e-05,
+      "loss": 1.5500086975097656,
+      "step": 7650
+    },
+    {
+      "epoch": 0.21274685196825374,
+      "grad_norm": 0.37500861287117004,
+      "learning_rate": 2.9013923960182153e-05,
+      "loss": 1.5431840515136719,
+      "step": 7700
+    },
+    {
+      "epoch": 0.21412832503298268,
+      "grad_norm": 0.4327012598514557,
+      "learning_rate": 2.019524110329557e-05,
+      "loss": 1.5467581176757812,
+      "step": 7750
+    },
+    {
+      "epoch": 0.21412832503298268,
+      "eval_loss": 1.548779010772705,
+      "eval_runtime": 5.0548,
+      "eval_samples_per_second": 403.183,
+      "eval_steps_per_second": 3.165,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2155097980977116,
+      "grad_norm": 0.3742528557777405,
+      "learning_rate": 1.2961170247303034e-05,
+      "loss": 1.543158721923828,
+      "step": 7800
+    },
+    {
+      "epoch": 0.2168912711624405,
+      "grad_norm": 0.39662519097328186,
+      "learning_rate": 7.319457816470754e-06,
+      "loss": 1.5452882385253905,
+      "step": 7850
+    },
+    {
+      "epoch": 0.21827274422716944,
+      "grad_norm": 0.439929336309433,
+      "learning_rate": 3.276145097779237e-06,
+      "loss": 1.54449462890625,
+      "step": 7900
+    },
+    {
+      "epoch": 0.21965421729189835,
+      "grad_norm": 0.3996962904930115,
+      "learning_rate": 8.355617717617503e-07,
+      "loss": 1.5445120239257812,
+      "step": 7950
+    },
+    {
+      "epoch": 0.22103569035662726,
+      "grad_norm": 0.3886605203151703,
+      "learning_rate": 3.2127617349830827e-10,
+      "loss": 1.541259765625,
+      "step": 8000
+    },
+    {
+      "epoch": 0.22103569035662726,
+      "eval_loss": 1.5484822988510132,
+      "eval_runtime": 10.2225,
+      "eval_samples_per_second": 199.364,
+      "eval_steps_per_second": 1.565,
+      "step": 8000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 8000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 110584922112000.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}