Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,225 @@
+---
+license: apache-2.0
+base_model: Qwen/Qwen3-8B
+tags:
+- coreference-resolution
+- ner
+- entity-extraction
+- qwen3
+- lora
+- peft
+datasets:
+- wjbmattingly/synthetic-coref
+language:
+- en
+pipeline_tag: text-generation
+---
+
+# Qwen3-8B-Coref-NER
+
+A fine-tuned [Qwen3-8B](Qwen/Qwen3-8B) model for **coreference resolution** and **named entity recognition**. 
+
+This model resolves pronouns and other referring expressions by replacing them with the full entity names, while also tracking entity mentions and their variants.
+
+## Model Description
+
+- **Base Model:** [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B)
+- **Training Dataset:** [wjbmattingly/synthetic-coref](https://huggingface.co/datasets/wjbmattingly/synthetic-coref)
+- **Task:** Coreference Resolution + Entity Tracking
+- **Method:** LoRA (Low-Rank Adaptation)
+
+## Usage
+
+### Installation
+
+```bash
+pip install transformers peft torch
+```
+
+### Quick Start
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+# Load base model and tokenizer
+base_model = "Qwen/Qwen3-8B"
+tokenizer = AutoTokenizer.from_pretrained(base_model)
+model = AutoModelForCausalLM.from_pretrained(
+    base_model,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+# Load LoRA adapter
+model = PeftModel.from_pretrained(model, "wjbmattingly/Qwen3-8B-Coref-NER")
+
+# Sample text
+text = """Alcuin of York was an Anglo-Latin scholar and teacher. He was born around 735 and became the student of Archbishop Ecgbert at York. At the invitation of Charlemagne, he became a leading scholar at the Carolingian court.
+
+In this role as adviser, he took issue with the emperor's policy of forcing pagans to be baptised on pain of death. His arguments seem to have prevailed – Charlemagne abolished the death penalty for paganism in 797."""
+
+# Create prompt
+prompt = "Resolve all pronouns in this text, replacing them with the full entity names. Also identify any entity references you find.\n\n" + text
+
+messages = [{"role": "user", "content": prompt}]
+input_text = tokenizer.apply_chat_template(
+    messages, 
+    tokenize=False, 
+    add_generation_prompt=True,
+    enable_thinking=False  # Disable thinking mode
+)
+
+# Generate
+inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
+with torch.no_grad():
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=2048,
+        do_sample=False,
+        pad_token_id=tokenizer.pad_token_id,
+    )
+
+response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
+print(response)
+```
+
+### Paragraph-by-Paragraph Processing with Entity Tracking
+
+For longer documents, process paragraph by paragraph while tracking entities:
+
+```python
+import torch
+import re
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+def parse_entity_mappings(response):
+    """Parse model response to extract resolved text and entity mappings."""
+    if "NEW ENTITY MAPPINGS:" in response:
+        parts = response.split("NEW ENTITY MAPPINGS:")
+        resolved_text = parts[0].strip()
+        mappings_text = parts[1].strip() if len(parts) > 1 else ""
+        
+        entities = {}
+        for line in mappings_text.split("\n"):
+            line = line.strip()
+            if line.startswith("-"):
+                match = re.match(r'-\s*([^:]+):\s*\[([^\]]*)\]', line)
+                if match:
+                    entity_name = match.group(1).strip()
+                    variants = re.findall(r'"([^"]*)"', match.group(2))
+                    if variants:
+                        entities[entity_name] = variants
+        return resolved_text, entities
+    return response.strip(), {}
+
+def format_entities_for_prompt(entities):
+    """Format known entities for the prompt."""
+    lines = ["Entities and their possible references:"]
+    for entity_name, variants in entities.items():
+        variants_str = ", ".join(f'"{v}"' for v in variants)
+        lines.append(f"- {entity_name}: [{variants_str}]")
+    return "\n".join(lines)
+
+# Load model
+base_model = "Qwen/Qwen3-8B"
+tokenizer = AutoTokenizer.from_pretrained(base_model)
+model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=torch.bfloat16, device_map="auto")
+model = PeftModel.from_pretrained(model, "wjbmattingly/Qwen3-8B-Coref-NER")
+
+# Your document
+text = """Alcuin of York was an Anglo-Latin scholar and teacher. He was born around 735 and became the student of Archbishop Ecgbert at York. At the invitation of Charlemagne, he became a leading scholar at the Carolingian court.
+
+In this role as adviser, he took issue with the emperor's policy of forcing pagans to be baptised on pain of death. His arguments seem to have prevailed – Charlemagne abolished the death penalty for paganism in 797."""
+
+# Split into paragraphs
+paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+resolved_paragraphs = []
+cumulative_entities = {}
+
+for i, paragraph in enumerate(paragraphs):
+    print(f"Processing paragraph {i+1}/{len(paragraphs)}...")
+    
+    # Build prompt
+    if i == 0:
+        prompt = f"Resolve all pronouns in this text, replacing them with the full entity names. Also identify any entity references you find.\n\n{paragraph}"
+    else:
+        context = "\n\n".join(resolved_paragraphs[max(0, i-2):i])
+        if cumulative_entities:
+            known_str = format_entities_for_prompt(cumulative_entities)
+            prompt = f"Known {known_str}\n\nGiven this context of preceding text (already resolved):\n\n{context}\n\nResolve all pronouns in this paragraph using the known entities. Also identify any NEW entity references:\n\n{paragraph}"
+        else:
+            prompt = f"Given this context of preceding text (already resolved):\n\n{context}\n\nResolve all pronouns in this paragraph. Also identify any NEW entity references:\n\n{paragraph}"
+    
+    messages = [{"role": "user", "content": prompt}]
+    input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+    inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
+    
+    with torch.no_grad():
+        outputs = model.generate(**inputs, max_new_tokens=2048, do_sample=False, pad_token_id=tokenizer.pad_token_id)
+    
+    response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip()
+    
+    # Parse response
+    resolved, new_entities = parse_entity_mappings(response)
+    resolved_paragraphs.append(resolved)
+    
+    # Update cumulative entities
+    for entity_name, variants in new_entities.items():
+        if entity_name not in cumulative_entities:
+            cumulative_entities[entity_name] = set()
+        cumulative_entities[entity_name].update(variants)
+    
+    if new_entities:
+        print(f"  New entities found: {new_entities}")
+
+# Final output
+print("\n" + "="*50)
+print("RESOLVED TEXT:")
+print("="*50)
+print("\n\n".join(resolved_paragraphs))
+
+print("\n" + "="*50)
+print("ALL ENTITIES:")
+print("="*50)
+for entity, variants in cumulative_entities.items():
+    print(f"  {entity}: {list(variants)}")
+```
+
+## Sample Output
+
+**Input:**
+```
+Alcuin of York was an Anglo-Latin scholar and teacher. He was born around 735 and became the student of Archbishop Ecgbert at York. At the invitation of Charlemagne, he became a leading scholar at the Carolingian court.
+
+In this role as adviser, he took issue with the emperor's policy of forcing pagans to be baptised on pain of death. His arguments seem to have prevailed – Charlemagne abolished the death penalty for paganism in 797.
+```
+
+**Output:**
+```
+Alcuin of York was an Anglo-Latin scholar and teacher. Alcuin of York was born around 735 and became the student of Archbishop Ecgbert at York. At the invitation of Charlemagne, Alcuin of York became a leading scholar at the Carolingian court.
+
+In Alcuin of York's role as adviser, Alcuin of York took issue with Charlemagne's policy of forcing pagans to be baptised on pain of death. Alcuin of York's arguments seem to have prevailed – Charlemagne abolished the death penalty for paganism in 797.
+
+NEW ENTITY MAPPINGS:
+- Alcuin of York: ["He", "his", "he"]
+- Charlemagne: ["the emperor"]
+```
+
+## Training Details
+
+This model was trained using:
+- **LoRA rank:** 16
+- **LoRA alpha:** 32
+- **Target modules:** q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj
+- **Training mode:** Paragraph-by-paragraph with progressive entity tracking
+
+## Citation
+
+If you use this model, please cite the training dataset and base model.
+
+## License
+
+Apache 2.0

adapter_config.json

@@ -0,0 +1,49 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-8B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922a160cecccc7ed4ae8eadd43850c87fba78782b64483034ce76bb00d9cb307
+size 2663975192

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

inference.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""
+Coreference Resolution with Qwen3-8B-Coref-NER
+
+Usage:
+    uv run inference.py input.txt
+    uv run inference.py input.txt --output resolved.txt
+    uv run inference.py input.txt --html report.html
+"""
+
+import argparse
+import torch
+import re
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+MODEL_ID = "wjbmattingly/Qwen3-8B-Coref-NER"
+BASE_MODEL = "Qwen/Qwen3-8B"
+
+
+def parse_entity_mappings(response):
+    """Parse model response to extract resolved text and entity mappings."""
+    if "NEW ENTITY MAPPINGS:" in response:
+        parts = response.split("NEW ENTITY MAPPINGS:")
+        resolved_text = parts[0].strip()
+        mappings_text = parts[1].strip() if len(parts) > 1 else ""
+        
+        entities = {}
+        for line in mappings_text.split("\n"):
+            line = line.strip()
+            if line.startswith("-"):
+                match = re.match(r'-\s*([^:]+):\s*\[([^\]]*)\]', line)
+                if match:
+                    entity_name = match.group(1).strip()
+                    variants = re.findall(r'"([^"]*)"', match.group(2))
+                    if variants:
+                        entities[entity_name] = variants
+        return resolved_text, entities
+    return response.strip(), {}
+
+
+def format_entities_for_prompt(entities):
+    """Format known entities for the prompt."""
+    lines = ["Entities and their possible references:"]
+    for entity_name, variants in entities.items():
+        variants_str = ", ".join(f'"{v}"' for v in variants)
+        lines.append(f"- {entity_name}: [{variants_str}]")
+    return "\n".join(lines)
+
+
+def load_model():
+    """Load the model and tokenizer."""
+    print(f"Loading {BASE_MODEL}...")
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+    )
+    print(f"Loading adapter {MODEL_ID}...")
+    model = PeftModel.from_pretrained(model, MODEL_ID)
+    return model, tokenizer
+
+
+def process_document(text, model, tokenizer, max_new_tokens=2048):
+    """Process document paragraph by paragraph with entity tracking."""
+    paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+    resolved_paragraphs = []
+    paragraph_entities = []
+    cumulative_entities = {}
+    
+    for i, paragraph in enumerate(paragraphs):
+        print(f"Processing paragraph {i+1}/{len(paragraphs)}...")
+        
+        # Build prompt
+        if i == 0:
+            prompt = f"Resolve all pronouns in this text, replacing them with the full entity names. Also identify any entity references you find.\n\n{paragraph}"
+        else:
+            context = "\n\n".join(resolved_paragraphs[max(0, i-2):i])
+            if cumulative_entities:
+                known_str = format_entities_for_prompt(cumulative_entities)
+                prompt = f"Known {known_str}\n\nGiven this context of preceding text (already resolved):\n\n{context}\n\nResolve all pronouns in this paragraph using the known entities. Also identify any NEW entity references:\n\n{paragraph}"
+            else:
+                prompt = f"Given this context of preceding text (already resolved):\n\n{context}\n\nResolve all pronouns in this paragraph. Also identify any NEW entity references:\n\n{paragraph}"
+        
+        messages = [{"role": "user", "content": prompt}]
+        input_text = tokenizer.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
+        )
+        inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
+        
+        with torch.no_grad():
+            outputs = model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                do_sample=False,
+                pad_token_id=tokenizer.pad_token_id,
+            )
+        
+        response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip()
+        
+        # Clean up
+        response = re.sub(r'<think>.*?</think>\s*', '', response, flags=re.DOTALL)
+        
+        # Parse response
+        resolved, new_entities = parse_entity_mappings(response)
+        resolved_paragraphs.append(resolved)
+        paragraph_entities.append(new_entities)
+        
+        # Update cumulative entities
+        for entity_name, variants in new_entities.items():
+            if entity_name not in cumulative_entities:
+                cumulative_entities[entity_name] = set()
+            cumulative_entities[entity_name].update(variants)
+        
+        if new_entities:
+            print(f"  New entities: {new_entities}")
+    
+    # Convert sets to lists
+    cumulative_entities = {k: list(v) for k, v in cumulative_entities.items()}
+    
+    return {
+        "paragraphs": paragraphs,
+        "resolved_paragraphs": resolved_paragraphs,
+        "paragraph_entities": paragraph_entities,
+        "cumulative_entities": cumulative_entities,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Coreference Resolution")
+    parser.add_argument("input", help="Input text file")
+    parser.add_argument("-o", "--output", help="Output file for resolved text")
+    parser.add_argument("--max-tokens", type=int, default=2048, help="Max tokens to generate")
+    args = parser.parse_args()
+    
+    # Load input
+    with open(args.input, "r") as f:
+        text = f.read().strip()
+    
+    # Load model
+    model, tokenizer = load_model()
+    
+    # Process
+    result = process_document(text, model, tokenizer, args.max_tokens)
+    
+    # Output
+    resolved_text = "\n\n".join(result["resolved_paragraphs"])
+    
+    print("\n" + "="*60)
+    print("RESOLVED TEXT:")
+    print("="*60)
+    print(resolved_text)
+    
+    print("\n" + "="*60)
+    print("ENTITIES FOUND:")
+    print("="*60)
+    for entity, variants in result["cumulative_entities"].items():
+        print(f"  {entity}: {variants}")
+    
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(resolved_text)
+        print(f"\nSaved to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34357c2e564432b2cb4a560dd6b43843e1b736e4dc134193a1e1cebb5f215f96
+size 5328249521

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e8573ad950caaf8d31c70708b6c77b84a02a0a6dfe1ba2103a628dbf8264866
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5b940bf4670a51654bd371c9c5f586938ff842092e681c496849b0ebb572130
+size 1465

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,684 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.6113796950539234,
+  "eval_steps": 500,
+  "global_step": 6500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.9991093099117279,
+      "epoch": 0.024792363951902815,
+      "grad_norm": 0.7828091382980347,
+      "learning_rate": 0.00019800000000000002,
+      "loss": 1.2105257415771484,
+      "mean_token_accuracy": 0.7592608261108399,
+      "num_tokens": 428432.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.4560545237362385,
+      "epoch": 0.04958472790380563,
+      "grad_norm": 0.6142551898956299,
+      "learning_rate": 0.00019996642557446167,
+      "loss": 0.4457200241088867,
+      "mean_token_accuracy": 0.9013872227072716,
+      "num_tokens": 860175.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.2749050757288933,
+      "epoch": 0.07437709185570844,
+      "grad_norm": 0.5504621267318726,
+      "learning_rate": 0.0001998643654103928,
+      "loss": 0.26112810134887693,
+      "mean_token_accuracy": 0.9401605477929116,
+      "num_tokens": 1288393.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.18620084561407566,
+      "epoch": 0.09916945580761126,
+      "grad_norm": 0.49035999178886414,
+      "learning_rate": 0.00019969388600561932,
+      "loss": 0.168951416015625,
+      "mean_token_accuracy": 0.9586734220385551,
+      "num_tokens": 1697950.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.13362589346244932,
+      "epoch": 0.12396181975951406,
+      "grad_norm": 0.332853227853775,
+      "learning_rate": 0.00019945510415927565,
+      "loss": 0.1165453052520752,
+      "mean_token_accuracy": 0.9707631063461304,
+      "num_tokens": 2121801.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.114101482629776,
+      "epoch": 0.14875418371141688,
+      "grad_norm": 0.25856056809425354,
+      "learning_rate": 0.00019914818346597883,
+      "loss": 0.09927322387695313,
+      "mean_token_accuracy": 0.9743164613842964,
+      "num_tokens": 2557586.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.11320277975872159,
+      "epoch": 0.1735465476633197,
+      "grad_norm": 0.20196805894374847,
+      "learning_rate": 0.00019877333420374656,
+      "loss": 0.09849776268005371,
+      "mean_token_accuracy": 0.974927342236042,
+      "num_tokens": 2987323.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.10322847299277782,
+      "epoch": 0.19833891161522252,
+      "grad_norm": 0.2440531700849533,
+      "learning_rate": 0.00019833081318993095,
+      "loss": 0.09059928894042969,
+      "mean_token_accuracy": 0.9767567777633667,
+      "num_tokens": 3409712.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.09775902647525073,
+      "epoch": 0.22313127556712534,
+      "grad_norm": 0.3308379054069519,
+      "learning_rate": 0.00019782092360526762,
+      "loss": 0.08334708213806152,
+      "mean_token_accuracy": 0.9783057284355163,
+      "num_tokens": 3841892.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.09744730332866311,
+      "epoch": 0.24792363951902813,
+      "grad_norm": 0.2839490473270416,
+      "learning_rate": 0.00019724401478615975,
+      "loss": 0.08426895141601562,
+      "mean_token_accuracy": 0.9779949200153351,
+      "num_tokens": 4270558.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.09355678971856833,
+      "epoch": 0.27271600347093095,
+      "grad_norm": 0.22099336981773376,
+      "learning_rate": 0.00019660048198534043,
+      "loss": 0.07872037887573242,
+      "mean_token_accuracy": 0.9789282914996147,
+      "num_tokens": 4692785.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.08774024970829487,
+      "epoch": 0.29750836742283376,
+      "grad_norm": 0.25038641691207886,
+      "learning_rate": 0.00019589076610107613,
+      "loss": 0.07532472610473633,
+      "mean_token_accuracy": 0.9797207751870155,
+      "num_tokens": 5123896.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.0822139031253755,
+      "epoch": 0.3223007313747366,
+      "grad_norm": 0.1757524311542511,
+      "learning_rate": 0.0001951153533750981,
+      "loss": 0.0712879753112793,
+      "mean_token_accuracy": 0.9809118565917015,
+      "num_tokens": 5558585.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.08247557899914681,
+      "epoch": 0.3470930953266394,
+      "grad_norm": 0.3174690902233124,
+      "learning_rate": 0.00019427477505946724,
+      "loss": 0.07155597686767579,
+      "mean_token_accuracy": 0.9806134033203125,
+      "num_tokens": 5993135.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.07734406786970795,
+      "epoch": 0.3718854592785422,
+      "grad_norm": 0.20085565745830536,
+      "learning_rate": 0.00019336960705260218,
+      "loss": 0.06758695125579833,
+      "mean_token_accuracy": 0.9812126770615578,
+      "num_tokens": 6422291.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.07385200836695731,
+      "epoch": 0.39667782323044504,
+      "grad_norm": 0.23364907503128052,
+      "learning_rate": 0.00019240046950471843,
+      "loss": 0.06287895202636719,
+      "mean_token_accuracy": 0.9826329717040062,
+      "num_tokens": 6859860.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.07059250806458295,
+      "epoch": 0.42147018718234786,
+      "grad_norm": 0.23875373601913452,
+      "learning_rate": 0.00019136802639295038,
+      "loss": 0.06304484367370605,
+      "mean_token_accuracy": 0.9824053046107292,
+      "num_tokens": 7290073.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.06737875854596495,
+      "epoch": 0.4462625511342507,
+      "grad_norm": 0.3208062946796417,
+      "learning_rate": 0.00019027298506644585,
+      "loss": 0.059958081245422366,
+      "mean_token_accuracy": 0.9828743213415145,
+      "num_tokens": 7725201.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.06706910256296396,
+      "epoch": 0.4710549150861535,
+      "grad_norm": 0.21684925258159637,
+      "learning_rate": 0.0001891160957617461,
+      "loss": 0.058564453125,
+      "mean_token_accuracy": 0.9831320327520371,
+      "num_tokens": 8158313.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.06947937468066812,
+      "epoch": 0.49584727903805625,
+      "grad_norm": 0.18770049512386322,
+      "learning_rate": 0.00018789815108878247,
+      "loss": 0.06172544956207275,
+      "mean_token_accuracy": 0.9823081752657891,
+      "num_tokens": 8578452.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.06397987404838204,
+      "epoch": 0.5206396429899591,
+      "grad_norm": 0.3015916347503662,
+      "learning_rate": 0.00018661998548784206,
+      "loss": 0.05666701316833496,
+      "mean_token_accuracy": 0.9837381663918495,
+      "num_tokens": 9008577.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.06192670753225684,
+      "epoch": 0.5454320069418619,
+      "grad_norm": 0.14773066341876984,
+      "learning_rate": 0.00018528247465787458,
+      "loss": 0.05475767612457275,
+      "mean_token_accuracy": 0.9839018094539642,
+      "num_tokens": 9442416.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.057383716171607375,
+      "epoch": 0.5702243708937648,
+      "grad_norm": 0.2786010503768921,
+      "learning_rate": 0.00018388653495653227,
+      "loss": 0.05079104423522949,
+      "mean_token_accuracy": 0.9851340425014495,
+      "num_tokens": 9866601.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.061573395412415265,
+      "epoch": 0.5950167348456675,
+      "grad_norm": 0.2154165506362915,
+      "learning_rate": 0.00018243312277235307,
+      "loss": 0.052988133430480956,
+      "mean_token_accuracy": 0.9846190690994263,
+      "num_tokens": 10287014.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.061048444965854286,
+      "epoch": 0.6198090987975704,
+      "grad_norm": 0.23032571375370026,
+      "learning_rate": 0.00018092323386951844,
+      "loss": 0.05165313720703125,
+      "mean_token_accuracy": 0.9843217238783837,
+      "num_tokens": 10717509.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.055967804118990896,
+      "epoch": 0.6446014627494732,
+      "grad_norm": 0.19078344106674194,
+      "learning_rate": 0.00017935790270563344,
+      "loss": 0.04785560607910156,
+      "mean_token_accuracy": 0.9853029519319534,
+      "num_tokens": 11157747.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.05705006473697722,
+      "epoch": 0.6693938267013759,
+      "grad_norm": 0.18230348825454712,
+      "learning_rate": 0.0001777382017229977,
+      "loss": 0.04942546844482422,
+      "mean_token_accuracy": 0.9849184802174569,
+      "num_tokens": 11593422.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.05708774453029036,
+      "epoch": 0.6941861906532788,
+      "grad_norm": 0.1923963725566864,
+      "learning_rate": 0.00017606524061385167,
+      "loss": 0.049316325187683106,
+      "mean_token_accuracy": 0.9849100703001022,
+      "num_tokens": 12015482.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.056122053125873206,
+      "epoch": 0.7189785546051816,
+      "grad_norm": 0.13828027248382568,
+      "learning_rate": 0.00017434016556010264,
+      "loss": 0.048600535392761234,
+      "mean_token_accuracy": 0.9853047552704811,
+      "num_tokens": 12444156.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.051999541874974964,
+      "epoch": 0.7437709185570844,
+      "grad_norm": 0.24286241829395294,
+      "learning_rate": 0.00017256415844805063,
+      "loss": 0.04440320014953613,
+      "mean_token_accuracy": 0.9864806136488915,
+      "num_tokens": 12876720.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.04768232893198729,
+      "epoch": 0.7685632825089872,
+      "grad_norm": 0.1362326294183731,
+      "learning_rate": 0.00017073843605865243,
+      "loss": 0.042284860610961914,
+      "mean_token_accuracy": 0.9867907965183258,
+      "num_tokens": 13316689.0,
+      "step": 3100
+    },
+    {
+      "entropy": 0.05134817799553275,
+      "epoch": 0.7933556464608901,
+      "grad_norm": 0.1539483368396759,
+      "learning_rate": 0.00016886424923387859,
+      "loss": 0.04517318725585937,
+      "mean_token_accuracy": 0.9861972403526306,
+      "num_tokens": 13745299.0,
+      "step": 3200
+    },
+    {
+      "entropy": 0.051489096255972984,
+      "epoch": 0.8181480104127928,
+      "grad_norm": 0.14677491784095764,
+      "learning_rate": 0.00016694288201973457,
+      "loss": 0.04445661067962647,
+      "mean_token_accuracy": 0.9860166025161743,
+      "num_tokens": 14174424.0,
+      "step": 3300
+    },
+    {
+      "entropy": 0.04676903900690377,
+      "epoch": 0.8429403743646957,
+      "grad_norm": 0.21468833088874817,
+      "learning_rate": 0.0001649756507865328,
+      "loss": 0.041012239456176755,
+      "mean_token_accuracy": 0.987050573527813,
+      "num_tokens": 14611781.0,
+      "step": 3400
+    },
+    {
+      "entropy": 0.0482857808098197,
+      "epoch": 0.8677327383165985,
+      "grad_norm": 0.14730653166770935,
+      "learning_rate": 0.00016296390332701924,
+      "loss": 0.04293826103210449,
+      "mean_token_accuracy": 0.9866457509994507,
+      "num_tokens": 15038473.0,
+      "step": 3500
+    },
+    {
+      "entropy": 0.04790851186960936,
+      "epoch": 0.8925251022685013,
+      "grad_norm": 0.2903175950050354,
+      "learning_rate": 0.0001609090179329709,
+      "loss": 0.041955056190490725,
+      "mean_token_accuracy": 0.9864692252874374,
+      "num_tokens": 15470046.0,
+      "step": 3600
+    },
+    {
+      "entropy": 0.048887748084962365,
+      "epoch": 0.9173174662204041,
+      "grad_norm": 0.18117748200893402,
+      "learning_rate": 0.0001588124024508986,
+      "loss": 0.04415608406066895,
+      "mean_token_accuracy": 0.985989620089531,
+      "num_tokens": 15892623.0,
+      "step": 3700
+    },
+    {
+      "entropy": 0.04632032319903374,
+      "epoch": 0.942109830172307,
+      "grad_norm": 0.16654178500175476,
+      "learning_rate": 0.00015667549331750067,
+      "loss": 0.042152628898620606,
+      "mean_token_accuracy": 0.9865523239970208,
+      "num_tokens": 16316100.0,
+      "step": 3800
+    },
+    {
+      "entropy": 0.044644428994506595,
+      "epoch": 0.9669021941242097,
+      "grad_norm": 0.2481166422367096,
+      "learning_rate": 0.0001544997545755291,
+      "loss": 0.04001152038574219,
+      "mean_token_accuracy": 0.9875221633911133,
+      "num_tokens": 16747053.0,
+      "step": 3900
+    },
+    {
+      "entropy": 0.04754056423902511,
+      "epoch": 0.9916945580761125,
+      "grad_norm": 0.12741857767105103,
+      "learning_rate": 0.00015228667687074214,
+      "loss": 0.04207493782043457,
+      "mean_token_accuracy": 0.9866985288262368,
+      "num_tokens": 17169662.0,
+      "step": 4000
+    },
+    {
+      "entropy": 0.04459555751302434,
+      "epoch": 1.0163629602082558,
+      "grad_norm": 0.09805303066968918,
+      "learning_rate": 0.00015003777643063054,
+      "loss": 0.03870807647705078,
+      "mean_token_accuracy": 0.9877826987798489,
+      "num_tokens": 17596478.0,
+      "step": 4100
+    },
+    {
+      "entropy": 0.0423293265234679,
+      "epoch": 1.0411553241601588,
+      "grad_norm": 0.15998226404190063,
+      "learning_rate": 0.0001477545940256172,
+      "loss": 0.03742676258087158,
+      "mean_token_accuracy": 0.9879223209619522,
+      "num_tokens": 18017906.0,
+      "step": 4200
+    },
+    {
+      "entropy": 0.03891188260167837,
+      "epoch": 1.0659476881120615,
+      "grad_norm": 0.1506214290857315,
+      "learning_rate": 0.00014543869391344224,
+      "loss": 0.03518922805786133,
+      "mean_token_accuracy": 0.9887040394544602,
+      "num_tokens": 18454216.0,
+      "step": 4300
+    },
+    {
+      "entropy": 0.04142872649244964,
+      "epoch": 1.0907400520639643,
+      "grad_norm": 0.1666085124015808,
+      "learning_rate": 0.00014309166276745564,
+      "loss": 0.03670289039611816,
+      "mean_token_accuracy": 0.9879066833853721,
+      "num_tokens": 18878519.0,
+      "step": 4400
+    },
+    {
+      "entropy": 0.03755377263762057,
+      "epoch": 1.115532416015867,
+      "grad_norm": 0.2086615115404129,
+      "learning_rate": 0.00014071510858955336,
+      "loss": 0.03333067655563354,
+      "mean_token_accuracy": 0.989110766351223,
+      "num_tokens": 19317824.0,
+      "step": 4500
+    },
+    {
+      "entropy": 0.04046432740055025,
+      "epoch": 1.14032477996777,
+      "grad_norm": 0.20940445363521576,
+      "learning_rate": 0.00013831065960850038,
+      "loss": 0.03586224794387818,
+      "mean_token_accuracy": 0.9884641316533088,
+      "num_tokens": 19745984.0,
+      "step": 4600
+    },
+    {
+      "entropy": 0.03863211246207356,
+      "epoch": 1.1651171439196728,
+      "grad_norm": 0.1907690167427063,
+      "learning_rate": 0.0001358799631643955,
+      "loss": 0.03435335397720337,
+      "mean_token_accuracy": 0.9888206848502159,
+      "num_tokens": 20180245.0,
+      "step": 4700
+    },
+    {
+      "entropy": 0.04046371933072805,
+      "epoch": 1.1899095078715756,
+      "grad_norm": 0.13110345602035522,
+      "learning_rate": 0.00013342468458004347,
+      "loss": 0.03644887685775757,
+      "mean_token_accuracy": 0.9881230041384697,
+      "num_tokens": 20606265.0,
+      "step": 4800
+    },
+    {
+      "entropy": 0.03736146904528141,
+      "epoch": 1.2147018718234783,
+      "grad_norm": 0.1462518870830536,
+      "learning_rate": 0.00013094650602000582,
+      "loss": 0.03372693538665771,
+      "mean_token_accuracy": 0.9891441905498505,
+      "num_tokens": 21044494.0,
+      "step": 4900
+    },
+    {
+      "entropy": 0.03839201644994319,
+      "epoch": 1.239494235775381,
+      "grad_norm": 0.11341901123523712,
+      "learning_rate": 0.00012844712533811376,
+      "loss": 0.03437334060668945,
+      "mean_token_accuracy": 0.9888800847530365,
+      "num_tokens": 21480042.0,
+      "step": 5000
+    },
+    {
+      "entropy": 0.037934175664559006,
+      "epoch": 1.264286599727284,
+      "grad_norm": 0.23413598537445068,
+      "learning_rate": 0.0001259282549142315,
+      "loss": 0.03376242399215698,
+      "mean_token_accuracy": 0.9890937665104866,
+      "num_tokens": 21912071.0,
+      "step": 5100
+    },
+    {
+      "entropy": 0.03757761628367007,
+      "epoch": 1.2890789636791868,
+      "grad_norm": 0.14774999022483826,
+      "learning_rate": 0.0001233916204810679,
+      "loss": 0.03416965961456299,
+      "mean_token_accuracy": 0.9889910009503364,
+      "num_tokens": 22335448.0,
+      "step": 5200
+    },
+    {
+      "entropy": 0.03822989201173186,
+      "epoch": 1.3138713276310896,
+      "grad_norm": 0.11656388640403748,
+      "learning_rate": 0.00012083895994183948,
+      "loss": 0.03509602785110474,
+      "mean_token_accuracy": 0.9885432028770447,
+      "num_tokens": 22756521.0,
+      "step": 5300
+    },
+    {
+      "entropy": 0.03775340311229229,
+      "epoch": 1.3386636915829924,
+      "grad_norm": 0.1481839120388031,
+      "learning_rate": 0.00011827202217959523,
+      "loss": 0.0340172815322876,
+      "mean_token_accuracy": 0.9888760358095169,
+      "num_tokens": 23181534.0,
+      "step": 5400
+    },
+    {
+      "entropy": 0.03720983518287539,
+      "epoch": 1.3634560555348951,
+      "grad_norm": 0.13698537647724152,
+      "learning_rate": 0.00011569256585901918,
+      "loss": 0.03371585130691528,
+      "mean_token_accuracy": 0.9890251788496971,
+      "num_tokens": 23613467.0,
+      "step": 5500
+    },
+    {
+      "entropy": 0.037667012261226776,
+      "epoch": 1.3882484194867981,
+      "grad_norm": 0.09335887432098389,
+      "learning_rate": 0.00011310235822153073,
+      "loss": 0.03394256591796875,
+      "mean_token_accuracy": 0.9890615126490593,
+      "num_tokens": 24039912.0,
+      "step": 5600
+    },
+    {
+      "entropy": 0.03788006491959095,
+      "epoch": 1.4130407834387009,
+      "grad_norm": 0.1678655594587326,
+      "learning_rate": 0.0001105031738745096,
+      "loss": 0.034035165309906,
+      "mean_token_accuracy": 0.9887340119481087,
+      "num_tokens": 24464996.0,
+      "step": 5700
+    },
+    {
+      "entropy": 0.03637797945179045,
+      "epoch": 1.4378331473906036,
+      "grad_norm": 0.17894072830677032,
+      "learning_rate": 0.00010789679357547364,
+      "loss": 0.03286903142929077,
+      "mean_token_accuracy": 0.9891549465060234,
+      "num_tokens": 24901092.0,
+      "step": 5800
+    },
+    {
+      "entropy": 0.03953015809878707,
+      "epoch": 1.4626255113425066,
+      "grad_norm": 0.1642547845840454,
+      "learning_rate": 0.00010528500301204345,
+      "loss": 0.03447899341583252,
+      "mean_token_accuracy": 0.988511001765728,
+      "num_tokens": 25320711.0,
+      "step": 5900
+    },
+    {
+      "entropy": 0.03696547055616975,
+      "epoch": 1.4874178752944094,
+      "grad_norm": 0.15598466992378235,
+      "learning_rate": 0.0001026695915785289,
+      "loss": 0.03327722549438476,
+      "mean_token_accuracy": 0.9893461042642593,
+      "num_tokens": 25750634.0,
+      "step": 6000
+    },
+    {
+      "entropy": 0.03747725712135434,
+      "epoch": 1.5122102392463121,
+      "grad_norm": 0.09570539742708206,
+      "learning_rate": 0.00010005235114997651,
+      "loss": 0.03426354885101318,
+      "mean_token_accuracy": 0.9889142262935638,
+      "num_tokens": 26175757.0,
+      "step": 6100
+    },
+    {
+      "entropy": 0.037917951373383406,
+      "epoch": 1.537002603198215,
+      "grad_norm": 0.2064983993768692,
+      "learning_rate": 9.743507485451688e-05,
+      "loss": 0.03446398973464966,
+      "mean_token_accuracy": 0.9886316114664078,
+      "num_tokens": 26601186.0,
+      "step": 6200
+    },
+    {
+      "entropy": 0.037627564212307334,
+      "epoch": 1.5617949671501177,
+      "grad_norm": 0.17442478239536285,
+      "learning_rate": 9.48195558448538e-05,
+      "loss": 0.0337642502784729,
+      "mean_token_accuracy": 0.988581260740757,
+      "num_tokens": 27028439.0,
+      "step": 6300
+    },
+    {
+      "entropy": 0.035229697693139314,
+      "epoch": 1.5865873311020207,
+      "grad_norm": 0.19224514067173004,
+      "learning_rate": 9.220758606973652e-05,
+      "loss": 0.03189241409301758,
+      "mean_token_accuracy": 0.989690189063549,
+      "num_tokens": 27461882.0,
+      "step": 6400
+    },
+    {
+      "entropy": 0.035260174684226514,
+      "epoch": 1.6113796950539234,
+      "grad_norm": 0.11715493351221085,
+      "learning_rate": 8.960095504625676e-05,
+      "loss": 0.032297356128692625,
+      "mean_token_accuracy": 0.9895079037547112,
+      "num_tokens": 27898972.0,
+      "step": 6500
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 12102,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8627923803091927e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d8efe0593d9b11bf440e19ce9e027f4c1a8f145ce1ea5ceedafeb64107093d
+size 5585