fix: v5 remove empty tokenizer.json, unify think tokens, integrate tokenizer into training pipeline

README.md

@@ -79,7 +79,7 @@ python train/full_finetune.py --model_size 8B
 ```python
 from models.fusion_model import FusionModel, FusionConfig
 
-config = FusionConfig(vocab_size=10000, hidden_size=256, num_layers=2)
+config = FusionConfig(vocab_size=10000, hidden_size=256, num_hidden_layers=2)
 model = FusionModel(config)
 model.eval()
 

configs/fusion-mini-config.json

@@ -24,6 +24,9 @@
   "rms_norm_eps": 1e-5,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
+  "enable_thinking_dial": true,
+  "num_thinking_depths": 4,
+  "think_rank": 0,
 
   "torch_dtype": "float32",
   "transformers_version": "4.36.0",

inference/dashboard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3
 """
 Fusion Inference Dashboard - Interactive inference control panel
 
@@ -128,7 +128,7 @@ class InferenceEngine:
         if hasattr(self.model.config, 'enable_thinking_dial') and self.model.config.enable_thinking_dial:
             try:
                 from models.thinking_dial import ThinkingDialProcessor
-                processor = ThinkingDialProcessor(self._tokenizer or get_tokenizer("gpt2"))
+                processor = ThinkingDialProcessor(self._tokenizer or get_tokenizer("fusion"))
                 self._thinking_depth_token = processor.get_think_token(rank)
             except Exception:
                 pass
@@ -138,7 +138,7 @@ class InferenceEngine:
     
         try:
             from models.tokenizer import get_tokenizer
-            self._tokenizer = get_tokenizer("gpt2")
+            self._tokenizer = get_tokenizer("fusion")
             print(f"Tokenizer loaded: vocab_size={self._tokenizer.vocab_size}")
         except Exception:
             self._tokenizer = None

inference/ollama_deploy.py

@@ -181,8 +181,8 @@ TEMPLATE \"\"\"{{ if .System }}<|im_start|>system
     if thinking_dial:
         content += f"""
 # Thinking Dial 示例（训练时注入）
-# <|think| depth=0|> 简单问题，直接回答
-# <|think| depth=3|> 复杂问题，详细推理
+# <|think_depth_0|> 简单问题，直接回答
+# <|think_depth_3|> 复杂问题，详细推理
 """
     
     # 写入文件
@@ -319,16 +319,16 @@ Fusion 支持动态推理强度控制。在问题前添加控制 token：
 
 ```bash
 # depth=0：直接回答（闲聊、翻译）
-> <|think| depth=0|> 今天天气怎么样？
+> <|think_depth_0|> 今天天气怎么样？
 
 # depth=1：简单推理
-> <|think| depth=1|> 计算 123 * 456
+> <|think_depth_1|> 计算 123 * 456
 
 # depth=2：中等推理
-> <|think| depth=2|> 证明勾股定理
+> <|think_depth_2|> 证明勾股定理
 
 # depth=3：深度推理（思维链）
-> <|think| depth=3|> 解决这个算法问题：...
+> <|think_depth_3|> 解决这个算法问题：...
 ```
 
 ## 3. REST API
@@ -363,7 +363,7 @@ print(response["response"])
 # 带 Thinking Dial
 response = ollama.generate(
     model="{model_name}",
-    prompt="<|think| depth=2|> 证明勾股定理",
+    prompt="<|think_depth_2|> 证明勾股定理",
 )
 
 print(response["response"])

inference/ollama_deploy_v2.py

@@ -266,8 +266,8 @@ TEMPLATE \"\"\"{{{{ if .System }}}}<|im_start|>system
     if thinking_dial:
         content += f"""
 # Thinking Dial examples (injected during training)
-# <|think| depth=0|> Simple question, direct answer
-# <|think| depth=3|> Complex question, detailed reasoning
+# <|think_depth_0|> Simple question, direct answer
+# <|think_depth_3|> Complex question, detailed reasoning
 """
     
     # Write file
@@ -416,16 +416,16 @@ Fusion supports dynamic reasoning intensity control. Add control token before qu
 
 ```bash
 # depth=0: direct answer (casual chat, translation)
-> <|think| depth=0|> How's the weather today?
+> <|think_depth_0|> How's the weather today?
 
 # depth=1: simple reasoning
-> <|think| depth=1|> Calculate 123 * 456
+> <|think_depth_1|> Calculate 123 * 456
 
 # depth=2: medium reasoning
-> <|think| depth=2|> Prove Pythagorean theorem
+> <|think_depth_2|> Prove Pythagorean theorem
 
 # depth=3: deep reasoning (chain-of-thought)
-> <|think| depth=3|> Solve this algorithm problem: ...
+> <|think_depth_3|> Solve this algorithm problem: ...
 ```
 
 ## 3. REST API
@@ -460,7 +460,7 @@ print(response["response"])
 # With Thinking Dial
 response = ollama.generate(
     model="{model_name}",
-    prompt="<|think| depth=2|> Prove Pythagorean theorem",
+    prompt="<|think_depth_2|> Prove Pythagorean theorem",
 )
 
 print(response["response"])

models/thinking_dial.py

@@ -45,9 +45,9 @@ from transformers import PreTrainedModel, GenerationMixin
 # 特殊 Token 定义
 # ============================================================
 
-THINK_START = "<|think|"
+THINK_START = "<|think_depth_"
 THINK_END = "|>"
-THINK_DEPTH_PATTERN = re.compile(r"<\|think\| depth=(\d+)\|>")
+THINK_DEPTH_PATTERN = re.compile(r"<\|think_depth_(\d+)\|>")
 
 # Depth 0-3 的描述
 THINK_DEPTH_DESCRIPTIONS = {
@@ -71,7 +71,7 @@ def build_think_token(depth: int) -> str:
     if not 0 <= depth <= 3:
         raise ValueError(f"depth 必须在 0-3 之间，当前值：{depth}")
     
-    return f"{THINK_START} depth={depth}{THINK_END}"
+    return f"{THINK_START}{depth}{THINK_END}"
 
 
 def parse_think_token(text: str) -> Optional[int]:

scripts/fix_think_tokens.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Fix think token naming consistency across the project."""
+import re
+import glob
+
+# Target format: <|think_depth_0|>, <|think_depth_1|>, etc.
+
+def fix_file(filepath):
+    with open(filepath, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    original = content
+    
+    # Replace THINK_START/THINK_END constants
+    content = content.replace('THINK_START = "<|think_depth_"', 'THINK_START = "<|think_depth_"')
+    content = content.replace('THINK_END = "|>"', 'THINK_END = "|>"')
+    
+    # Replace build_think_token return
+    content = content.replace(
+        'return f"{THINK_START}{depth}{THINK_END}"',
+        'return f"{THINK_START}{depth}{THINK_END}"'
+    )
+    
+    # Replace THINK_DEPTH_PATTERN regex
+    content = content.replace(
+        'THINK_DEPTH_PATTERN = re.compile(r"<\\|think\\| depth=(\\d+)\\|>")',
+        'THINK_DEPTH_PATTERN = re.compile(r"<\\|think_depth_(\\d+)\\|>")'
+    )
+    
+    # Replace any inline <|think| depth=N|> with <|think_depth_N|>
+    content = re.sub(r'<\|think\|\s*depth=(\d+)\|>', r'<|think_depth_\1|>', content)
+    
+    if content != original:
+        with open(filepath, 'w', encoding='utf-8') as f:
+            f.write(content)
+        print(f"  Fixed: {filepath}")
+        return True
+    else:
+        print(f"  No change: {filepath}")
+        return False
+
+files = glob.glob("**/*.py", recursive=True) + glob.glob("**/*.json", recursive=True)
+fixed = 0
+for f in sorted(files):
+    # Skip data files and output
+    if any(skip in f for skip in ['node_modules', '.git', 'output/']):
+        continue
+    try:
+        with open(f, 'r', encoding='utf-8') as fh:
+            text = fh.read()
+        if '<|think|' in text or 'think| depth=' in text:
+            if fix_file(f):
+                fixed += 1
+    except:
+        pass
+
+print(f"\nTotal files fixed: {fixed}")

tests/run_tests.py

@@ -87,7 +87,7 @@ class TestThinkingDial(unittest.TestCase):
         
         # 测试解析
         depth, clean = processor.parse_thinking_depth(
-            "<|think| depth=2|> 证明勾股定理"
+            "<|think_depth_2|> 证明勾股定理"
         )
         
         self.assertEqual(depth, 2)
@@ -109,7 +109,7 @@ class TestThinkingDial(unittest.TestCase):
             depth=1,
         )
         
-        self.assertIn("<|think| depth=1|>", result)
+        self.assertIn("<|think_depth_1|>", result)
         print("✅ Thinking Dial 注入测试通过")
 
 

tokenizer.json

@@ -1,40 +0,0 @@
-{
-  "version": "1.0",
-  "truncation": null,
-  "padding": null,
-  "added_tokens": [
-    {"id": 0, "content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 1, "content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 2, "content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 3, "content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 32000, "content": "<|think| depth=0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 32001, "content": "<|think| depth=1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 32002, "content": "<|think| depth=2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
-    {"id": 32003, "content": "<|think| depth=3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}
-  ],
-  "normalizer": {
-    "type": "PrependIfMissing",
-    "prepend": "</s>",
-    "add_prefix_space": true
-  },
-  "pre_tokenizer": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false
-  },
-  "post_processor": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false
-  },
-  "decoder": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false
-  },
-  "model": {
-    "type": "BPE",
-    "vocab": {},
-    "merges": []
-  }
-}

tokenizer_config.json

@@ -1,11 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>",
-  "model_max_length": 32768,
-  "tokenizer_type": "SentencePiece",
-  "clean_up_tokenization_spaces": true
-}

train/dpo_finetune.py

@@ -78,7 +78,7 @@ class DPOTrainer:
         """Get tokenizer with fallback to character-level encoding."""
         try:
             from models.tokenizer import get_tokenizer
-            return get_tokenizer("gpt2")
+            return get_tokenizer("fusion")
         except Exception:
             return None
     

train/full_finetune.py

@@ -158,12 +158,9 @@ def create_local_model(
     return model, config
 
 
-def create_tokenizer(tokenizer_type: str = "gpt2", vocab_size: int = 32000):
+def create_tokenizer(tokenizer_type: str = "fusion", vocab_size: int = 32000):
     """
     Create tokenizer using the unified tokenizer module.
-    
-    Note: Currently uses GPT2 as placeholder until SentencePiece model is trained.
-    The model config vocab_size will be auto-adjusted to match.
     """
     effective_vocab = get_effective_vocab_size(tokenizer_type, vocab_size)
     logger.info(f"[create_tokenizer] Creating tokenizer: type={tokenizer_type}, effective_vocab={effective_vocab}")

train/lora_finetune.py

@@ -196,27 +196,24 @@ def create_local_model(
 
 def create_tokenizer(vocab_size: int = 32000):
     """
-    创建与模型 vocab_size 匹配的 tokenizer
-    
-    使用 GPT2Tokenizer 作为基础，resize 到匹配的 vocab 大小
+    Create tokenizer matching model vocab_size.
+    Uses unified tokenizer module with SentencePiece if available, falls back to GPT2.
     """
-    logger.info(f"[create_tokenizer] 创建 tokenizer（vocab_size={vocab_size}）")
+    logger.info(f"[create_tokenizer] Creating tokenizer (vocab_size={vocab_size})")
+    
+    try:
+        from models.tokenizer import get_tokenizer
+        tokenizer = get_tokenizer("fusion", vocab_size=vocab_size)
+        return tokenizer
+    except Exception as e:
+        logger.warning(f"Fusion tokenizer failed ({e}), falling back to GPT2")
     
-    # 使用 GPT2 tokenizer 作为基础
     tokenizer = AutoTokenizer.from_pretrained("gpt2")
     tokenizer.pad_token = tokenizer.eos_token
-    
-    # 如果 vocab_size 与 GPT2 不同，调整 embedding 层
-    if vocab_size != tokenizer.vocab_size:
-        logger.info(f"[create_tokenizer] 调整词表大小：{tokenizer.vocab_size} -> {vocab_size}")
-        model_torch_dtype = torch.bfloat16
-        # 获取模型的 embedding 层（在 create_local_model 中创建）
-        # 这里先 resize tokenizer，实际 embedding 在模型中也会自动处理
-        tokenizer.add_special_tokens({'pad_token': '[PAD]'})
-    
     return tokenizer
 
 
+
 def apply_lora(
     model,
     lora_rank: int = 64,
@@ -343,7 +340,7 @@ def main():
     parser.add_argument("--model_size", type=str, default="1.5B",
                         choices=["0.5B", "1.5B", "8B", "14B"],
                         help="模型大小（0.5B/1.5B/8B/14B）")
-    parser.add_argument("--local_model", action="store_true", default=True",
+    parser.add_argument("--local_model", action="store_true", default=True,
                         help="使用本地 FusionModel（默认，无需预训练权重）")
     parser.add_argument("--quantize", action="store_true",
                         help="是否使用量化（QLoRA）")

train/train_mini.py

@@ -36,6 +36,18 @@ project_root = Path(__file__).parent.parent
 sys.path.insert(0, str(project_root))
 
 from models.fusion_mini import FusionMini, FusionMiniConfig
+from models.tokenizer import get_tokenizer
+
+
+def _try_get_tokenizer():
+    """Try to load Fusion tokenizer, return None on failure."""
+    try:
+        tok = get_tokenizer("fusion")
+        if tok is not None:
+            print(f"   [Tokenizer] Loaded Fusion tokenizer, vocab_size={len(tok)}")
+        return tok
+    except Exception:
+        return None
 
 
 class MiniDataset(Dataset):
@@ -164,9 +176,14 @@ def train_mini_model(
     
     # 2. 加载数据集
     print(f"\n[数据] 加载数据集...")
+    tok = _try_get_tokenizer()
+    if tok is not None:
+        vocab_size = len(tok)
+    else:
+        vocab_size = 1000
     dataset = MiniDataset(
         data_path=data_path,
-        tokenizer=None,  # 使用字符级编码
+        tokenizer=tok,  # Use fusion tokenizer if available, else char-level
         max_length=max_length,
     )
     
@@ -179,7 +196,7 @@ def train_mini_model(
     # 3. 创建模型配置
     print(f"\n[模型] 创建模型...")
     config = FusionMiniConfig(
-        vocab_size=1000,  # 字符级，实际会根据数据调整
+        vocab_size=vocab_size,
         hidden_size=hidden_size,
         num_hidden_layers=num_hidden_layers,
         num_attention_heads=4,
@@ -187,8 +204,9 @@ def train_mini_model(
         max_position_embeddings=max_length,
     )
     
-    # 调整词表大小（根据数据）
-    config.vocab_size = len(dataset.char_to_idx) + 10  # 加点余量
+    # If char-level tokenizer, adjust vocab_size from data
+    if tok is None and hasattr(dataset, 'char_to_idx'):
+        config.vocab_size = len(dataset.char_to_idx) + 10
     
     print(f"   词表大小：{config.vocab_size}")
     print(f"   隐层大小：{config.hidden_size}")