fix: replace random embeddings in MoverScore with deterministic hash

MoverScore was using torch.randn causing non-reproducible results each run.
Now uses the same hash-based pseudo-embedding as BERTScore for determinism.

Also created evaluation/benchmark_runner.py that was missing:
- Perplexity benchmark
- Generation quality (BERTScore/MoverScore)
- Inference speed benchmark
- Full benchmark suite CLI

Updated documentation headers to clarify these are simplified placeholder
implementations for demonstration, not production-quality metrics.

Verified: bertscore_simple and moverscore_simple both deterministic
25/25 pytest passed, all imports working.

evaluation/benchmark_runner.py

@@ -0,0 +1,341 @@
+"""
+基准测试运行器
+支持多种评估任务和配置
+"""
+import sys
+import json
+import time
+import torch
+import argparse
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+from datetime import datetime
+
+sys.path.insert(0, '.')
+
+from evaluation.metrics import ModelEvaluator, EvaluationMetrics
+from evaluation.bertscore_moverscore import bertscore_simple, moverscore_simple
+
+
+class BenchmarkRunner:
+    """
+    基准测试运行器
+    
+    支持的测试类型：
+    - perplexity: 困惑度评估
+    - generation: 生成质量评估
+    - accuracy: 任务准确率
+    - speed: 推理速度基准
+    """
+    
+    def __init__(self, model_path: str, device: str = "auto"):
+        """
+        初始化基准测试运行器
+        
+        Args:
+            model_path: 模型路径
+            device: 计算设备 (auto/cpu/cuda)
+        """
+        self.model_path = model_path
+        self.device = self._resolve_device(device)
+        self.model = None
+        self.config = None
+        self.tokenizer = None
+        
+    def _resolve_device(self, device: str) -> str:
+        """解析设备字符串"""
+        if device == "auto":
+            return "cuda" if torch.cuda.is_available() else "cpu"
+        return device
+    
+    def load_model(self):
+        """加载模型"""
+        print(f"[Loading] Model from {self.model_path} on {self.device}")
+        
+        # 尝试加载 FusionMini
+        try:
+            from models.fusion_mini import FusionMini, FusionMiniConfig
+            self.model = FusionMini._load_from_safetensors(self.model_path)
+            self.config = self.model.config
+            print(f"[Loaded] FusionMini model")
+        except Exception as e:
+            # 回退到 FusionModel
+            from models.fusion_model import FusionModel, FusionConfig
+            self.model = FusionModel.from_pretrained(self.model_path)
+            self.config = self.model.config
+            print(f"[Loaded] FusionModel: {e}")
+        
+        self.model.to(self.device)
+        self.model.eval()
+        
+        # 创建简单 tokenizer
+        self.tokenizer = self._create_tokenizer()
+        
+    def _create_tokenizer(self):
+        """创建简单 tokenizer（用于测试）"""
+        vocab_size = getattr(self.config, 'vocab_size', 10000)
+        
+        class SimpleTokenizer:
+            def __init__(self, vs):
+                self.vocab_size = vs
+                
+            def encode(self, text):
+                # 简单字符级编码
+                return [ord(c) % self.vocab_size for c in text[:512]]
+                
+            def decode(self, ids):
+                return ''.join(chr(i % 128 + 32) for i in ids if 0 <= i < self.vocab_size)
+        
+        return SimpleTokenizer(vocab_size)
+    
+    def run_perplexity(self, texts: List[str]) -> Dict[str, float]:
+        """
+        计算困惑度
+        
+        Args:
+            texts: 测试文本列表
+            
+        Returns:
+            困惑度指标
+        """
+        print(f"\n[Benchmark] Perplexity on {len(texts)} texts")
+        
+        self.model.eval()
+        total_loss = 0.0
+        total_tokens = 0
+        
+        with torch.no_grad():
+            for text in texts:
+                ids = self.tokenizer.encode(text)
+                if len(ids) < 2:
+                    continue
+                    
+                input_ids = torch.tensor([ids], device=self.device)
+                labels = input_ids.clone()
+                
+                outputs = self.model(input_ids, labels=labels)
+                loss = outputs.loss
+                
+                if loss is not None:
+                    total_loss += loss.item() * len(ids)
+                    total_tokens += len(ids)
+        
+        if total_tokens == 0:
+            return {"perplexity": float('inf')}
+        
+        avg_loss = total_loss / total_tokens
+        perplexity = torch.exp(torch.tensor(avg_loss)).item()
+        
+        return {
+            "perplexity": perplexity,
+            "avg_loss": avg_loss,
+            "total_tokens": total_tokens
+        }
+    
+    def run_generation_quality(
+        self, 
+        prompts: List[str], 
+        references: List[str],
+        max_new_tokens: int = 50
+    ) -> Dict[str, Any]:
+        """
+        生成质量评估
+        
+        Args:
+            prompts: 提示列表
+            references: 参考答案列表
+            max_new_tokens: 最大生成 token 数
+            
+        Returns:
+            生成质量指标
+        """
+        print(f"\n[Benchmark] Generation quality on {len(prompts)} prompts")
+        
+        generations = []
+        
+        self.model.eval()
+        with torch.no_grad():
+            for prompt in prompts:
+                ids = self.tokenizer.encode(prompt)
+                input_ids = torch.tensor([ids], device=self.device)
+                
+                # 简单贪婪生成
+                generated = input_ids.clone()
+                for _ in range(max_new_tokens):
+                    outputs = self.model(generated)
+                    next_token = outputs.logits[:, -1, :].argmax(dim=-1, keepdim=True)
+                    generated = torch.cat([generated, next_token], dim=1)
+                    
+                    # EOS 检查（假设 2 是 EOS）
+                    if next_token.item() == 2:
+                        break
+                
+                gen_text = self.tokenizer.decode(generated[0].tolist())
+                generations.append(gen_text)
+        
+        # 计算 BERTScore 和 MoverScore
+        bert_scores = []
+        mover_scores = []
+        
+        for gen, ref in zip(generations, references):
+            gen_ids = self.tokenizer.encode(gen)
+            ref_ids = self.tokenizer.encode(ref)
+            
+            _, _, bert_f1 = bertscore_simple(gen_ids, ref_ids)
+            mover_score = moverscore_simple(gen_ids, ref_ids)
+            
+            bert_scores.append(bert_f1)
+            mover_scores.append(mover_score)
+        
+        return {
+            "bertscore_f1": sum(bert_scores) / len(bert_scores) if bert_scores else 0.0,
+            "moverscore": sum(mover_scores) / len(mover_scores) if mover_scores else 0.0,
+            "generations": generations[:5],  # 只返回前5个样本
+            "num_samples": len(generations)
+        }
+    
+    def run_speed_benchmark(
+        self, 
+        batch_sizes: List[int] = [1, 2, 4],
+        seq_lengths: List[int] = [32, 64, 128, 256],
+        warmup: int = 3,
+        runs: int = 10
+    ) -> Dict[str, Any]:
+        """
+        推理速度基准测试
+        
+        Args:
+            batch_sizes: 批大小列表
+            seq_lengths: 序列长度列表
+            warmup: 预热次数
+            runs: 测试次数
+            
+        Returns:
+            速度指标
+        """
+        print(f"\n[Benchmark] Speed benchmark")
+        
+        results = []
+        vocab_size = getattr(self.config, 'vocab_size', 10000)
+        
+        self.model.eval()
+        
+        for batch_size in batch_sizes:
+            for seq_len in seq_lengths:
+                # 预热
+                for _ in range(warmup):
+                    dummy = torch.randint(0, vocab_size, (batch_size, seq_len), device=self.device)
+                    with torch.no_grad():
+                        _ = self.model(dummy)
+                
+                # 计时
+                torch.cuda.synchronize() if self.device == "cuda" else None
+                start = time.perf_counter()
+                
+                for _ in range(runs):
+                    dummy = torch.randint(0, vocab_size, (batch_size, seq_len), device=self.device)
+                    with torch.no_grad():
+                        _ = self.model(dummy)
+                
+                torch.cuda.synchronize() if self.device == "cuda" else None
+                end = time.perf_counter()
+                
+                avg_time = (end - start) / runs
+                throughput = batch_size * seq_len / avg_time
+                
+                results.append({
+                    "batch_size": batch_size,
+                    "seq_len": seq_len,
+                    "latency_ms": avg_time * 1000,
+                    "throughput_tokens_per_sec": throughput
+                })
+                
+                print(f"  batch={batch_size}, seq={seq_len}: {avg_time*1000:.2f}ms")
+        
+        return {
+            "results": results,
+            "device": self.device,
+            "runs": runs
+        }
+    
+    def run_full_benchmark(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        运行完整基准测试
+        
+        Args:
+            config: 测试配置
+            
+        Returns:
+            完整测试结果
+        """
+        print("="*60)
+        print("Fusion-LLM Benchmark Runner")
+        print("="*60)
+        
+        self.load_model()
+        
+        results = {
+            "model_path": self.model_path,
+            "device": self.device,
+            "timestamp": datetime.now().isoformat(),
+            "config": self.config.to_dict() if hasattr(self.config, 'to_dict') else {}
+        }
+        
+        # 困惑度
+        if config.get("perplexity", True):
+            test_texts = [
+                "The quick brown fox jumps over the lazy dog.",
+                "Machine learning models require large amounts of data.",
+                "Natural language processing enables computers to understand text."
+            ]
+            results["perplexity"] = self.run_perplexity(test_texts)
+        
+        # 生成质量
+        if config.get("generation", True):
+            prompts = ["The future of AI is", "Machine learning helps"]
+            references = ["The future of AI is bright and transformative.", "Machine learning helps solve complex problems."]
+            results["generation"] = self.run_generation_quality(prompts, references)
+        
+        # 速度基准
+        if config.get("speed", False):
+            results["speed"] = self.run_speed_benchmark(
+                batch_sizes=config.get("batch_sizes", [1]),
+                seq_lengths=config.get("seq_lengths", [32, 64, 128])
+            )
+        
+        print("\n[Benchmark] Complete")
+        return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fusion-LLM Benchmark Runner")
+    parser.add_argument("--model", required=True, help="Path to model checkpoint")
+    parser.add_argument("--device", default="auto", help="Device (auto/cpu/cuda)")
+    parser.add_argument("--output", default="benchmark_results.json", help="Output file")
+    parser.add_argument("--perplexity", action="store_true", help="Run perplexity benchmark")
+    parser.add_argument("--generation", action="store_true", help="Run generation benchmark")
+    parser.add_argument("--speed", action="store_true", help="Run speed benchmark")
+    parser.add_argument("--all", action="store_true", help="Run all benchmarks")
+    
+    args = parser.parse_args()
+    
+    # 配置
+    config = {
+        "perplexity": args.perplexity or args.all,
+        "generation": args.generation or args.all,
+        "speed": args.speed or args.all
+    }
+    
+    # 运行
+    runner = BenchmarkRunner(args.model, args.device)
+    results = runner.run_full_benchmark(config)
+    
+    # 保存
+    with open(args.output, 'w', encoding='utf-8') as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    
+    print(f"\n[Saved] Results to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

evaluation/bertscore_moverscore.py

@@ -1,7 +1,11 @@
 """
 BERTScore 和 MoverScore 评估指标
-注意：这些是简化版实现，用于演示目的
-实际使用时应该安装官方包：
+
+【重要说明】这些是简化版实现（演示用）：
+- BERTScore: 使用 hash 伪嵌入（确定性但非真正语义嵌入）
+- MoverScore: 使用 hash 伪嵌入（确定性，模拟词嵌入距离）
+
+生产环境请使用官方包：
 - BERTScore: pip install bert-score
 - MoverScore: pip install moverscore
 """
@@ -15,9 +19,14 @@ sys.path.insert(0, '.')
 
 def bertscore_simple(candidate, reference, model_name="bert-base-uncased"):
     """
-    Simplified BERTScore using cosine similarity on token ID embeddings.
+    Simplified BERTScore using deterministic hash embeddings.
     
-    For production use, install the official package:
+    【警告】此版本使用 hash 伪嵌入，非真正语义嵌入：
+    - 结果是确定性的（相同输入总是相同输出）
+    - 但不捕获真正语义相似性
+    - 仅用于快速测试和流程验证
+    
+    生产环境请安装官方包：
         pip install bert-score
         from bert_score import score
         P, R, F1 = score(candidates, references, lang="en")
@@ -72,7 +81,7 @@ def bertscore_simple(candidate, reference, model_name="bert-base-uncased"):
 
 def moverscore_simple(candidate, reference):
     """
-    简化版 MoverScore（演示用）
+    简化版 MoverScore（演示用，确定性版本）
     
     实际使用时请安装官方包：pip install moverscore
     然后使用：
@@ -86,22 +95,43 @@ def moverscore_simple(candidate, reference):
     Returns:
         float: MoverScore
     """
-    # 简化版：使用 token 嵌入的 Earth Mover's Distance
-    # 实际 MoverScore 使用 BERT 嵌入的 WMD
+    # [FIX] 使用与 BERTScore 相同的 hash 嵌入，确保结果可复现
+    # 之前用 torch.randn 导致每次调用结果不同
+    
+    if len(candidate) == 0 or len(reference) == 0:
+        return 0.0
+    
+    embed_dim = 128  # MoverScore 用较小维度即可
+    
+    def _hash_embed(token_ids):
+        """确定性伪嵌入，与 bertscore_simple 保持一致"""
+        emb = torch.zeros(len(token_ids), embed_dim)
+        for i, tid in enumerate(token_ids):
+            for j in range(embed_dim):
+                # 使用不同的 hash 种子（2654435761 是黄金比例常数的整数部分）
+                emb[i, j] = ((tid * (j + 1) * 2654435761) % (2**31)) / (2**31) * 2 - 1
+        return emb
     
-    # 转换为嵌入（简化版：随机嵌入）
-    # 实际使用时应该使用 BERT 嵌入
-    cand_embeddings = torch.randn(len(candidate), 768)  # (seq_len, hidden_size)
-    ref_embeddings = torch.randn(len(reference), 768)
+    cand_embeddings = _hash_embed(candidate)  # (len_c, dim)
+    ref_embeddings = _hash_embed(reference)    # (len_r, dim)
     
-    # 计算成本矩阵
+    # 计算成本矩阵（Euclidean 距离）
     cost_matrix = torch.cdist(cand_embeddings.unsqueeze(0), ref_embeddings.unsqueeze(0)).squeeze(0)
     
-    # 简化版：使用最小成本作为分数
-    min_cost = cost_matrix.min().item()
+    # 简化版：使用平均最小成本作为分数（双向）
+    # Precision: 对每个候选 token 找最近的参考 token
+    precision_cost = cost_matrix.min(dim=1).values.mean().item()
+    # Recall: 对每个参考 token 找最近的候选 token
+    recall_cost = cost_matrix.min(dim=0).values.mean().item()
     
     # 归一化到 0-1（越低越好 → 越高越好）
-    score = 1.0 / (1.0 + min_cost)
+    precision_score = 1.0 / (1.0 + precision_cost)
+    recall_score = 1.0 / (1.0 + recall_cost)
+    
+    # F1 综合
+    if precision_score + recall_score == 0:
+        return 0.0
+    score = 2 * precision_score * recall_score / (precision_score + recall_score)
     
     return score