#!/usr/bin/env python3 """ MODEL MANAGER ============= Harvester's tool for managing its own model stack. Discovers, evaluates, and adopts models autonomously — with rules. Rules (non-negotiable): 1. NEVER remove a working model without a tested replacement 2. New model must BEAT current model on target domain tasks 3. Must fit within hardware constraints (VRAM/RAM) 4. Every decision logged to growth_memory.db with reasoning 5. Benchmark before adopt — no untested models in production 6. Keep at least one fallback model per role (coder, reasoner) 7. Human can override any decision via config or env var Roles: - coder: Code generation, reflection, troubleshooting - reasoner: OSINT, investigation, complex reasoning Usage: python model_manager.py --scan # Discover models on disk + HuggingFace python model_manager.py --benchmark # Benchmark current model for a role python model_manager.py --evaluate # Test a candidate model python model_manager.py --adopt # Adopt after passing benchmarks python model_manager.py --status # Current stack status python model_manager.py --history # Decision log """ import os import sys import json import sqlite3 import time import hashlib from pathlib import Path from datetime import datetime from dataclasses import dataclass, asdict from typing import Optional sys.path.insert(0, os.path.dirname(__file__)) # ── Config ──────────────────────────────────────────────────────────────────── MODEL_DIR = os.environ.get("MODEL_DIR", r"D:\va_data\models") DB_PATH = os.environ.get("GROWTH_DB_PATH", os.path.join(os.path.dirname(__file__), "growth_memory.db")) LMSTUDIO_CACHE = os.path.expanduser(os.path.join("~", ".cache", "lm-studio", "models")) # Hardware constraints — updated via env or auto-detect MAX_RAM_GB = float(os.environ.get("MAX_RAM_GB", "64")) MAX_VRAM_GB = float(os.environ.get("MAX_VRAM_GB", "0")) # 0 = CPU only GPU_LAYERS = int(os.environ.get("NATIVE_GPU_LAYERS", "0")) # Adoption rules MIN_BENCHMARK_IMPROVEMENT = float(os.environ.get("MIN_BENCHMARK_IMPROVEMENT", "0.05")) # 5% better MIN_BENCHMARK_TASKS = int(os.environ.get("MIN_BENCHMARK_TASKS", "5")) # at least 5 tasks MAX_MODEL_SIZE_GB = float(os.environ.get("MAX_MODEL_SIZE_GB", "0")) # 0 = auto from hardware # ── Data structures ─────────────────────────────────────────────────────────── @dataclass class ModelInfo: name: str path: str size_gb: float role: str # "coder", "reasoner", "unknown" source: str # "local", "lmstudio_cache", "huggingface" quantization: str # "Q4_K_M", "Q8_0", etc. status: str # "active", "candidate", "retired", "fallback" benchmark_score: float = 0.0 benchmark_tasks: int = 0 adopted_at: Optional[str] = None notes: str = "" @dataclass class ModelDecision: timestamp: str action: str # "scan", "benchmark", "adopt", "reject", "retire" model_name: str role: str reasoning: str score_before: float score_after: float approved: bool # ── Database ────────────────────────────────────────────────────────────────── def _init_db(): """Create model management tables if they don't exist.""" conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute(""" CREATE TABLE IF NOT EXISTS model_registry ( name TEXT NOT NULL, path TEXT NOT NULL, size_gb REAL, role TEXT, source TEXT, quantization TEXT, status TEXT DEFAULT 'candidate', benchmark_score REAL DEFAULT 0.0, benchmark_tasks INTEGER DEFAULT 0, adopted_at TEXT, notes TEXT, updated_at TEXT, PRIMARY KEY (name, role) ) """) c.execute(""" CREATE TABLE IF NOT EXISTS model_decisions ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT, action TEXT, model_name TEXT, role TEXT, reasoning TEXT, score_before REAL, score_after REAL, approved INTEGER ) """) conn.commit() conn.close() def _log_decision(decision: ModelDecision): """Log every model decision to the database.""" conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute(""" INSERT INTO model_decisions (timestamp, action, model_name, role, reasoning, score_before, score_after, approved) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, (decision.timestamp, decision.action, decision.model_name, decision.role, decision.reasoning, decision.score_before, decision.score_after, int(decision.approved))) conn.commit() conn.close() def _save_model(model: ModelInfo): """Upsert model info in registry.""" conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute(""" INSERT INTO model_registry (name, path, size_gb, role, source, quantization, status, benchmark_score, benchmark_tasks, adopted_at, notes, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(name, role) DO UPDATE SET path=excluded.path, size_gb=excluded.size_gb, source=excluded.source, quantization=excluded.quantization, status=excluded.status, benchmark_score=excluded.benchmark_score, benchmark_tasks=excluded.benchmark_tasks, adopted_at=excluded.adopted_at, notes=excluded.notes, updated_at=excluded.updated_at """, (model.name, model.path, model.size_gb, model.role, model.source, model.quantization, model.status, model.benchmark_score, model.benchmark_tasks, model.adopted_at, model.notes, datetime.now().isoformat())) conn.commit() conn.close() def _get_active_model(role: str) -> Optional[dict]: """Get the currently active model for a role.""" conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row c = conn.cursor() c.execute("SELECT * FROM model_registry WHERE role=? AND status='active'", (role,)) row = c.fetchone() conn.close() return dict(row) if row else None def _get_all_models(role: str = None) -> list: """Get all registered models, optionally filtered by role.""" conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row c = conn.cursor() if role: c.execute("SELECT * FROM model_registry WHERE role=? ORDER BY benchmark_score DESC", (role,)) else: c.execute("SELECT * FROM model_registry ORDER BY role, benchmark_score DESC") rows = c.fetchall() conn.close() return [dict(r) for r in rows] # ── Hardware detection ──────────────────────────────────────────────────────── def detect_hardware() -> dict: """Detect available hardware for model sizing decisions.""" hw = { "ram_gb": MAX_RAM_GB, "vram_gb": MAX_VRAM_GB, "gpu_available": MAX_VRAM_GB > 0 or GPU_LAYERS != 0, "gpu_layers": GPU_LAYERS, "max_model_gb": 0.0, } # Try to detect GPU via torch try: import torch if torch.cuda.is_available(): vram = torch.cuda.get_device_properties(0).total_memory / (1024**3) hw["vram_gb"] = vram hw["gpu_available"] = True except ImportError: pass # Max model size: 80% of available memory (leave room for KV cache, OS) if hw["gpu_available"] and hw["vram_gb"] > 0: hw["max_model_gb"] = hw["vram_gb"] * 0.8 else: # CPU-only: leave 16GB for OS + apps hw["max_model_gb"] = max(0, hw["ram_gb"] - 16) * 0.8 if MAX_MODEL_SIZE_GB > 0: hw["max_model_gb"] = MAX_MODEL_SIZE_GB return hw # ── Model discovery ─────────────────────────────────────────────────────────── def _parse_model_name(filename: str) -> dict: """Extract model info from GGUF filename.""" info = {"name": filename, "quantization": "unknown"} # Common quantization patterns for q in ["Q8_0", "Q6_K", "Q5_K_M", "Q5_K_S", "Q4_K_M", "Q4_K_S", "Q4_0", "Q3_K_M", "Q3_K_S", "Q2_K", "IQ4_XS"]: if q in filename: info["quantization"] = q break # Guess role from name name_lower = filename.lower() if any(kw in name_lower for kw in ["coder", "code", "starcoder", "codellama", "deepseek-coder"]): info["role"] = "coder" elif any(kw in name_lower for kw in ["reason", "r1", "think", "opus", "deepseek-r1"]): info["role"] = "reasoner" else: info["role"] = "unknown" info["name"] = filename.replace(".gguf", "") return info def scan_local_models() -> list: """Scan local directories for GGUF models.""" found = [] scan_dirs = [MODEL_DIR] # Also check LM Studio cache if os.path.isdir(LMSTUDIO_CACHE): scan_dirs.append(LMSTUDIO_CACHE) for scan_dir in scan_dirs: if not os.path.isdir(scan_dir): continue source = "lmstudio_cache" if "lm-studio" in scan_dir else "local" for root, dirs, files in os.walk(scan_dir): for f in files: if f.endswith(".gguf"): full_path = os.path.join(root, f) size_gb = os.path.getsize(full_path) / (1024**3) info = _parse_model_name(f) model = ModelInfo( name=info["name"], path=full_path, size_gb=round(size_gb, 2), role=info.get("role", "unknown"), source=source, quantization=info["quantization"], status="candidate", ) found.append(model) return found def scan_and_register() -> list: """Scan for models and register them in the database.""" _init_db() hw = detect_hardware() models = scan_local_models() registered = [] for m in models: # Rule: skip models that won't fit if m.size_gb > hw["max_model_gb"]: m.notes = f"Too large ({m.size_gb:.1f}GB > {hw['max_model_gb']:.1f}GB limit)" m.status = "oversized" _save_model(m) registered.append(m) _log_decision(ModelDecision( timestamp=datetime.now().isoformat(), action="scan", model_name="*", role="*", reasoning=f"Scanned {len(registered)} models. Hardware: {hw['ram_gb']:.0f}GB RAM, {hw['vram_gb']:.0f}GB VRAM, max model {hw['max_model_gb']:.1f}GB", score_before=0, score_after=0, approved=True )) return registered # ── Benchmarking ────────────────────────────────────────────────────────────── def benchmark_model(model_path: str, role: str, task_bank: str = None) -> dict: """Benchmark a model against a task bank for a specific role. Returns dict with score, task_count, avg_confidence, avg_time. """ from reflection_engine import set_task_mode, reason if task_bank is None: if role == "reasoner": task_bank = os.path.join(os.path.dirname(__file__), "bellingcat_tasks.json") else: task_bank = os.path.join(os.path.dirname(__file__), "reasoning_tasks.json") if not os.path.isfile(task_bank): return {"error": f"Task bank not found: {task_bank}", "score": 0} with open(task_bank, "r", encoding="utf-8") as f: tasks = json.load(f) # Use first N tasks for benchmarking (not full bank — save time) bench_tasks = tasks[:MIN_BENCHMARK_TASKS] results = [] total_confidence = 0.0 high_confidence = 0 total_time = 0.0 set_task_mode(role) for task in bench_tasks: prompt = task.get("prompt", task.get("description", "")) if not prompt: continue start = time.time() try: plan = reason(prompt) elapsed = time.time() - start conf = getattr(plan, "confidence", 0.3) total_confidence += conf if conf >= 0.7: high_confidence += 1 total_time += elapsed results.append({ "task": task.get("name", "unnamed"), "confidence": conf, "time_secs": round(elapsed, 1), "parsed": conf > 0.3, # Did it actually reason or fallback? }) except Exception as e: elapsed = time.time() - start total_time += elapsed results.append({ "task": task.get("name", "unnamed"), "confidence": 0.0, "time_secs": round(elapsed, 1), "error": str(e), }) n = len(results) return { "model_path": model_path, "role": role, "task_bank": task_bank, "task_count": n, "high_confidence": high_confidence, "avg_confidence": round(total_confidence / n, 3) if n > 0 else 0, "avg_time_secs": round(total_time / n, 1) if n > 0 else 0, "total_time_secs": round(total_time, 1), "score": round(high_confidence / n, 3) if n > 0 else 0, "results": results, } # ── Adoption rules engine ──────────────────────────────────────────────────── def evaluate_adoption(candidate_name: str, role: str, benchmark_result: dict) -> ModelDecision: """Apply rules to decide whether to adopt a candidate model. Rules: 1. Must have benchmarked at least MIN_BENCHMARK_TASKS tasks 2. Score must beat current active model by MIN_BENCHMARK_IMPROVEMENT 3. Model must fit within hardware constraints 4. Must not be the only model (keep fallback) """ _init_db() current = _get_active_model(role) current_score = current["benchmark_score"] if current else 0.0 candidate_score = benchmark_result.get("score", 0) # Rule 1: enough tasks? if benchmark_result.get("task_count", 0) < MIN_BENCHMARK_TASKS: return ModelDecision( timestamp=datetime.now().isoformat(), action="reject", model_name=candidate_name, role=role, reasoning=f"Insufficient benchmark tasks: {benchmark_result.get('task_count', 0)} < {MIN_BENCHMARK_TASKS}", score_before=current_score, score_after=candidate_score, approved=False ) # Rule 2: improvement threshold? improvement = candidate_score - current_score if current_score > 0 and improvement < MIN_BENCHMARK_IMPROVEMENT: return ModelDecision( timestamp=datetime.now().isoformat(), action="reject", model_name=candidate_name, role=role, reasoning=f"Improvement too small: {improvement:.3f} < {MIN_BENCHMARK_IMPROVEMENT} threshold. Current: {current_score:.3f}, Candidate: {candidate_score:.3f}", score_before=current_score, score_after=candidate_score, approved=False ) # Rule 3: fits hardware? hw = detect_hardware() models = _get_all_models() candidate_models = [m for m in models if m["name"] == candidate_name] if candidate_models: candidate_size = candidate_models[0]["size_gb"] if candidate_size > hw["max_model_gb"]: return ModelDecision( timestamp=datetime.now().isoformat(), action="reject", model_name=candidate_name, role=role, reasoning=f"Model too large: {candidate_size:.1f}GB > {hw['max_model_gb']:.1f}GB hardware limit", score_before=current_score, score_after=candidate_score, approved=False ) # All rules passed — approve adoption return ModelDecision( timestamp=datetime.now().isoformat(), action="adopt", model_name=candidate_name, role=role, reasoning=f"Passed all rules. Score: {candidate_score:.3f} vs current {current_score:.3f} (+{improvement:.3f}). Fits hardware.", score_before=current_score, score_after=candidate_score, approved=True ) def adopt_model(model_name: str, role: str, benchmark_result: dict) -> bool: """Adopt a model for a role if it passes all rules. Returns True if adopted, False if rejected. """ _init_db() # Evaluate decision = evaluate_adoption(model_name, role, benchmark_result) _log_decision(decision) if not decision.approved: print(f" ❌ REJECTED: {decision.reasoning}") return False # Retire current active model to fallback (Rule 1: never remove without replacement) current = _get_active_model(role) if current: conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute("UPDATE model_registry SET status='fallback' WHERE name=? AND role=?", (current["name"], role)) conn.commit() conn.close() print(f" 📦 {current['name']} moved to fallback") # Activate new model conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute("""UPDATE model_registry SET status='active', benchmark_score=?, benchmark_tasks=?, adopted_at=? WHERE name=? AND role=?""", (benchmark_result["score"], benchmark_result["task_count"], datetime.now().isoformat(), model_name, role)) conn.commit() conn.close() print(f" ✅ ADOPTED: {model_name} as {role} (score: {benchmark_result['score']:.3f})") return True # ── Status & history ────────────────────────────────────────────────────────── def print_status(): """Print current model stack status.""" _init_db() hw = detect_hardware() print("=" * 60) print("MODEL MANAGER — Stack Status") print("=" * 60) print(f" Hardware: {hw['ram_gb']:.0f}GB RAM | {hw['vram_gb']:.0f}GB VRAM | GPU layers: {hw['gpu_layers']}") print(f" Max model size: {hw['max_model_gb']:.1f}GB") print() for role in ["coder", "reasoner"]: models = _get_all_models(role) print(f" [{role.upper()}]") if not models: print(f" (no models registered)") for m in models: marker = "→" if m["status"] == "active" else " " score = f"score={m['benchmark_score']:.3f}" if m["benchmark_score"] > 0 else "not benchmarked" print(f" {marker} {m['name']} [{m['status']}] {m['size_gb']:.1f}GB {m['quantization']} ({score})") print() # Unknown role models unknowns = _get_all_models("unknown") if unknowns: print(f" [UNCLASSIFIED]") for m in unknowns: print(f" {m['name']} {m['size_gb']:.1f}GB {m['quantization']} ({m['source']})") print() # ── Autonomous scanning / absorption ────────────────────────────────────────── # When to trigger an auto-scan (called from growth_engine after N tasks) AUTO_SCAN_INTERVAL = int(os.environ.get("AUTO_SCAN_INTERVAL", "20")) # every N tasks # HuggingFace search filters for model discovery HF_SEARCH_TAGS = { "coder": ["code", "coder", "codegen", "instruct"], "reasoner": ["reasoning", "think", "r1", "cot"], } HF_QUANT_PREFERENCE = ["Q4_K_M", "Q5_K_M", "Q4_K_S", "Q6_K"] # in preference order def search_huggingface(role: str, max_results: int = 5) -> list[dict]: """Search HuggingFace for GGUF models that might fit a role. Returns list of dicts with: repo_id, filename, size_gb, quantization. Does NOT download — just discovers candidates. """ results = [] hw = detect_hardware() try: from huggingface_hub import HfApi api = HfApi() except ImportError: print(" [HF] huggingface_hub not installed — skip HF search") return results tags = HF_SEARCH_TAGS.get(role, ["instruct"]) for tag in tags: try: models = api.list_models( search=f"{tag} GGUF", sort="downloads", direction=-1, limit=10, ) for model in models: repo_id = model.id # Look for GGUF files we can use try: siblings = api.list_repo_tree(repo_id, recursive=False) for item in siblings: if not hasattr(item, "rfilename"): continue fname = item.rfilename if not fname.endswith(".gguf"): continue # Check if it's a quantization we want quant = "unknown" for q in HF_QUANT_PREFERENCE: if q in fname: quant = q break if quant == "unknown": continue # skip unrecognized quants size_gb = item.size / (1024**3) if hasattr(item, "size") and item.size else 0 # Skip if too large for hardware if size_gb > hw["max_model_gb"] and size_gb > 0: continue results.append({ "repo_id": repo_id, "filename": fname, "size_gb": round(size_gb, 2), "quantization": quant, "role": role, }) except Exception: continue # skip repos we can't inspect except Exception as e: print(f" [HF] Search error for tag '{tag}': {e}") continue # Deduplicate and limit seen = set() unique = [] for r in results: key = f"{r['repo_id']}/{r['filename']}" if key not in seen: seen.add(key) unique.append(r) return unique[:max_results] def auto_scan_and_evaluate(roles: list[str] = None) -> dict: """Autonomous model management: scan local → check HF → benchmark → adopt/reject. This is the main entry point for Harvester's self-managed model stack. Called by growth_engine every AUTO_SCAN_INTERVAL tasks. Returns dict with scan results and any decisions made. """ _init_db() if roles is None: roles = ["coder", "reasoner"] report = { "timestamp": datetime.now().isoformat(), "local_scanned": 0, "hf_discovered": 0, "benchmarked": 0, "adopted": 0, "rejected": 0, "decisions": [], } # 1. Scan local models print("\n 🔍 Scanning local model directories...") local = scan_and_register() report["local_scanned"] = len(local) print(f" Found {len(local)} local model(s)") for role in roles: current = _get_active_model(role) current_score = current["benchmark_score"] if current else 0.0 # 2. Check if any local candidates beat current candidates = [m for m in _get_all_models(role) if m["status"] == "candidate" and m["name"] != (current["name"] if current else "")] if candidates: print(f"\n 📊 {len(candidates)} candidate(s) for {role}:") for c in candidates[:3]: # limit to top 3 by size print(f" {c['name']} ({c['size_gb']:.1f}GB)") # Benchmark only if not already benchmarked if c["benchmark_score"] == 0 and c["benchmark_tasks"] == 0: print(f" ⏳ Benchmarking {c['name']}...") try: bench = benchmark_model(c["path"], role) report["benchmarked"] += 1 # Try to adopt adopted = adopt_model(c["name"], role, bench) if adopted: report["adopted"] += 1 report["decisions"].append(f"Adopted {c['name']} for {role}") else: report["rejected"] += 1 report["decisions"].append(f"Rejected {c['name']} for {role}") except Exception as e: print(f" [ERROR] Benchmark failed: {e}") report["decisions"].append(f"Benchmark error for {c['name']}: {e}") # 3. Search HuggingFace (only if no local candidates or current is weak) if not candidates or current_score < 0.5: print(f"\n 🌐 Searching HuggingFace for {role} models...") try: hf_models = search_huggingface(role, max_results=3) report["hf_discovered"] += len(hf_models) for hf in hf_models: print(f" 📦 {hf['repo_id']}/{hf['filename']} ({hf['size_gb']:.1f}GB, {hf['quantization']})") if hf_models: print(f" ℹ️ Use --adopt to evaluate and adopt HF models") except Exception as e: print(f" [HF] Search error: {e}") _log_decision(ModelDecision( timestamp=datetime.now().isoformat(), action="auto_scan", model_name="*", role=",".join(roles), reasoning=f"Auto-scan: {report['local_scanned']} local, {report['hf_discovered']} HF, {report['adopted']} adopted, {report['rejected']} rejected", score_before=0, score_after=0, approved=True )) return report def should_auto_scan() -> bool: """Check if it's time for an automatic model scan. Returns True if AUTO_SCAN_INTERVAL tasks have passed since last scan. Called by growth_engine after each task completes. """ _init_db() conn = sqlite3.connect(DB_PATH) c = conn.cursor() # Count tasks since last auto_scan decision c.execute("SELECT MAX(id) as last_id FROM model_decisions WHERE action='auto_scan'") row = c.fetchone() last_scan_id = row[0] if row and row[0] else 0 # Count tasks in task_history since then (approximate — uses timestamp ordering) c.execute("SELECT COUNT(*) FROM task_history WHERE id > ?", (last_scan_id,)) tasks_since = c.fetchone()[0] conn.close() return tasks_since >= AUTO_SCAN_INTERVAL def print_history(limit: int = 20): """Print recent model decisions.""" _init_db() conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row c = conn.cursor() c.execute("SELECT * FROM model_decisions ORDER BY id DESC LIMIT ?", (limit,)) rows = c.fetchall() conn.close() print("=" * 60) print("MODEL MANAGER — Decision History") print("=" * 60) for r in reversed(rows): approved = "✅" if r["approved"] else "❌" print(f" {r['timestamp'][:19]} | {approved} {r['action']:10} | {r['model_name']}") print(f" Role: {r['role']} | {r['reasoning'][:80]}") if r["score_before"] > 0 or r["score_after"] > 0: print(f" Score: {r['score_before']:.3f} → {r['score_after']:.3f}") print() # ── CLI ─────────────────────────────────────────────────────────────────────── def seed_current_stack(): """Register the models Harvester already knows about from reflection_engine config.""" _init_db() try: from reflection_engine import ( NATIVE_MODEL_PATH, NATIVE_HF_FILE, NATIVE_HF_REPO, REASONER_MODEL_PATH, REASONER_HF_FILE, REASONER_HF_REPO, ) except ImportError: print(" ⚠️ Could not import reflection_engine — seeding with hardcoded defaults") NATIVE_MODEL_PATH = r"D:\va_data\models\Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf" NATIVE_HF_FILE = "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf" REASONER_MODEL_PATH = r"D:\va_data\models\Qwen3.5-27B.Q4_K_M.gguf" REASONER_HF_FILE = "Qwen3.5-27B.Q4_K_M.gguf" seeded = 0 for path, role, hf_file in [ (NATIVE_MODEL_PATH, "coder", NATIVE_HF_FILE), (REASONER_MODEL_PATH, "reasoner", REASONER_HF_FILE), ]: name = hf_file.replace(".gguf", "") info = _parse_model_name(hf_file) size_gb = round(os.path.getsize(path) / (1024**3), 2) if os.path.isfile(path) else 0.0 status = "active" if os.path.isfile(path) else "configured" model = ModelInfo( name=name, path=path, size_gb=size_gb, role=role, source="local", quantization=info["quantization"], status=status, adopted_at=datetime.now().isoformat(), notes="Seeded from reflection_engine config" ) _save_model(model) on_disk = "✅ on disk" if os.path.isfile(path) else "⚠️ NOT on disk" print(f" → {name} [{role}] {size_gb:.1f}GB ({on_disk})") seeded += 1 _log_decision(ModelDecision( timestamp=datetime.now().isoformat(), action="seed", model_name="*", role="*", reasoning=f"Seeded {seeded} models from reflection_engine config", score_before=0, score_after=0, approved=True )) print(f"\n Seeded {seeded} models.") def main(): import argparse parser = argparse.ArgumentParser(description="Harvester Model Manager") parser.add_argument("--scan", action="store_true", help="Scan for local models") parser.add_argument("--seed", action="store_true", help="Seed current config models into registry") parser.add_argument("--status", action="store_true", help="Show current stack status") parser.add_argument("--history", action="store_true", help="Show decision log") parser.add_argument("--benchmark", type=str, metavar="ROLE", help="Benchmark current model for role (coder/reasoner)") parser.add_argument("--evaluate", type=str, metavar="MODEL", help="Evaluate a candidate model name") parser.add_argument("--adopt", nargs=2, metavar=("MODEL", "ROLE"), help="Adopt model for role after benchmarking") parser.add_argument("--bank", type=str, help="Task bank JSON for benchmarking") args = parser.parse_args() if args.seed: seed_current_stack() print() print_status() elif args.scan: models = scan_and_register() print(f"\nScanned {len(models)} models:") for m in models: flag = "⚠️ " if m.status == "oversized" else " " print(f" {flag}{m.name} | {m.size_gb:.1f}GB | {m.quantization} | {m.role} | {m.source}") print() print_status() elif args.status: print_status() elif args.history: print_history() elif args.benchmark: role = args.benchmark.lower() print(f"Benchmarking current {role} model...") current = _get_active_model(role) if not current: print(f" No active {role} model. Run --scan first, then --adopt.") return result = benchmark_model(current["path"], role, args.bank) print(f" Score: {result['score']:.3f} ({result['high_confidence']}/{result['task_count']} high confidence)") print(f" Avg time: {result['avg_time_secs']:.1f}s per task") # Update registry _save_model(ModelInfo( name=current["name"], path=current["path"], size_gb=current["size_gb"], role=role, source=current["source"], quantization=current["quantization"], status="active", benchmark_score=result["score"], benchmark_tasks=result["task_count"], adopted_at=current.get("adopted_at"), )) elif args.adopt: model_name, role = args.adopt role = role.lower() models = _get_all_models() candidate = next((m for m in models if m["name"] == model_name), None) if not candidate: print(f" Model '{model_name}' not in registry. Run --scan first.") return print(f" Benchmarking {model_name} for {role}...") result = benchmark_model(candidate["path"], role, args.bank) print(f" Benchmark: {result['score']:.3f} ({result['high_confidence']}/{result['task_count']})") adopt_model(model_name, role, result) else: parser.print_help() if __name__ == "__main__": main()