import logging import re import subprocess from pathlib import Path import gradio as gr logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) # ── Intent Router (inlined, zero-dependency, μs-level) ─────────────── INTENT_RULES = [ (r'\b(prove|reason|explain why|why does|why is|how does|logic|proof)', "reasoning", 0.96), (r'\b(calc|compute|math|add |sub |mul|div|sin|cos|tan|sqrt |factorial|solve )', "tools", 0.95), (r'\b(web |search |find |look up|google|who is |define )', "tools", 0.90), (r'\b(what is (the |a |an ))(?!meaning|purpose|point|reason)', "tools", 0.88), (r'\b(implement |write |code |function |program|algorithm|debug|syntax)', "reasoning", 0.88), (r'\b(convert |translate |summarize |remind |timer |alarm |schedule)', "tools", 0.82), (r'\b(compare|contrast|analyze|evaluate|what if|step by step|break down)', "reasoning", 0.78), (r'\b(think|deep|explain|why|how (do|does|did|would|can|could|should|will|may|might)|would |should |could |might )', "reasoning", 0.60), (r'\b(what is the (meaning|purpose|point|reason) of)', "reasoning", 0.55), ] INTENT_LABELS = { "chat": "💬 Chat", "reasoning": "🧠 Reasoning", "tools": "🔧 Tools", } def classify_intent(text: str) -> str: text_lower = text.lower() for pattern, intent, _score in INTENT_RULES: if re.search(pattern, text_lower): return intent return "chat" # ── LiteRTModel (inlined, no hermes package dependency) ────────────── SUPPORTED_BACKENDS = ["auto", "cpu", "gpu", "ane", "metal", "vulkan"] class LiteRTModel: def __init__(self, model_path: str, cli_path: str = "litert-lm", backend: str = "auto"): self.model_path = Path(model_path).resolve() self.cli_path = cli_path self.backend = backend self.enable_mtp = True self._loaded = False self._detected_backends = ["cpu"] if self.backend not in SUPPORTED_BACKENDS: raise ValueError(f"Unsupported backend '{backend}'. Choose from: {SUPPORTED_BACKENDS}") def load(self) -> bool: if not self.model_path.exists(): log.warning("Model not found: %s", self.model_path) return False with open(self.model_path, "rb") as f: header = f.read(16) if header[:8] != b"LITERTLM": log.warning("Invalid model file (bad magic): %s", self.model_path) return False self._detect_backends() self._loaded = True mb = self.model_path.stat().st_size / 1024 / 1024 log.info("Model loaded: %s (%.1f MB) backends=%s", self.model_path.name, mb, self._detected_backends) return True def _detect_backends(self): keywords = { "gpu": [b"GPU", b"gpu", b"Gpu"], "ane": [b"ANE", b"ane", b"Apple Neural"], "metal": [b"Metal", b"metal", b"MTL"], "vulkan": [b"Vulkan", b"vulkan"], "coreml": [b"CoreML", b"coreml"], "opencl": [b"OpenCL", b"opencl"], "cpu": [b"CPU", b"cpu"], } try: with open(self.model_path, "rb") as f: data = f.read() detected = set() for backend, kws in keywords.items(): if any(kw in data for kw in kws): detected.add(backend) break self._detected_backends = sorted(detected) if detected else ["cpu"] except Exception: self._detected_backends = ["cpu"] def get_supported_backends(self) -> list[str]: return list(self._detected_backends) def get_recommended_backend(self) -> str: for p in ["ane", "metal", "gpu", "vulkan", "coreml"]: if p in self._detected_backends: return p return "cpu" def generate(self, prompt: str, max_tokens: int = 256, temperature: float = 0.6, top_k: int = 40) -> str: if not self._loaded: return "Model not loaded" try: cmd = [self.cli_path, "run", str(self.model_path), "--prompt", prompt, "--max_tokens", str(max_tokens), "--temperature", str(temperature), "--top_k", str(top_k)] if self.backend != "auto": cmd.extend(["--backend", self.backend]) if self.enable_mtp: cmd.append("--enable_mtp") result = subprocess.run(cmd, capture_output=True, text=True, timeout=90) if result.returncode == 0 and result.stdout.strip(): return result.stdout.strip() except FileNotFoundError: log.info("litert-lm CLI not available") except subprocess.TimeoutExpired: log.warning("Model inference timed out") except Exception as exc: log.warning("Inference error: %s", exc) return self._simulate_response(prompt) def _simulate_response(self, prompt: str) -> str: p = prompt.lower() if any(g in p for g in ["hello", "hi", "hey"]): return "Hey! I'm Hermes. What's up?" if any(w in p for w in ["tool", "function"]): return "I support calculator, search, memory, and timer tools. Just ask!" if any(w in p for w in ["reason", "deep", "think"]): return "Let me break this down step by step..." name = self.model_path.stem return f"[{name} · {self.backend}] Received: \"{prompt[:100]}\"" def get_metadata(self) -> dict: return { "path": str(self.model_path), "size_mb": round(self.model_path.stat().st_size / 1024 / 1024, 1), "loaded": self._loaded, "format": "LITERTLM", "supported_backends": self._detected_backends, "recommended_backend": self.get_recommended_backend(), "backend": self.backend, } # ── Model catalog ──────────────────────────────────────────────────── MODELS = { "fast": { "label": "Hermes Edge 270M", "file": "dist/hermes-mobile-270m-int4.litertlm", "url": "https://huggingface.co/bclermo/hermes-edge/resolve/main/dist/hermes-mobile-270m-int4.litertlm", "size": "475 MB", "base": "Qwen3-0.6B", "quant": "Mixed INT4", "speed": "~69 tok/s (GPU)", "strengths": ["Fast chat", "Low memory (180 MB)"], "native": ["ChatML", "General conversation"], "weaknesses": ["No native reasoning", "No native tool calling"], "best_for": "Quick answers, simple Q&A, fast responses", "bench": {"GPU": "69 tok/s", "CPU": "13 tok/s", "ANE": "N/A", "RAM": "180 MB"}, "color": "green", }, "reasoning": { "label": "Hermes Edge Reasoning", "file": "dist/hermes-edge-reasoning-litertlm.litertlm", "url": "https://huggingface.co/bclermo/hermes-edge/resolve/main/dist/hermes-edge-reasoning-litertlm.litertlm", "size": "1.75 GB", "base": "DeepSeek-R1-Distill-Qwen-1.5B", "quant": "INT8", "speed": "~26 tok/s (GPU)", "strengths": ["Native reasoning", "Multi-step logic", "Math & code"], "native": [" chain-of-thought", "ChatML"], "weaknesses": ["Larger (1.75 GB)", "No tool calling", "Slower than 270M"], "best_for": "Math, logic, coding, deep reasoning tasks", "bench": {"GPU": "26 tok/s", "CPU": "8 tok/s", "ANE": "N/A", "RAM": "~1 GB"}, "color": "purple", }, "pro": { "label": "Hermes Edge Pro", "file": "dist/hermes-edge-pro-litertlm.litertlm", "url": "https://huggingface.co/bclermo/hermes-edge/resolve/main/dist/hermes-edge-pro-litertlm.litertlm", "size": "2.5 GB", "base": "Gemma-4-E2B", "quant": "INT4", "speed": "~56 tok/s (GPU, with MTP)", "strengths": ["Native function calling", "Thinking mode toggle", "MTP 2.2x"], "native": ["Function calling (tools)", "Thinking mode", "System role", "MTP"], "weaknesses": ["Largest (2.5 GB)", "Requires iOS 18.2+"], "best_for": "Tool calling, agents, structured tasks, multi-turn", "bench": {"GPU": "56 tok/s", "CPU": "16 tok/s", "ANE": "~52 tok/s", "RAM": "~1.5 GB"}, "color": "amber", }, } LOCAL_DIR = Path("dist") available = {k: (LOCAL_DIR / Path(v["file"]).name).exists() for k, v in MODELS.items()} model = None detected_backends = ["cpu"] recommended_backend = "cpu" backend_options = ["auto"] default_model = "fast" current_model_key = default_model if available.get(default_model): try: m = LiteRTModel(str(LOCAL_DIR / Path(MODELS[default_model]["file"]).name), backend="auto") if m.load(): model = m detected_backends = m.get_supported_backends() recommended_backend = m.get_recommended_backend() backend_options = ["auto"] + [b for b in detected_backends if b != "cpu"] if recommended_backend != "cpu" and model: model.backend = recommended_backend log.info("Backends: %s (recommended: %s)", detected_backends, recommended_backend) except Exception as exc: log.warning("Load failed: %s", exc) # ── UI ─────────────────────────────────────────────────────────────── THEME = gr.themes.Base(primary_hue="indigo", secondary_hue="purple", neutral_hue="slate", font=gr.themes.GoogleFont("Inter")) CUSTOM_CSS = """ #brand-header { text-align: center; margin-bottom: 1.5rem; } #brand-header h1 { font-size: 2rem; font-weight: 700; } .cap-badge { display: inline-block; padding: 0.15rem 0.5rem; border-radius: 999px; font-size: 0.7rem; font-weight: 600; margin: 0.15rem; } .cap-green { background: #dcfce7; color: #166534; } .cap-purple { background: #f3e8ff; color: #6b21a8; } .cap-amber { background: #fef3c7; color: #92400e; } .cap-gray { background: #f3f4f6; color: #374151; } .cap-blue { background: #dbeafe; color: #1e40af; } .model-card { border: 1px solid #e5e7eb; border-radius: 8px; padding: 1rem; margin: 0.5rem 0; } .model-card h3 { margin: 0 0 0.5rem 0; } .model-detail { color: #6b7280; font-size: 0.85rem; margin: 0.15rem 0; } """ def get_model_card(model_key: str) -> str: m = MODELS[model_key] badges = " ".join( f'{s}' for s in m["strengths"] ) native_badges = " ".join( f'{s}' for s in m["native"] ) return f"""

{m["label"]}

{badges}
{native_badges}

Base: {m["base"]} · Quant: {m["quant"]} · Size: {m["size"]}

Speed: {m["speed"]}

Best for: {m["best_for"]}

Limitations

{'
'.join('• ' + w for w in m["weaknesses"])}

""" def on_model_change(model_key: str): global model, current_model_key current_model_key = model_key m = MODELS[model_key] local_path = LOCAL_DIR / Path(m["file"]).name if not local_path.exists(): return get_model_card(model_key) try: new = LiteRTModel(str(local_path), backend="auto") if new.load(): model = new if recommended_backend != "cpu": model.backend = recommended_backend except Exception: pass return get_model_card(model_key) def chat_response(message: str, history: list, model_key: str, backend: str) -> str: global model, current_model_key if model_key != current_model_key: on_model_change(model_key) intent = classify_intent(message) m = MODELS[model_key] routed_intent_label = INTENT_LABELS.get(intent, "💬 Chat") intent_tag = f"[Router → {routed_intent_label}]" if model: model.backend = backend if backend in SUPPORTED_BACKENDS else "auto" try: result = model.generate(message, max_tokens=256, temperature=0.6) # Show routing tag on first response only if history is None or len(history) == 0: return f"{intent_tag}\n\n{result}" return result except Exception as exc: return f"{intent_tag}\n\nError: {exc}" return ( f"{intent_tag}\n\n" f"[{m['label']} · {backend}] " f"Using {m['base']}. Heard: \"{message[:80]}\". " f"Download from Install tab to run offline." ) def update_backend_info(): return ( f"**Detected backends**: {', '.join(detected_backends)}" f" · **Recommended**: `{recommended_backend}`" ) current_backend_val = model.backend if model else recommended_backend with gr.Blocks(theme=THEME, css=CUSTOM_CSS) as demo: gr.HTML( """

Hermes Edge

On-Device AI · 3 Models · LiteRT-LM

""" ) model_selector = gr.Radio( choices=[(MODELS[k]["label"], k) for k in MODELS], value=default_model, label="Select Model", info="Each model excels at different tasks.", ) model_details = gr.HTML(get_model_card(default_model)) model_selector.change(fn=on_model_change, inputs=model_selector, outputs=model_details) with gr.Accordion("Backend Settings", open=False): backend_info = gr.Markdown(update_backend_info()) backend_dropdown = gr.Dropdown( choices=backend_options, value=current_backend_val, label="Compute Backend", interactive=bool(model), ) with gr.Tabs(): with gr.TabItem("Chat"): gr.ChatInterface( fn=chat_response, additional_inputs=[model_selector, backend_dropdown], title=None, theme=THEME, ) with gr.TabItem("Install on iPhone 16"): rows = "".join( f""" {MODELS[k]['label']} {MODELS[k]['size']} {MODELS[k]['speed']} {MODELS[k]['best_for']} Download """ for k in MODELS ) gr.HTML(f"""

Choose your model

Each model has different strengths. Download the one that fits your needs.

{rows}
Model Size Speed Best For Download

How to install

  1. Open Google AI Edge Gallery on your iPhone 16
  2. Tap Import Model
  3. Paste the download URL for your chosen model
  4. The model auto-downloads. Check Settings → Model → Backend for GPU/ANE selection.
""") with gr.TabItem("Capabilities & Benchmarks"): cap_rows = "".join( f""" {label} {val} {val2} {val3} """ for label, val, val2, val3 in [ ("General Chat", "Excellent", "Good", "Excellent"), ("Reasoning (think)", "Prompted only", "Native", "Via toggle"), ("Tool Calling", "Not supported", "Not supported", "Native"), ("Multi-Token Pred.", "Supported", "Supported", "Supported"), ] ) bench_rows = "".join( f""" {label} {v1} {v2} {v3} """ for label, v1, v2, v3 in [ ("GPU (with MTP)", MODELS["fast"]["bench"]["GPU"], MODELS["reasoning"]["bench"]["GPU"], MODELS["pro"]["bench"]["GPU"]), ("CPU", MODELS["fast"]["bench"]["CPU"], MODELS["reasoning"]["bench"]["CPU"], MODELS["pro"]["bench"]["CPU"]), ("ANE", MODELS["fast"]["bench"]["ANE"], MODELS["reasoning"]["bench"]["ANE"], MODELS["pro"]["bench"]["ANE"]), ("Peak RAM", MODELS["fast"]["bench"]["RAM"], MODELS["reasoning"]["bench"]["RAM"], MODELS["pro"]["bench"]["RAM"]), ] ) gr.HTML(f"""

Model Comparison

{cap_rows}
Capability {MODELS['fast']['label']} {MODELS['reasoning']['label']} {MODELS['pro']['label']}
Model Size {MODELS['fast']['size']} {MODELS['reasoning']['size']} {MODELS['pro']['size']}

Speed Benchmarks

{bench_rows}
Backend {MODELS['fast']['label']} {MODELS['reasoning']['label']} {MODELS['pro']['label']}

Benchmarks from LiteRT-LM community data. Actual performance varies by device and iOS version.

""") with gr.TabItem("Routing"): rule_rows = "".join( f""" {p} {INTENT_LABELS.get(i, i)} {s} """ for p, i, s in INTENT_RULES ) gr.HTML(f"""

Intent Routing — Zero Inference

Every user message is classified by a pure keyword+regex pre-router (~1μs, no model call). Based on intent, the agent selects the optimal model and generation parameters:

{rule_rows}
Pattern Intent Confidence

Per-Intent Configuration

Setting 💬 Chat 🧠 Reasoning 🔧 Tools
Model {MODELS['fast']['label']} (always hot) {MODELS['reasoning']['label']} (load on demand) {MODELS['pro']['label']} (load on demand)
Temperature 0.7 (creative) 0.6 (precise) 0.5 (deterministic)
Max Tokens 128 (short) 384 (deep) 256 (structured)
MTP (Speed) ✅ Enabled (2.2×) ❌ Disabled (quality) ✅ Enabled (2.2×)

On device, the 270M model stays hot in ~180 MB RAM. Specialist models load via LiteRT-LM's fast init on demand.

""") gr.HTML( """

Hermes Edge · Built on Raven AI Ecosystem · bclermo · GitHub · Apache 2.0

""" ) if __name__ == "__main__": demo.launch()