Translate one segment at a time for the local CPU backend

Batching doesn't help on CPU the way it does on GPU: a single sequence's
matmuls already use all available threads, so grouping several segments into
one generate() call just pads shorter ones to the longest in the group
(wasted compute) and delays every result in the group until the slowest one
finishes. Looping one at a time removes that padding waste and lets the UI
show each translation as soon as it's ready, with per-segment progress and
stop-checking instead of per-chunk. OpenRouter's batched path (which has a
real reason to batch: fewer HTTP round-trips and shared "preceding" context)
is unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

engine/translate.py

@@ -46,6 +46,24 @@ def translate_segments(
     total = len(translatable)
     print(f"[INFO] Translating {total} segment(s) via {'OpenRouter' if key else 'local CPU model'}.")
 
+    if not key:
+        # No GPU to exploit here, so there's no throughput win from batching on CPU
+        # (a single sequence's matmuls already use all available threads) — batching
+        # would only add padding waste and delay every result in a chunk until the
+        # slowest one finishes. One at a time also means the UI can show each
+        # translation the moment it's ready instead of waiting on a whole chunk.
+        for i, (idx, text) in enumerate(translatable):
+            if stop is not None and stop.is_set():
+                break
+            if progress_callback:
+                progress_callback(i, total)
+            try:
+                segments[idx]["target"] = local_backend.translate_batch([text])[0]
+            except Exception as e:
+                print(f"[WARN] Local translation failed for segment {idx + 1}: {e}")
+                errors.append(f"Segment {idx + 1}: {e}")
+        return segments, errors
+
     recent: list[str] = []
     for batch_start in range(0, total, _BATCH_SIZE):
         if stop is not None and stop.is_set():
@@ -57,34 +75,25 @@ def translate_segments(
         if progress_callback:
             progress_callback(batch_start, total)
 
-        if key:
-            preceding = recent[-3:] if recent else None
-            translations = openrouter_backend.translate_batch(texts, key, model, template, preceding=preceding)
-            if translations is None:
-                translations = []
-                for idx, text in zip(indices, texts):
-                    if stop is not None and stop.is_set():
-                        break
-                    try:
-                        t = openrouter_backend.translate_one(text, key, model, template)
-                    except Exception as e:
-                        print(f"[WARN] Segment {idx + 1} failed: {e}")
-                        errors.append(f"Segment {idx + 1}: {e}")
-                        t = ""
-                    translations.append(t)
-                    recent.append(t)
-                for idx, t in zip(indices, translations):
-                    if t:
-                        segments[idx]["target"] = t
-                continue
-        else:
-            try:
-                translations = local_backend.translate_batch(texts)
-            except Exception as e:
-                print(f"[WARN] Local batch translation failed: {e}")
-                for idx in indices:
+        preceding = recent[-3:] if recent else None
+        translations = openrouter_backend.translate_batch(texts, key, model, template, preceding=preceding)
+        if translations is None:
+            translations = []
+            for idx, text in zip(indices, texts):
+                if stop is not None and stop.is_set():
+                    break
+                try:
+                    t = openrouter_backend.translate_one(text, key, model, template)
+                except Exception as e:
+                    print(f"[WARN] Segment {idx + 1} failed: {e}")
                     errors.append(f"Segment {idx + 1}: {e}")
-                continue
+                    t = ""
+                translations.append(t)
+                recent.append(t)
+            for idx, t in zip(indices, translations):
+                if t:
+                    segments[idx]["target"] = t
+            continue
 
         for idx, t in zip(indices, translations):
             segments[idx]["target"] = t

tests/test_translate.py

@@ -1,5 +1,5 @@
 import threading
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 import pytest
 
@@ -87,12 +87,70 @@ def test_translate_segments_skips_empty_source():
 def test_translate_segments_local_backend_path(monkeypatch):
     monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
     segments = _segments("a", "b")
-    with patch("engine.translate.local_backend.translate_batch", return_value=["A", "B"]):
+    # local backend is called one segment at a time (see test_local_backend_calls_one_at_a_time),
+    # so the fake must respond per-call rather than returning one fixed list for all calls.
+    with patch("engine.translate.local_backend.translate_batch", side_effect=lambda texts: [texts[0].upper()]):
         result, errors = tr.translate_segments(segments, None, "model")
     assert [s["target"] for s in result] == ["A", "B"]
     assert errors == []
 
 
+def test_translate_segments_local_backend_calls_one_at_a_time(monkeypatch):
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    segments = _segments("a", "b", "c")
+    with patch(
+        "engine.translate.local_backend.translate_batch", side_effect=lambda texts: [t.upper() for t in texts]
+    ) as mock_local:
+        tr.translate_segments(segments, None, "model")
+    assert mock_local.call_args_list == [call(["a"]), call(["b"]), call(["c"])]
+
+
+def test_translate_segments_local_backend_progress_callback_per_segment(monkeypatch):
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    segments = _segments("a", "b", "c")
+    progress_calls = []
+    with patch("engine.translate.local_backend.translate_batch", side_effect=lambda texts: [t.upper() for t in texts]):
+        tr.translate_segments(
+            segments, None, "model",
+            progress_callback=lambda i, total: progress_calls.append((i, total)),
+        )
+    assert progress_calls == [(0, 3), (1, 3), (2, 3)]
+
+
+def test_translate_segments_local_backend_respects_stop_between_segments(monkeypatch):
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    segments = _segments("a", "b", "c")
+    stop = threading.Event()
+
+    def fake_batch(texts):
+        if texts == ["b"]:
+            stop.set()
+        return [t.upper() for t in texts]
+
+    with patch("engine.translate.local_backend.translate_batch", side_effect=fake_batch):
+        result, errors = tr.translate_segments(segments, None, "model", stop=stop)
+
+    assert [s["target"] for s in result] == ["A", "B", ""]
+    assert errors == []
+
+
+def test_translate_segments_local_backend_one_segment_error_does_not_stop_others(monkeypatch):
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    segments = _segments("a", "b", "c")
+
+    def fake_batch(texts):
+        if texts == ["b"]:
+            raise RuntimeError("boom")
+        return [t.upper() for t in texts]
+
+    with patch("engine.translate.local_backend.translate_batch", side_effect=fake_batch):
+        result, errors = tr.translate_segments(segments, None, "model")
+
+    assert [s["target"] for s in result] == ["A", "", "C"]
+    assert len(errors) == 1
+    assert "Segment 2" in errors[0]
+
+
 def test_translate_segments_local_backend_error_recorded(monkeypatch):
     monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
     segments = _segments("a", "b")