Commit ·
8fc86db
1
Parent(s): 0b9e562
Fix gpt-oss harmony leak: drop truncated analysis channel + raise narrator budget
Browse filesThe Narrator's 160-token budget truncated gpt-oss mid-analysis-channel, so
harmony_final (which only stripped when 'assistantfinal' was present) returned
the raw chain-of-thought and it leaked into the public headline and the JSON
parser. Now harmony_final drops an unfinished analysis channel (returns ''),
the narrator gets 512 tokens + a 'Reasoning: low' hint so it reaches its final
channel, and the prompt no longer names a TV show. Reviewer APPROVED.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- tests/test_normalize.py +16 -0
- ttw/narrator.py +6 -2
- ttw/normalize.py +8 -3
tests/test_normalize.py
CHANGED
|
@@ -33,6 +33,22 @@ def test_harmony_without_marker_is_unchanged():
|
|
| 33 |
assert harmony_final(text) == text
|
| 34 |
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
def test_identity_returns_text_unchanged():
|
| 37 |
text = "anything at all, even {broken json"
|
| 38 |
assert identity(text) == text
|
|
|
|
| 33 |
assert harmony_final(text) == text
|
| 34 |
|
| 35 |
|
| 36 |
+
def test_harmony_truncated_analysis_is_dropped():
|
| 37 |
+
"""When the budget runs out before `assistantfinal`, the model is still in
|
| 38 |
+
its analysis channel; raw chain-of-thought must never leak downstream."""
|
| 39 |
+
text = "analysisWe need to produce a punchy headline, optionally a short qu"
|
| 40 |
+
assert harmony_final(text) == ""
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_harmony_empty_string_is_empty():
|
| 44 |
+
assert harmony_final("") == ""
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def test_harmony_final_channel_is_stripped():
|
| 48 |
+
"""Leading/trailing whitespace around the final-channel answer is removed."""
|
| 49 |
+
assert harmony_final('analysis ...assistantfinal {"x":1} ') == '{"x":1}'
|
| 50 |
+
|
| 51 |
+
|
| 52 |
def test_identity_returns_text_unchanged():
|
| 53 |
text = "anything at all, even {broken json"
|
| 54 |
assert identity(text) == text
|
ttw/narrator.py
CHANGED
|
@@ -125,7 +125,10 @@ class DummyNarrator:
|
|
| 125 |
class LLMNarrator:
|
| 126 |
"""gpt-oss-backed Narrator. Reuses the council's gptoss ModalLLM (no add)."""
|
| 127 |
|
| 128 |
-
def __init__(self, client, max_tokens: int =
|
|
|
|
|
|
|
|
|
|
| 129 |
self.client = client
|
| 130 |
self.max_tokens = max_tokens
|
| 131 |
|
|
@@ -142,8 +145,9 @@ class LLMNarrator:
|
|
| 142 |
{
|
| 143 |
"role": "system",
|
| 144 |
"content": (
|
|
|
|
| 145 |
"You are the Narrator of the Wood Street Journal, a "
|
| 146 |
-
"
|
| 147 |
"optionally one short character quip. No spoilers."
|
| 148 |
),
|
| 149 |
},
|
|
|
|
| 125 |
class LLMNarrator:
|
| 126 |
"""gpt-oss-backed Narrator. Reuses the council's gptoss ModalLLM (no add)."""
|
| 127 |
|
| 128 |
+
def __init__(self, client, max_tokens: int = 512):
|
| 129 |
+
# gpt-oss reasons in an analysis channel before its final answer; a tight
|
| 130 |
+
# budget truncates before `assistantfinal`, so give it room to land the
|
| 131 |
+
# final channel (the normalizer drops anything that never gets there).
|
| 132 |
self.client = client
|
| 133 |
self.max_tokens = max_tokens
|
| 134 |
|
|
|
|
| 145 |
{
|
| 146 |
"role": "system",
|
| 147 |
"content": (
|
| 148 |
+
"Reasoning: low\n"
|
| 149 |
"You are the Narrator of the Wood Street Journal, a "
|
| 150 |
+
"high-stakes financial dramatist. One punchy headline, "
|
| 151 |
"optionally one short character quip. No spoilers."
|
| 152 |
),
|
| 153 |
},
|
ttw/normalize.py
CHANGED
|
@@ -14,11 +14,16 @@ from typing import Callable
|
|
| 14 |
def harmony_final(text: str) -> str:
|
| 15 |
"""Return the harmony final channel: text after the first `assistantfinal` marker.
|
| 16 |
|
| 17 |
-
gpt-oss emits `...analysis...assistantfinal<answer>`.
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
| 19 |
"""
|
| 20 |
if "assistantfinal" in text:
|
| 21 |
-
return text.split("assistantfinal", 1)[1]
|
|
|
|
|
|
|
| 22 |
return text
|
| 23 |
|
| 24 |
|
|
|
|
| 14 |
def harmony_final(text: str) -> str:
|
| 15 |
"""Return the harmony final channel: text after the first `assistantfinal` marker.
|
| 16 |
|
| 17 |
+
gpt-oss emits `...analysis...assistantfinal<answer>`. When the generation
|
| 18 |
+
budget runs out before `assistantfinal` the model is still in its analysis
|
| 19 |
+
(chain-of-thought) channel; return "" so raw reasoning never leaks downstream
|
| 20 |
+
(into a headline or into the JSON parser). With no marker and no analysis
|
| 21 |
+
channel the text is already bare, so return it unchanged.
|
| 22 |
"""
|
| 23 |
if "assistantfinal" in text:
|
| 24 |
+
return text.split("assistantfinal", 1)[1].strip()
|
| 25 |
+
if text.lstrip().lower().startswith("analysis"):
|
| 26 |
+
return "" # truncated reasoning, no final channel emitted -- drop it
|
| 27 |
return text
|
| 28 |
|
| 29 |
|