diff --git a/deepspec/eval/dspark/evaluator.py b/deepspec/eval/dspark/evaluator.py index eba2b34..ffc4fd0 100644 --- a/deepspec/eval/dspark/evaluator.py +++ b/deepspec/eval/dspark/evaluator.py @@ -88,12 +88,32 @@ class Qwen3DSparkEvaluator(BaseEvaluator): initial_output, **kwargs, ) -> SimpleNamespace: + import os + _w = int(os.environ.get("DSPARK_DRAFT_CTX_WINDOW", "0")) + target_hidden_states = extract_context_feature( + initial_output.hidden_states, + self.draft_model.target_layer_ids, + ) + if _w: + from deepspec.eval.windowed_cache import make_draft_cache + past_key_values_draft = make_draft_cache( + self.draft_model, _w, pad=int(self.max_proposal_tokens) + 64 + ) + # window the init context tensor (its 5-layer hidden states are the big init spike) and + # offset the draft cache cumulative so absolute positions stay correct downstream. + _num_input = int(kwargs.get("num_input_tokens", target_hidden_states.shape[1])) + if target_hidden_states.shape[1] > _w: + target_hidden_states = target_hidden_states[:, -_w:, :].contiguous() + # absolute offset = where the retained window starts (prefill may have pre-trimmed it) + _offset = _num_input - target_hidden_states.shape[1] + if _offset > 0: + for _layer in past_key_values_draft.layers: + _layer.cumulative_length = _offset + else: + past_key_values_draft = DynamicCache() return SimpleNamespace( - past_key_values_draft=DynamicCache(), - target_hidden_states=extract_context_feature( - initial_output.hidden_states, - self.draft_model.target_layer_ids, - ), + past_key_values_draft=past_key_values_draft, + target_hidden_states=target_hidden_states, ) def _propose( @@ -164,6 +184,8 @@ class Qwen3DSparkEvaluator(BaseEvaluator): *, input_ids: torch.Tensor, stop_token_ids: list[int] | None, + stream_callback=None, + prefill_mm=None, ) -> SimpleNamespace: return generate_decoding_sample( target_model=self.target_model, @@ -176,6 +198,9 @@ class Qwen3DSparkEvaluator(BaseEvaluator): propose=self._propose, update=self._update, post_verify=self._post_verify, + prefill_keep_hidden_layers=[0 if l == -1 else l + 1 for l in self.draft_model.target_layer_ids], + stream_callback=stream_callback, + prefill_mm=prefill_mm, ) def evaluate(self) -> None: