From 205bcfa149e50224944e8f40b6885ef5e1e6fe9d Mon Sep 17 00:00:00 2001 From: maocheng23 Date: Sun, 5 Jul 2026 16:51:37 -0700 Subject: [PATCH 1/4] fix(sglang): make the DataFlow capture backend work on sglang 0.5.14 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the sglang 0.5.14 upgrade onto up-16's new-layout target-capture backend. On `main` the 0.5.14 bump landed via #605 (eagle3 / custom backends) plus the follow-up `patch/fix_sgl0.5.14_dflash` (dflash), but both target the OLD `specforge/modeling/target/dflash_target_model.py` layout. up-16 consolidated eagle3 + dflash extend into a single `SGLangCaptureBackend` (`inference/target_engine/sglang_backend/capture.py`), so the same API adaptations are applied there once, covering both paths: - import `prepare_mlp_sync_batch_raw` from `managers.scheduler_components.dp_attn` (moved from `managers.scheduler_dp_attn_mixin` in 0.5.14). - `build()`: replicate the post-load setup 0.5.14 split out of `ModelRunner.initialize()` — `alloc_memory_pool()` / `init_attention_backends()` / `init_cuda_graphs()` — since we drive the ModelRunner directly; and set `chunked_prefill_size=-1` (we prefill the whole batch in one `_forward_extend`, no scheduler chunking). - `_forward_extend()`: drop the removed `get_model_worker_batch()` step — `ForwardBatch.init_new` now consumes the `ScheduleBatch` directly and reads `capture_hidden_mode` off it; and materialize `prefill_input_ids_cpu` -> device (the scheduler normally does this in `resolve_forward_inputs`). - Req construction (dflash + eagle3 + eagle3-vlm): `fill_ids` was removed in favor of `full_untruncated_fill_ids` (array) + integer `fill_len`. - capture `input_lens` BEFORE the forward — `origin_input_ids` is released during prepare_for_extend/forward on 0.5.13+. - bump pyproject pin to `sglang==0.5.14`. Benefits eagle3 + dflash (+ dspark, which builds on the dflash path). Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 +- .../target_engine/sglang_backend/capture.py | 80 ++++++++++++++++--- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9d8fc8cff..01c30ef82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "accelerate", "pydantic", "pyyaml", - "sglang==0.5.9", + "sglang==0.5.14", "openai-harmony", "ninja", "packaging", diff --git a/specforge/inference/target_engine/sglang_backend/capture.py b/specforge/inference/target_engine/sglang_backend/capture.py index 2e5901e86..222a4f636 100644 --- a/specforge/inference/target_engine/sglang_backend/capture.py +++ b/specforge/inference/target_engine/sglang_backend/capture.py @@ -36,6 +36,7 @@ from __future__ import annotations +from array import array from typing import List, Optional, Tuple import sglang.srt.managers.mm_utils as mm_utils @@ -55,8 +56,10 @@ ScheduleBatch, ) -# prepare_mlp_sync_batch_raw is a module-level function, not a Scheduler method -from sglang.srt.managers.scheduler_dp_attn_mixin import prepare_mlp_sync_batch_raw +# prepare_mlp_sync_batch_raw is a module-level function, not a Scheduler method. +# sglang 0.5.14 moved it from managers.scheduler_dp_attn_mixin to +# managers.scheduler_components.dp_attn. +from sglang.srt.managers.scheduler_components.dp_attn import prepare_mlp_sync_batch_raw from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.radix_cache import RadixCache from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardBatch @@ -119,15 +122,21 @@ def build( original eagle3 path). """ tp_size = dist.get_world_size(get_tp_group()) - # NOTE: sglang 0.5.9 requires dtype to be non-None. If torch_dtype is None, + # NOTE: sglang 0.5.13+ requires dtype to be non-None. If torch_dtype is None, # use "auto" to let sglang decide the dtype. dtype_arg = torch_dtype if torch_dtype is not None else "auto" + # NOTE: DFlash/EAGLE3 prefill the whole batch in one _forward_extend call + # (no scheduler chunking). sglang 0.5.14's eager runner pre-allocates + # per-call buffers sized to chunked_prefill_size (default 8192), and copies + # into them fail with a shape mismatch when our token count exceeds that. + # Disable chunked prefill so the ceiling grows to max_total_num_tokens. server_args = ServerArgs( model_path=pretrained_model_name_or_path, trust_remote_code=trust_remote_code, dtype=dtype_arg, enable_return_hidden_states=True, disable_cuda_graph=True, # we use piecewise cuda graph for prefill instead + chunked_prefill_size=-1, tp_size=tp_size, pp_size=1, **kwargs, @@ -152,6 +161,14 @@ def build( nccl_port=None, is_draft_worker=False, ) + # sglang 0.5.14 split the post-load setup out of ModelRunner.initialize() + # (which now only loads the weights). The scheduler/TpModelWorker perform + # these steps explicitly; since we drive the ModelRunner directly, we must + # replicate them so `req_to_token_pool`/`token_to_kv_pool_allocator` exist + # and forward() has an attention backend and (eager) runner. + model_runner.alloc_memory_pool() + model_runner.init_attention_backends() + model_runner.init_cuda_graphs() if wrap_eagle3_logits: wrap_eagle3_logits_processors_in_module( model_runner.model, return_full_logits=return_full_logits @@ -256,8 +273,20 @@ def _forward_extend(self, reqs): ) batch.prepare_for_extend() self._maybe_prepare_mlp_sync_batch(batch) - model_worker_batch = batch.get_model_worker_batch() - forward_batch = ForwardBatch.init_new(model_worker_batch, self.model_runner) + # sglang 0.5.13+: prepare_for_extend stages input_ids on pinned CPU + # (prefill_input_ids_cpu) and leaves batch.input_ids=None; the scheduler + # normally materializes them to device via resolve_forward_inputs. We bypass + # the scheduler, so perform that prefill H2D copy here. + if getattr(batch, "prefill_input_ids_cpu", None) is not None: + batch.input_ids = batch.prefill_input_ids_cpu.to( + batch.device, non_blocking=True + ) + batch.prefill_input_ids_cpu = None + # sglang 0.5.13+: the ModelWorkerBatch step was removed. ForwardBatch.init_new + # now consumes the ScheduleBatch directly and reads capture_hidden_mode from + # it, so set it on the batch before init_new. + batch.capture_hidden_mode = CaptureHiddenMode.FULL + forward_batch = ForwardBatch.init_new(batch, self.model_runner) forward_batch.capture_hidden_mode = CaptureHiddenMode.FULL output = self.model_runner.forward(forward_batch) return output.logits_output if hasattr(output, "logits_output") else output @@ -291,8 +320,10 @@ def _forward_eagle3_reqs( self._set_eagle3_logits_flags( return_last_hidden_states, return_logits, shard_returns ) - eagle3_output = self._forward_extend(reqs) + # sglang 0.5.13+: capture input lengths BEFORE the forward — prepare_for_extend + # / forward release per-req fields (origin_input_ids becomes None afterwards). input_lens = [len(req.origin_input_ids) for req in reqs] + eagle3_output = self._forward_extend(reqs) logits = eagle3_output.logits aux_hidden_states = eagle3_output.aux_hidden_states @@ -370,8 +401,15 @@ def extend_eagle3( origin_input_ids=input_id_.view(-1).tolist(), sampling_params=sampling_params, ) - req.fill_ids = req.origin_input_ids - req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + # sglang 0.5.13+: Req.fill_ids was removed in favor of + # full_untruncated_fill_ids (origin + output ids) plus an integer + # fill_len, which the scheduler's PrefillAdder sets during admission. + # We bypass the scheduler, so replicate that here with no prefix-cache + # reuse (prefix_indices stays empty). prepare_for_extend asserts + # fill_len - len(prefix_indices) == extend_input_len. + req.full_untruncated_fill_ids = array("q", req.origin_input_ids) + req.fill_len = len(req.full_untruncated_fill_ids) + req.extend_input_len = req.fill_len - len(req.prefix_indices) req.logprob_start_len = len(req.origin_input_ids) - 1 data_cache.append([input_id_, attention_mask_, loss_mask_]) reqs.append(req) @@ -542,8 +580,15 @@ def extend_eagle3_vlm( origin_input_ids=input_id_list, sampling_params=sampling_params, ) - req.fill_ids = req.origin_input_ids - req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + # sglang 0.5.13+: Req.fill_ids was removed in favor of + # full_untruncated_fill_ids (origin + output ids) plus an integer + # fill_len, which the scheduler's PrefillAdder sets during admission. + # We bypass the scheduler, so replicate that here with no prefix-cache + # reuse (prefix_indices stays empty). prepare_for_extend asserts + # fill_len - len(prefix_indices) == extend_input_len. + req.full_untruncated_fill_ids = array("q", req.origin_input_ids) + req.fill_len = len(req.full_untruncated_fill_ids) + req.extend_input_len = req.fill_len - len(req.prefix_indices) req.logprob_start_len = len(req.origin_input_ids) - 1 req.multimodal_inputs = mm_inputs data_cache.append([input_id_, attention_mask_, loss_mask_]) @@ -595,13 +640,22 @@ def extend_dflash( origin_input_ids=curr_ids.view(-1).tolist(), sampling_params=sampling_params, ) - req.fill_ids = req.origin_input_ids - req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + # sglang 0.5.13+: Req.fill_ids was removed in favor of + # full_untruncated_fill_ids (origin + output ids) plus an integer + # fill_len, which the scheduler's PrefillAdder sets during admission. + # We bypass the scheduler, so replicate that here with no prefix-cache + # reuse (prefix_indices stays empty). prepare_for_extend asserts + # fill_len - len(prefix_indices) == extend_input_len. + req.full_untruncated_fill_ids = array("q", req.origin_input_ids) + req.fill_len = len(req.full_untruncated_fill_ids) + req.extend_input_len = req.fill_len - len(req.prefix_indices) data_cache.append((curr_ids, curr_attn, curr_loss)) reqs.append(req) - output = self._forward_extend(reqs) + # sglang 0.5.13+: capture input lengths BEFORE the forward (origin_input_ids + # becomes None afterwards). input_lens = [len(req.origin_input_ids) for req in reqs] + output = self._forward_extend(reqs) if ( hasattr(output, "aux_hidden_states") and output.aux_hidden_states is not None From a9793957839e6989600c94a82b289a3eb6036b54 Mon Sep 17 00:00:00 2001 From: maocheng23 Date: Sun, 5 Jul 2026 16:57:57 -0700 Subject: [PATCH 2/4] fix(sglang): port parallel-state init in patch.py to sglang 0.5.14 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more 0.5.14 API drifts in the vendored parallel_state setup (surfaced building the target on 0.5.14, mirrors main's post-#605 patch.py): - init_model_parallel_group() dropped the pynccl_use_current_stream kwarg (removed in 0.5.13) — remove it from the tp and pdmux_prefill_tp calls. - compute_dp_attention_world_info() now returns a 4-tuple (attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size); the attn-tp rank/size are no longer module globals, so keep only _ATTN_DP_RANK. Validated: sglang DFlash capture (Qwen2.5-0.5B) runs on sglang 0.5.14 and returns correctly-shaped concatenated hidden states. Co-Authored-By: Claude Opus 4.8 --- .../inference/target_engine/sglang_backend/patch.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specforge/inference/target_engine/sglang_backend/patch.py b/specforge/inference/target_engine/sglang_backend/patch.py index 1ec608f8e..0b15d38d0 100644 --- a/specforge/inference/target_engine/sglang_backend/patch.py +++ b/specforge/inference/target_engine/sglang_backend/patch.py @@ -132,6 +132,8 @@ def initialize_model_parallel( # message queue broadcaster is only used in tensor model parallel group # NOTE: torch_compile parameter was removed in sglang 0.5.9 + # NOTE: pynccl_use_current_stream was removed from init_model_parallel_group + # in sglang 0.5.13 (was present in 0.5.9) parallel_state._TP = init_model_parallel_group( group_ranks, parallel_state._WORLD.local_rank, @@ -140,7 +142,6 @@ def initialize_model_parallel( "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER", "true" ), group_name="tp", - pynccl_use_current_stream=duplicate_tp_group, ) if duplicate_tp_group: @@ -156,7 +157,6 @@ def initialize_model_parallel( "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER", "true" ), group_name="pdmux_prefill_tp", - pynccl_use_current_stream=True, ) # NOTE: Check pynccl_comm exists before accessing it (may be None in sglang 0.5.9) if parallel_state._TP.pynccl_comm is not None: @@ -356,10 +356,15 @@ def initialize_dp_attention( dp_attention._ENABLE_DP_ATTENTION_FLAG = enable_dp_attention # NOTE: Added attn_cp_size parameter for sglang 0.5.9 + # NOTE: sglang 0.5.13 - compute_dp_attention_world_info now returns a 4-tuple + # (attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size). The attn-tp + # rank/size are no longer module globals (derived from the _ATTN_TP group), + # so we only keep _ATTN_DP_RANK, mirroring sglang's initialize_dp_attention. ( - dp_attention._ATTN_TP_RANK, - dp_attention._ATTN_TP_SIZE, + _, + _, dp_attention._ATTN_DP_RANK, + _, ) = compute_dp_attention_world_info( enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size ) From cb54fcb274cfa1b87a935b7390db5f20449108f5 Mon Sep 17 00:00:00 2001 From: maocheng23 Date: Sun, 5 Jul 2026 17:20:21 -0700 Subject: [PATCH 3/4] style(sglang): trim verbose version-porting comments in capture backend The 0.5.14 port added multi-line rationale comments at each sglang-API adaptation; several were 5-6 lines and the Req.fill_ids explanation was duplicated verbatim across the three req-construction sites. Collapse each to a concise ": what changed / what we do" line, keeping the version markers (the useful part in a version-coupling boundary file) and dropping the prose. Comments only; no code changes. Co-Authored-By: Claude Opus 4.8 --- .../target_engine/sglang_backend/capture.py | 65 ++++++------------- .../target_engine/sglang_backend/patch.py | 12 ++-- 2 files changed, 25 insertions(+), 52 deletions(-) diff --git a/specforge/inference/target_engine/sglang_backend/capture.py b/specforge/inference/target_engine/sglang_backend/capture.py index 222a4f636..982e40aa2 100644 --- a/specforge/inference/target_engine/sglang_backend/capture.py +++ b/specforge/inference/target_engine/sglang_backend/capture.py @@ -56,9 +56,8 @@ ScheduleBatch, ) -# prepare_mlp_sync_batch_raw is a module-level function, not a Scheduler method. -# sglang 0.5.14 moved it from managers.scheduler_dp_attn_mixin to -# managers.scheduler_components.dp_attn. +# 0.5.14 relocated prepare_mlp_sync_batch_raw to scheduler_components.dp_attn +# (module-level function, not a Scheduler method). from sglang.srt.managers.scheduler_components.dp_attn import prepare_mlp_sync_batch_raw from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.radix_cache import RadixCache @@ -122,14 +121,11 @@ def build( original eagle3 path). """ tp_size = dist.get_world_size(get_tp_group()) - # NOTE: sglang 0.5.13+ requires dtype to be non-None. If torch_dtype is None, - # use "auto" to let sglang decide the dtype. + # 0.5.13+ requires a non-None dtype; "auto" lets sglang decide. dtype_arg = torch_dtype if torch_dtype is not None else "auto" - # NOTE: DFlash/EAGLE3 prefill the whole batch in one _forward_extend call - # (no scheduler chunking). sglang 0.5.14's eager runner pre-allocates - # per-call buffers sized to chunked_prefill_size (default 8192), and copies - # into them fail with a shape mismatch when our token count exceeds that. - # Disable chunked prefill so the ceiling grows to max_total_num_tokens. + # We prefill the whole batch in one forward, so disable chunked prefill: + # 0.5.14's eager runner otherwise caps per-call buffers at chunked_prefill_size + # (default 8192) and errors when our token count exceeds it. server_args = ServerArgs( model_path=pretrained_model_name_or_path, trust_remote_code=trust_remote_code, @@ -161,11 +157,9 @@ def build( nccl_port=None, is_draft_worker=False, ) - # sglang 0.5.14 split the post-load setup out of ModelRunner.initialize() - # (which now only loads the weights). The scheduler/TpModelWorker perform - # these steps explicitly; since we drive the ModelRunner directly, we must - # replicate them so `req_to_token_pool`/`token_to_kv_pool_allocator` exist - # and forward() has an attention backend and (eager) runner. + # 0.5.14 moved post-load setup out of ModelRunner.initialize() (now + # weights-only). We drive the runner directly, so run it here: pools for + # forward(), attention backend, and the (eager) runner. model_runner.alloc_memory_pool() model_runner.init_attention_backends() model_runner.init_cuda_graphs() @@ -273,18 +267,15 @@ def _forward_extend(self, reqs): ) batch.prepare_for_extend() self._maybe_prepare_mlp_sync_batch(batch) - # sglang 0.5.13+: prepare_for_extend stages input_ids on pinned CPU - # (prefill_input_ids_cpu) and leaves batch.input_ids=None; the scheduler - # normally materializes them to device via resolve_forward_inputs. We bypass - # the scheduler, so perform that prefill H2D copy here. + # 0.5.13+ stages input_ids on pinned CPU and leaves batch.input_ids=None; + # the scheduler normally does this H2D copy, but we bypass it. if getattr(batch, "prefill_input_ids_cpu", None) is not None: batch.input_ids = batch.prefill_input_ids_cpu.to( batch.device, non_blocking=True ) batch.prefill_input_ids_cpu = None - # sglang 0.5.13+: the ModelWorkerBatch step was removed. ForwardBatch.init_new - # now consumes the ScheduleBatch directly and reads capture_hidden_mode from - # it, so set it on the batch before init_new. + # 0.5.13+ dropped ModelWorkerBatch; ForwardBatch.init_new reads + # capture_hidden_mode off the ScheduleBatch, so set it first. batch.capture_hidden_mode = CaptureHiddenMode.FULL forward_batch = ForwardBatch.init_new(batch, self.model_runner) forward_batch.capture_hidden_mode = CaptureHiddenMode.FULL @@ -320,8 +311,7 @@ def _forward_eagle3_reqs( self._set_eagle3_logits_flags( return_last_hidden_states, return_logits, shard_returns ) - # sglang 0.5.13+: capture input lengths BEFORE the forward — prepare_for_extend - # / forward release per-req fields (origin_input_ids becomes None afterwards). + # Capture lengths before the forward: 0.5.13+ releases origin_input_ids in it. input_lens = [len(req.origin_input_ids) for req in reqs] eagle3_output = self._forward_extend(reqs) @@ -401,12 +391,8 @@ def extend_eagle3( origin_input_ids=input_id_.view(-1).tolist(), sampling_params=sampling_params, ) - # sglang 0.5.13+: Req.fill_ids was removed in favor of - # full_untruncated_fill_ids (origin + output ids) plus an integer - # fill_len, which the scheduler's PrefillAdder sets during admission. - # We bypass the scheduler, so replicate that here with no prefix-cache - # reuse (prefix_indices stays empty). prepare_for_extend asserts - # fill_len - len(prefix_indices) == extend_input_len. + # 0.5.13+ replaced Req.fill_ids with full_untruncated_fill_ids + int + # fill_len (set by PrefillAdder, which we bypass; no prefix reuse). req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = len(req.full_untruncated_fill_ids) req.extend_input_len = req.fill_len - len(req.prefix_indices) @@ -580,12 +566,8 @@ def extend_eagle3_vlm( origin_input_ids=input_id_list, sampling_params=sampling_params, ) - # sglang 0.5.13+: Req.fill_ids was removed in favor of - # full_untruncated_fill_ids (origin + output ids) plus an integer - # fill_len, which the scheduler's PrefillAdder sets during admission. - # We bypass the scheduler, so replicate that here with no prefix-cache - # reuse (prefix_indices stays empty). prepare_for_extend asserts - # fill_len - len(prefix_indices) == extend_input_len. + # 0.5.13+ replaced Req.fill_ids with full_untruncated_fill_ids + int + # fill_len (set by PrefillAdder, which we bypass; no prefix reuse). req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = len(req.full_untruncated_fill_ids) req.extend_input_len = req.fill_len - len(req.prefix_indices) @@ -640,20 +622,15 @@ def extend_dflash( origin_input_ids=curr_ids.view(-1).tolist(), sampling_params=sampling_params, ) - # sglang 0.5.13+: Req.fill_ids was removed in favor of - # full_untruncated_fill_ids (origin + output ids) plus an integer - # fill_len, which the scheduler's PrefillAdder sets during admission. - # We bypass the scheduler, so replicate that here with no prefix-cache - # reuse (prefix_indices stays empty). prepare_for_extend asserts - # fill_len - len(prefix_indices) == extend_input_len. + # 0.5.13+ replaced Req.fill_ids with full_untruncated_fill_ids + int + # fill_len (set by PrefillAdder, which we bypass; no prefix reuse). req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = len(req.full_untruncated_fill_ids) req.extend_input_len = req.fill_len - len(req.prefix_indices) data_cache.append((curr_ids, curr_attn, curr_loss)) reqs.append(req) - # sglang 0.5.13+: capture input lengths BEFORE the forward (origin_input_ids - # becomes None afterwards). + # Capture lengths before the forward: 0.5.13+ releases origin_input_ids in it. input_lens = [len(req.origin_input_ids) for req in reqs] output = self._forward_extend(reqs) if ( diff --git a/specforge/inference/target_engine/sglang_backend/patch.py b/specforge/inference/target_engine/sglang_backend/patch.py index 0b15d38d0..137e3bd23 100644 --- a/specforge/inference/target_engine/sglang_backend/patch.py +++ b/specforge/inference/target_engine/sglang_backend/patch.py @@ -131,9 +131,7 @@ def initialize_model_parallel( group_ranks.append(ranks) # message queue broadcaster is only used in tensor model parallel group - # NOTE: torch_compile parameter was removed in sglang 0.5.9 - # NOTE: pynccl_use_current_stream was removed from init_model_parallel_group - # in sglang 0.5.13 (was present in 0.5.9) + # NOTE: torch_compile removed in 0.5.9; pynccl_use_current_stream removed in 0.5.13 parallel_state._TP = init_model_parallel_group( group_ranks, parallel_state._WORLD.local_rank, @@ -355,11 +353,9 @@ def initialize_dp_attention( dp_attention._ENABLE_DP_ATTENTION_FLAG = enable_dp_attention - # NOTE: Added attn_cp_size parameter for sglang 0.5.9 - # NOTE: sglang 0.5.13 - compute_dp_attention_world_info now returns a 4-tuple - # (attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size). The attn-tp - # rank/size are no longer module globals (derived from the _ATTN_TP group), - # so we only keep _ATTN_DP_RANK, mirroring sglang's initialize_dp_attention. + # NOTE: attn_cp_size added in 0.5.9. 0.5.13: compute_dp_attention_world_info + # returns a 4-tuple; attn-tp rank/size are no longer module globals, so we keep + # only _ATTN_DP_RANK (mirrors sglang's initialize_dp_attention). ( _, _, From f59f014ca4311cc371719fece9cee4dd1a4214de Mon Sep 17 00:00:00 2001 From: maocheng23 Date: Sun, 5 Jul 2026 17:27:36 -0700 Subject: [PATCH 4/4] fix(sglang): port eagle3 logits processor to 0.5.13+ multi-item delimiter replaced_logits_processor_forward_for_eagle3 read self.multi_item_delimiter, which sglang removed from the LogitsProcessor in 0.5.13 (now carried per-request on the ForwardBatch as multi_item_delimiter_indices). On 0.5.14 that attribute access raises AttributeError on the eagle3 capture path (wrap_eagle3_logits=True); dflash sets it False, so the dflash smoke test did not exercise this. Extract the indices off the ForwardBatch before the LogitsMetadata conversion, mirroring sglang's own logits_processor. Matches the fix in #605. Co-Authored-By: Claude Opus 4.8 --- specforge/inference/target_engine/sglang_backend/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specforge/inference/target_engine/sglang_backend/utils.py b/specforge/inference/target_engine/sglang_backend/utils.py index 8af3bce92..816c8f99e 100644 --- a/specforge/inference/target_engine/sglang_backend/utils.py +++ b/specforge/inference/target_engine/sglang_backend/utils.py @@ -53,17 +53,21 @@ def replaced_logits_processor_forward_for_eagle3( Updated for sglang 0.5.9: - Added hidden_states_before_norm parameter for compatibility """ + # 0.5.13 moved the multi-item delimiter off the LogitsProcessor onto the + # ForwardBatch (multi_item_delimiter_indices); read it before the conversion. + multi_item_delimiter_indices = None if isinstance(logits_metadata, ForwardBatch): + multi_item_delimiter_indices = logits_metadata.multi_item_delimiter_indices logits_metadata = LogitsMetadata.from_forward_batch(logits_metadata) # Multi-item scoring only for prefill-only requests. - if self.multi_item_delimiter is not None and logits_metadata.is_prefill_only: + if multi_item_delimiter_indices is not None and logits_metadata.is_prefill_only: return self.compute_logprobs_for_multi_item_scoring( input_ids, hidden_states, lm_head, logits_metadata, - self.multi_item_delimiter, + multi_item_delimiter_indices, ) # Diffusion LLM only.