@@ -111,29 +111,39 @@ def _validate_trailing(Sources, Destinations, Operations, Expected, Count, Error
111111 WIDTH : tl .constexpr , MemoryCapacity , PAGE_TOKENS : tl .constexpr ,
112112 LIVE : tl .constexpr ):
113113 row = tl .program_id (0 )
114- source , destination = tl .load (Sources + row ), tl . load ( Destinations + row )
114+ source = tl .load (Sources + row )
115115 PAGES = tl .load (MemoryCapacity + 1 )
116- endpoint = tl .maximum ( 0 , tl . load (Expected + row ) - 1 )
116+ endpoint = tl .load (Expected + row )
117117 slot = tl .load (RequestSlots + row )
118118 state = tl .load (StateRows + slot * 8 , (slot >= 0 ) & (slot < LIVE ), other = 0 )
119119 snapshot = source == state
120120 active = (row < tl .load (Count )) & (tl .load (Errors + row ) == 0 ) & (tl .load (Operations + row ) == 0 )
121- tail = active & (endpoint % PAGE_TOKENS != 0 )
122- page = tl .load (Tables + slot * WIDTH + endpoint // PAGE_TOKENS , tail , other = - 1 )
123121 others = tl .arange (0 , LIVE )
124- other_source , other_dest = tl .load (Sources + others ), tl . load ( Destinations + others )
122+ other_source = tl .load (Sources + others )
125123 other_slot = tl .load (RequestSlots + others )
126124 other_state = tl .load (StateRows + other_slot * 8 , (other_slot >= 0 ) & (other_slot < LIVE ), other = 0 )
127125 other_snapshot = other_source == other_state
128- other_endpoint = tl .maximum ( 0 , tl . load (Expected + others ) - 1 )
126+ other_endpoint = tl .load (Expected + others )
129127 other_active = (others < tl .load (Count )) & (others != row ) & (tl .load (Operations + others ) == 0 )
130- other_active &= (other_slot >= 0 ) & (other_slot < LIVE ) & (other_endpoint // PAGE_TOKENS < WIDTH )
131- other_active &= other_endpoint % PAGE_TOKENS != 0
132- other_page = tl .load (Tables + other_slot * WIDTH + other_endpoint // PAGE_TOKENS , other_active , other = - 1 )
133- conflict = tl .sum ((other_active & (page == other_page ) & (~ snapshot | ~ other_snapshot )).to (tl .int32 ), 0 ) > 0
128+ other_active &= (other_slot >= 0 ) & (other_slot < LIVE )
134129 error = tl .load (Errors + row )
135- error = tl .where (tail & ((page < 0 ) | (page >= PAGES )), 6 , error )
136- error = tl .where (tail & conflict , 4 , error )
130+ for part in tl .static_range (2 ):
131+ logical = endpoint // PAGE_TOKENS + part - 1
132+ tail = active & (endpoint > 1 ) & (logical >= 0 )
133+ if part == 1 :
134+ tail &= endpoint % PAGE_TOKENS >= 2
135+ bounded = (logical >= 0 ) & (logical < WIDTH )
136+ page = tl .load (Tables + slot * WIDTH + logical , tail & bounded , other = - 1 )
137+ conflict = tl .full ((), False , tl .int1 )
138+ for other_part in tl .static_range (2 ):
139+ other_logical = other_endpoint // PAGE_TOKENS + other_part - 1
140+ used = other_active & (other_endpoint > 1 ) & (other_logical >= 0 ) & (other_logical < WIDTH )
141+ if other_part == 1 :
142+ used &= other_endpoint % PAGE_TOKENS >= 2
143+ other_page = tl .load (Tables + other_slot * WIDTH + other_logical , used , other = - 1 )
144+ conflict |= tl .sum ((used & (page == other_page ) & (~ snapshot | ~ other_snapshot )).to (tl .int32 ), 0 ) > 0
145+ error = tl .where (tail & ((page < 0 ) | (page >= PAGES )), 6 , error )
146+ error = tl .where (tail & conflict , 4 , error )
137147 tl .store (Errors + row , error )
138148
139149
@@ -156,18 +166,28 @@ def _copy_rows(Backing, Sources, Destinations, Operations, Count, Errors,
156166def _copy_pages (Pointers , Backing , Tables , Sources , Destinations , Operations , Expected , Count , Errors , RequestSlots , StateRows ,
157167 ROW_BYTES : tl .constexpr , PAGE_BYTES : tl .constexpr , WIDTH : tl .constexpr ,
158168 PAGE_TOKENS : tl .constexpr , BLOCK : tl .constexpr , LIVE : tl .constexpr = 4 ,
159- ENDPOINT_SHIFT : tl .constexpr = 0 ):
169+ DRAFT_PART : tl .constexpr = - 1 ):
160170 row , region , tile = tl .program_id (0 ), tl .program_id (1 ), tl .program_id (2 )
161171 source , destination = tl .load (Sources + row ), tl .load (Destinations + row )
162- endpoint = tl .maximum ( 0 , tl . load (Expected + row ) + ENDPOINT_SHIFT )
172+ endpoint = tl .load (Expected + row )
163173 slot = tl .load (RequestSlots + row )
164174 state = tl .load (StateRows + slot * 8 , (slot >= 0 ) & (slot < LIVE ), other = 0 )
165175 snapshot = source == state
166176 active = (row < tl .load (Count )) & (tl .load (Errors + row ) == 0 )
167- active &= (tl .load (Operations + row ) == 0 ) & (endpoint % PAGE_TOKENS != 0 )
168- page = tl .load (Tables + slot * WIDTH + endpoint // PAGE_TOKENS , active , other = 0 )
177+ active &= tl .load (Operations + row ) == 0
178+ logical = endpoint // PAGE_TOKENS
179+ if DRAFT_PART < 0 :
180+ active &= endpoint % PAGE_TOKENS != 0
181+ else :
182+ logical += DRAFT_PART - 1
183+ active &= (endpoint > 1 ) & (logical >= 0 )
184+ if DRAFT_PART == 1 :
185+ active &= endpoint % PAGE_TOKENS >= 2
186+ page = tl .load (Tables + slot * WIDTH + logical , active , other = 0 )
169187 cache = tl .load (Pointers + region * 2 ).to (tl .pointer_type (tl .uint8 ))
170188 offset = tl .load (Pointers + region * 2 + 1 )
189+ if DRAFT_PART >= 0 :
190+ offset += DRAFT_PART * PAGE_BYTES
171191 saved_slot = tl .where (snapshot , destination , source )
172192 index = tile * BLOCK + tl .arange (0 , BLOCK )
173193 saved = Backing + saved_slot * ROW_BYTES + offset + index
@@ -185,6 +205,9 @@ class PreparedStateTransfer:
185205
186206 ``pages`` contains (state region name, contiguous uint8 [pages, page_bytes]
187207 cache) pairs. Cache owners and pointer tables remain alive until graph drain.
208+ ``trailing_pages`` uses a one-page-shifted table with a one-token lag. Each
209+ saved region holds two pages: the preceding boundary page, then the current
210+ one. Both may be private even when the preceding draft page is complete.
188211 These maintenance operations are captured directly. Tensor-parallel callers
189212 must combine validation failures across ranks between validate() and apply().
190213 A nonzero command error suppresses both row and external-page writes.
@@ -233,11 +256,11 @@ def __init__(self, pool: StateStorage, commands: StateCommands, tables, *, pages
233256 _check_tensor (trailing_tables , tables .shape , torch .int32 , device , "trailing_page_tables" )
234257 for name , cache in trailing_pages :
235258 region = pool .layout .region (name )
236- if name in names or region .dtype != torch .uint8 or cache .shape != (page_count , region .nbytes ) :
237- raise ValueError ("trailing partial pages must match the model page capacity and state regions " )
259+ if name in names or region .dtype != torch .uint8 or cache .shape != (page_count , region .nbytes // 2 ) or region . nbytes % 2 :
260+ raise ValueError ("shifted draft pages require two saved pages per cache plane " )
238261 _check_tensor (cache , cache .shape , torch .uint8 , device , name )
239262 names .add (name )
240- trailing_groups [region . nbytes ].append ((cache .data_ptr (), region .offset ))
263+ trailing_groups [cache . shape [ 1 ] ].append ((cache .data_ptr (), region .offset ))
241264 self .trailing_pointers = {size : torch .tensor (values , dtype = torch .int64 , device = device )
242265 for size , values in trailing_groups .items ()}
243266 self .pointers = {size : torch .tensor (values , dtype = torch .int64 , device = device ) for size , values in groups .items ()}
@@ -253,7 +276,7 @@ def __init__(self, pool: StateStorage, commands: StateCommands, tables, *, pages
253276 for size , pointer in self .pointers .items ()]
254277 self .page_args .extend ((pointer , pool .backing , trailing_tables , c .source_slots , c .destination_slots , c .operation ,
255278 c .expected_consumed , c .num_ops , c .errors , c .request_slots , state_rows , pool .layout .row_bytes , size , tables .shape [1 ], page_tokens ,
256- 1024 , self .capacity , - 1 ) for size , pointer in self .trailing_pointers .items ())
279+ 1024 , self .capacity , part ) for size , pointer in self .trailing_pointers .items () for part in ( 0 , 1 ))
257280 self .trailing_validation_args = None if not trailing_pages else (
258281 c .source_slots , c .destination_slots , c .operation , c .expected_consumed , c .num_ops , c .errors ,
259282 trailing_tables , c .request_slots , state_rows , tables .shape [1 ], pool .memory_capacity , page_tokens , self .capacity )
0 commit comments