@@ -33,6 +33,7 @@ import {
3333 noopTuiObservability ,
3434 type TuiIncidentSink ,
3535 type TuiObservability ,
36+ type TuiRunLifecycleObservation ,
3637} from '../../../observability/index.js' ;
3738import { formatTuiRuntimeFailure , resolveTuiRuntimeFailure } from './runtime-error-presentation.js' ;
3839import {
@@ -47,6 +48,8 @@ const EVENT_BUS_RECONNECT_MAX_DELAY_MS = 10_000;
4748const INCIDENT_RECONNECT_WINDOW_MS = 30_000 ;
4849const INCIDENT_RECONNECT_FAILURE_THRESHOLD = 3 ;
4950const TERMINAL_RECONCILIATION_TIMEOUT_MS = 2_000 ;
51+ // A projected run must read as settled on two consecutive checks before the TUI clears it.
52+ const STALE_RUN_CHECK_INTERVAL_MS = 5_000 ;
5053
5154type AppendLocalCell = ( content : string , kind ?: 'final-summary' | 'warning' | 'error' ) => void ;
5255
@@ -86,6 +89,8 @@ export interface TuiRuntimeEventFlowOptions {
8689 readonly observability ?: TuiObservability ;
8790 readonly incidentReporter ?: TuiIncidentSink ;
8891 readonly terminalReconciliationTimeoutMs ?: number ;
92+ /** Interval for the stale-run safety net; `0` disables the timer. */
93+ readonly staleRunCheckIntervalMs ?: number ;
8994 readonly notify ?: ( kind : TuiTerminalNotificationKind , key : string ) => void ;
9095}
9196
@@ -105,6 +110,13 @@ export class TuiRuntimeEventFlow {
105110 }
106111 | undefined ;
107112 private readonly llmRetryCalls = new Map < string , TuiLlmRetryEvent > ( ) ;
113+ private staleRunTimer : ReturnType < typeof setInterval > | undefined ;
114+ private staleRunCheck : Promise < boolean > | undefined ;
115+ private staleRunGeneration = 0 ;
116+ private staleRunCandidate :
117+ | { readonly sessionId : string ; readonly turnId : string ; readonly firstSeenAtMs : number }
118+ | undefined ;
119+ private reportedStaleInProcessTurnId : string | undefined ;
108120
109121 constructor ( private readonly options : TuiRuntimeEventFlowOptions ) { }
110122
@@ -139,6 +151,7 @@ export class TuiRuntimeEventFlow {
139151
140152 start ( ) : void {
141153 if ( this . task || this . options . isStopped ( ) ) return ;
154+ this . startStaleRunWatchdog ( ) ;
142155 this . abortController = new AbortController ( ) ;
143156 const busController = this . abortController ;
144157 const task = this . consume ( busController ) ;
@@ -161,6 +174,10 @@ export class TuiRuntimeEventFlow {
161174
162175 stop ( ) : void {
163176 this . clearLlmRetry ( ) ;
177+ if ( this . staleRunTimer ) clearInterval ( this . staleRunTimer ) ;
178+ this . staleRunTimer = undefined ;
179+ this . staleRunGeneration += 1 ;
180+ this . staleRunCandidate = undefined ;
164181 this . abortController ?. abort ( ) ;
165182 this . liveTurn ?. controller . abort ( ) ;
166183 void this . task ?. catch ( ( ) => undefined ) ;
@@ -171,6 +188,8 @@ export class TuiRuntimeEventFlow {
171188
172189 restart ( ) : void {
173190 if ( this . options . isStopped ( ) ) return ;
191+ this . staleRunGeneration += 1 ;
192+ this . staleRunCandidate = undefined ;
174193 const previousTask = this . task ;
175194 this . abortController ?. abort ( ) ;
176195 if ( this . task === previousTask ) this . task = undefined ;
@@ -187,6 +206,172 @@ export class TuiRuntimeEventFlow {
187206 this . clearLlmRetry ( ) ;
188207 }
189208
209+ /**
210+ * Safety net for a run the TUI still projects after Runtime has settled it.
211+ *
212+ * Runtime is authoritative for Turn ownership. When the TUI keeps a Runtime-owned Turn
213+ * (adopted, recovered, or queue-started) while two consecutive reads report no running
214+ * Turn and no Queue handoff, the missed terminal is reconciled locally so the activity
215+ * line, Enter routing, and Queue admission stop treating the Session as busy. A
216+ * foreground in-process submission owns its own completion and is only reported.
217+ */
218+ reconcileStaleRun ( ) : Promise < boolean > {
219+ if ( ! this . staleRunCheck ) {
220+ const check = this . checkStaleRun ( )
221+ . catch ( ( error : unknown ) => {
222+ this . staleRunCandidate = undefined ;
223+ throw error ;
224+ } )
225+ . finally ( ( ) => {
226+ if ( this . staleRunCheck === check ) this . staleRunCheck = undefined ;
227+ } ) ;
228+ this . staleRunCheck = check ;
229+ }
230+ return this . staleRunCheck ;
231+ }
232+
233+ private startStaleRunWatchdog ( ) : void {
234+ const intervalMs = this . options . staleRunCheckIntervalMs ?? STALE_RUN_CHECK_INTERVAL_MS ;
235+ if ( this . staleRunTimer || intervalMs <= 0 ) return ;
236+ this . staleRunTimer = setInterval ( ( ) => {
237+ void this . reconcileStaleRun ( ) . catch ( ( ) => undefined ) ;
238+ } , intervalMs ) ;
239+ this . staleRunTimer . unref ?.( ) ;
240+ }
241+
242+ private projectedRunTurnId ( ) : string | undefined {
243+ return (
244+ this . options . controller . snapshot ( ) . activeTurnId ??
245+ this . options . runProjection . snapshot ( ) . latestRuntimeTurnId ??
246+ this . liveTurn ?. turnId
247+ ) ;
248+ }
249+
250+ private staleRunOwnershipBlocksReconciliation (
251+ sessionId : string ,
252+ projectedTurnId : string ,
253+ ) : boolean {
254+ const snapshot = this . options . controller . snapshot ( ) ;
255+ const runProjection = this . options . runProjection . snapshot ( ) ;
256+ return (
257+ this . options . isStopped ( ) ||
258+ snapshot . session ?. sessionId !== sessionId ||
259+ this . projectedRunTurnId ( ) !== projectedTurnId ||
260+ Boolean ( snapshot . retiringTurnId ) ||
261+ runProjection . queueHandoffPending ||
262+ Boolean ( runProjection . stoppingRuntimeTurnId ) ||
263+ this . options . interactionFlow . continuesTurn ( sessionId , projectedTurnId )
264+ ) ;
265+ }
266+
267+ private async checkStaleRun ( ) : Promise < boolean > {
268+ const snapshot = this . options . controller . snapshot ( ) ;
269+ const sessionId = snapshot . session ?. sessionId ;
270+ const projectedTurnId = this . projectedRunTurnId ( ) ;
271+ if (
272+ ! sessionId ||
273+ ! projectedTurnId ||
274+ this . staleRunOwnershipBlocksReconciliation ( sessionId , projectedTurnId )
275+ ) {
276+ this . staleRunCandidate = undefined ;
277+ return false ;
278+ }
279+ const generation = this . staleRunGeneration ;
280+ const [ activeRun , queued ] = await Promise . all ( [
281+ this . options . runtime . getActiveRun ( sessionId ) ,
282+ this . options . runtime . listQueuedMessages ( sessionId ) ,
283+ ] ) ;
284+ if ( generation !== this . staleRunGeneration ) return false ;
285+ if ( this . staleRunOwnershipBlocksReconciliation ( sessionId , projectedTurnId ) ) {
286+ this . staleRunCandidate = undefined ;
287+ return false ;
288+ }
289+ const runtimeBusy =
290+ activeRun . state === 'running' ||
291+ activeRun . state === 'decision-blocked' ||
292+ queued . some ( ( item ) => item . status === 'accepted' || item . status === 'running' ) ;
293+ if ( runtimeBusy ) {
294+ this . staleRunCandidate = undefined ;
295+ return false ;
296+ }
297+ const nowMs = Date . now ( ) ;
298+ const candidate = this . staleRunCandidate ;
299+ if ( candidate ?. sessionId !== sessionId || candidate . turnId !== projectedTurnId ) {
300+ this . staleRunCandidate = { sessionId, turnId : projectedTurnId , firstSeenAtMs : nowMs } ;
301+ return false ;
302+ }
303+ const inProcessTurnId = this . options . controller . snapshot ( ) . activeTurnId ;
304+ const observation = {
305+ sessionId,
306+ projectedTurnId,
307+ runtimeState : activeRun . state ,
308+ queuedCount : queued . filter ( ( item ) => item . status === 'queued' ) . length ,
309+ stalledForMs : Math . max ( 0 , nowMs - candidate . firstSeenAtMs ) ,
310+ ...( activeRun . turnId ? { turnId : activeRun . turnId } : { } ) ,
311+ ...( inProcessTurnId ? { inProcessTurnId } : { } ) ,
312+ } ;
313+ if ( inProcessTurnId ) {
314+ if ( this . reportedStaleInProcessTurnId !== inProcessTurnId ) {
315+ this . reportedStaleInProcessTurnId = inProcessTurnId ;
316+ this . recordRunLifecycle ( { kind : 'stale-in-process-run' , ...observation } ) ;
317+ this . breadcrumb ( 'cli.run.stale_in_process' , {
318+ runtimeState : activeRun . state ,
319+ stalledForMs : observation . stalledForMs ,
320+ } ) ;
321+ }
322+ return false ;
323+ }
324+
325+ this . staleRunCandidate = undefined ;
326+ const liveTurn = this . liveTurn ;
327+ if ( liveTurn ?. sessionId === sessionId && liveTurn . turnId === projectedTurnId ) {
328+ liveTurn . controller . abort ( ) ;
329+ void liveTurn . task . catch ( ( ) => undefined ) ;
330+ this . liveTurn = undefined ;
331+ this . options . controller . runtimeTurnSettlement . settleProjection ( projectedTurnId , 'succeeded' ) ;
332+ }
333+ this . options . runProjection . reconcileRuntimeTurn ( undefined ) ;
334+ this . recordRunLifecycle ( { kind : 'stale-run-reconciled' , ...observation } ) ;
335+ this . breadcrumb ( 'cli.run.stale_reconciled' , {
336+ runtimeState : activeRun . state ,
337+ stalledForMs : observation . stalledForMs ,
338+ } ) ;
339+ this . options . onChanged ( ) ;
340+ await this . refreshRuntimeSessionProjection ( sessionId , { includeDurableHistory : true } ) . catch (
341+ ( ) => false ,
342+ ) ;
343+ await this . options . activeRunFlow . refresh ( true ) . catch ( ( ) => undefined ) ;
344+ this . options . onChanged ( ) ;
345+ return true ;
346+ }
347+
348+ private recordRunLifecycle ( observation : TuiRunLifecycleObservation ) : void {
349+ try {
350+ this . options . observability ?. recordRunLifecycle ?.( observation ) ;
351+ } catch {
352+ // Diagnostics must not affect Runtime event handling.
353+ }
354+ }
355+
356+ private recordTerminalNotOwned (
357+ event : TuiSessionLifecycleEvent ,
358+ sessionId : string ,
359+ reason : 'in-process-turn' | 'projection-mismatch' ,
360+ context : { readonly inProcessTurnId ?: string ; readonly projectedTurnId ?: string } ,
361+ ) : void {
362+ // Only a terminal for a Turn other than the one the TUI holds can leave it stale.
363+ const heldTurnId = context . inProcessTurnId ?? context . projectedTurnId ;
364+ if ( ! event . turnId || event . turnId === heldTurnId ) return ;
365+ this . recordRunLifecycle ( {
366+ kind : 'terminal-not-owned' ,
367+ sessionId,
368+ turnId : event . turnId ,
369+ reason,
370+ ...( context . inProcessTurnId ? { inProcessTurnId : context . inProcessTurnId } : { } ) ,
371+ ...( context . projectedTurnId ? { projectedTurnId : context . projectedTurnId } : { } ) ,
372+ } ) ;
373+ }
374+
190375 applyActiveSessionProjection ( ) : void {
191376 this . clearLlmRetry ( ) ;
192377 const view = selectActiveSessionView ( this . options . stateStore . snapshot ( ) ) ;
@@ -523,6 +708,14 @@ export class TuiRuntimeEventFlow {
523708 ) {
524709 return ;
525710 }
711+ this . recordRunLifecycle ( {
712+ kind : 'runtime-turn-adopted' ,
713+ sessionId,
714+ turnId,
715+ ...( this . options . runProjection . snapshot ( ) . latestRuntimeTurnId
716+ ? { projectedTurnId : this . options . runProjection . snapshot ( ) . latestRuntimeTurnId }
717+ : { } ) ,
718+ } ) ;
526719 this . startLiveTurn (
527720 sessionId ,
528721 turnId ,
@@ -751,6 +944,14 @@ export class TuiRuntimeEventFlow {
751944
752945 private handleQueuedDrainStarted ( event : TuiSessionLifecycleEvent ) : void {
753946 this . options . runProjection . markQueueTurnStarted ( event . turnId ) ;
947+ if ( event . sessionId ) {
948+ this . recordRunLifecycle ( {
949+ kind : 'queue-turn-started' ,
950+ sessionId : event . sessionId ,
951+ ...( event . turnId ? { turnId : event . turnId } : { } ) ,
952+ queuedCount : event . queueItemIds . length ,
953+ } ) ;
954+ }
754955 for ( const itemId of event . queueItemIds ) {
755956 const cached = this . options . runProjection . findQueueItem ( itemId ) ;
756957 if ( ! cached ) continue ;
@@ -767,7 +968,11 @@ export class TuiRuntimeEventFlow {
767968 liveTurnDurationMs ?: number ,
768969 interactionHandled = false ,
769970 ) : Promise < boolean > {
770- if ( this . options . controller . snapshot ( ) . activeTurnId ) return false ;
971+ const inProcessTurnId = this . options . controller . snapshot ( ) . activeTurnId ;
972+ if ( inProcessTurnId ) {
973+ this . recordTerminalNotOwned ( event , sessionId , 'in-process-turn' , { inProcessTurnId } ) ;
974+ return false ;
975+ }
771976
772977 const activeRun = this . options . activeRunFlow . currentSnapshot ( ) ;
773978 const activeRuntimeTurnId =
@@ -787,7 +992,10 @@ export class TuiRuntimeEventFlow {
787992 ! event . turnId ||
788993 projectedTurnId === event . turnId ||
789994 projectedTurnId . startsWith ( 'session:' ) ;
790- if ( ! eventMatchesProjection ) return false ;
995+ if ( ! eventMatchesProjection ) {
996+ this . recordTerminalNotOwned ( event , sessionId , 'projection-mismatch' , { projectedTurnId } ) ;
997+ return false ;
998+ }
791999
7921000 // Late or background terminal events cannot settle the foreground turn.
7931001 const ownsTerminal = Boolean (
0 commit comments