Skip to content

Commit 5683465

Browse files
authored
fix(tui): sync 0.5.10 prompt and run recovery fixes (#410)
1 parent 3d369f5 commit 5683465

11 files changed

Lines changed: 1109 additions & 16 deletions

File tree

‎packages/tui/src/observability/local-observability.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,27 @@ export interface TuiEventStreamObservation {
3232
readonly errorKind?: string;
3333
}
3434

35+
/**
36+
* Run-ownership transitions the TUI cannot reconstruct from Runtime history alone.
37+
* Session and Turn ids are opaque identifiers; no message content is recorded.
38+
*/
39+
export interface TuiRunLifecycleObservation {
40+
readonly kind:
41+
| 'runtime-turn-adopted'
42+
| 'queue-turn-started'
43+
| 'terminal-not-owned'
44+
| 'stale-run-reconciled'
45+
| 'stale-in-process-run';
46+
readonly sessionId: string;
47+
readonly turnId?: string;
48+
readonly projectedTurnId?: string;
49+
readonly inProcessTurnId?: string;
50+
readonly reason?: string;
51+
readonly runtimeState?: string;
52+
readonly queuedCount?: number;
53+
readonly stalledForMs?: number;
54+
}
55+
3556
export interface TuiTerminalObservation {
3657
readonly terminalId: string;
3758
readonly platform: NodeJS.Platform;
@@ -138,6 +159,7 @@ export interface TuiObservability {
138159
recordStartup(observation: TuiStartupObservation): void;
139160
recordAccess(observation: TuiRuntimeAccessObservation): void;
140161
recordEventStream(observation: TuiEventStreamObservation): void;
162+
recordRunLifecycle?(observation: TuiRunLifecycleObservation): void;
141163
recordTerminal(observation: TuiTerminalObservation): void;
142164
recordProcessStop(observation: TuiProcessStopObservation): void;
143165
recordTheme?(observation: TuiThemeObservation): void;
@@ -268,6 +290,10 @@ class LocalTuiObservability implements TuiObservability {
268290
this.record('event.stream', observation);
269291
}
270292

293+
recordRunLifecycle(observation: TuiRunLifecycleObservation): void {
294+
this.record('run.lifecycle', observation);
295+
}
296+
271297
recordTerminal(observation: TuiTerminalObservation): void {
272298
this.terminal = Object.freeze({ ...observation });
273299
this.record('terminal.capability', observation);

‎packages/tui/src/tui/controller/runtime/runtime-event-flow.ts‎

Lines changed: 210 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
noopTuiObservability,
3434
type TuiIncidentSink,
3535
type TuiObservability,
36+
type TuiRunLifecycleObservation,
3637
} from '../../../observability/index.js';
3738
import { formatTuiRuntimeFailure, resolveTuiRuntimeFailure } from './runtime-error-presentation.js';
3839
import {
@@ -47,6 +48,8 @@ const EVENT_BUS_RECONNECT_MAX_DELAY_MS = 10_000;
4748
const INCIDENT_RECONNECT_WINDOW_MS = 30_000;
4849
const INCIDENT_RECONNECT_FAILURE_THRESHOLD = 3;
4950
const TERMINAL_RECONCILIATION_TIMEOUT_MS = 2_000;
51+
// A projected run must read as settled on two consecutive checks before the TUI clears it.
52+
const STALE_RUN_CHECK_INTERVAL_MS = 5_000;
5053

5154
type AppendLocalCell = (content: string, kind?: 'final-summary' | 'warning' | 'error') => void;
5255

@@ -86,6 +89,8 @@ export interface TuiRuntimeEventFlowOptions {
8689
readonly observability?: TuiObservability;
8790
readonly incidentReporter?: TuiIncidentSink;
8891
readonly terminalReconciliationTimeoutMs?: number;
92+
/** Interval for the stale-run safety net; `0` disables the timer. */
93+
readonly staleRunCheckIntervalMs?: number;
8994
readonly notify?: (kind: TuiTerminalNotificationKind, key: string) => void;
9095
}
9196

@@ -105,6 +110,13 @@ export class TuiRuntimeEventFlow {
105110
}
106111
| undefined;
107112
private readonly llmRetryCalls = new Map<string, TuiLlmRetryEvent>();
113+
private staleRunTimer: ReturnType<typeof setInterval> | undefined;
114+
private staleRunCheck: Promise<boolean> | undefined;
115+
private staleRunGeneration = 0;
116+
private staleRunCandidate:
117+
| { readonly sessionId: string; readonly turnId: string; readonly firstSeenAtMs: number }
118+
| undefined;
119+
private reportedStaleInProcessTurnId: string | undefined;
108120

109121
constructor(private readonly options: TuiRuntimeEventFlowOptions) {}
110122

@@ -139,6 +151,7 @@ export class TuiRuntimeEventFlow {
139151

140152
start(): void {
141153
if (this.task || this.options.isStopped()) return;
154+
this.startStaleRunWatchdog();
142155
this.abortController = new AbortController();
143156
const busController = this.abortController;
144157
const task = this.consume(busController);
@@ -161,6 +174,10 @@ export class TuiRuntimeEventFlow {
161174

162175
stop(): void {
163176
this.clearLlmRetry();
177+
if (this.staleRunTimer) clearInterval(this.staleRunTimer);
178+
this.staleRunTimer = undefined;
179+
this.staleRunGeneration += 1;
180+
this.staleRunCandidate = undefined;
164181
this.abortController?.abort();
165182
this.liveTurn?.controller.abort();
166183
void this.task?.catch(() => undefined);
@@ -171,6 +188,8 @@ export class TuiRuntimeEventFlow {
171188

172189
restart(): void {
173190
if (this.options.isStopped()) return;
191+
this.staleRunGeneration += 1;
192+
this.staleRunCandidate = undefined;
174193
const previousTask = this.task;
175194
this.abortController?.abort();
176195
if (this.task === previousTask) this.task = undefined;
@@ -187,6 +206,172 @@ export class TuiRuntimeEventFlow {
187206
this.clearLlmRetry();
188207
}
189208

209+
/**
210+
* Safety net for a run the TUI still projects after Runtime has settled it.
211+
*
212+
* Runtime is authoritative for Turn ownership. When the TUI keeps a Runtime-owned Turn
213+
* (adopted, recovered, or queue-started) while two consecutive reads report no running
214+
* Turn and no Queue handoff, the missed terminal is reconciled locally so the activity
215+
* line, Enter routing, and Queue admission stop treating the Session as busy. A
216+
* foreground in-process submission owns its own completion and is only reported.
217+
*/
218+
reconcileStaleRun(): Promise<boolean> {
219+
if (!this.staleRunCheck) {
220+
const check = this.checkStaleRun()
221+
.catch((error: unknown) => {
222+
this.staleRunCandidate = undefined;
223+
throw error;
224+
})
225+
.finally(() => {
226+
if (this.staleRunCheck === check) this.staleRunCheck = undefined;
227+
});
228+
this.staleRunCheck = check;
229+
}
230+
return this.staleRunCheck;
231+
}
232+
233+
private startStaleRunWatchdog(): void {
234+
const intervalMs = this.options.staleRunCheckIntervalMs ?? STALE_RUN_CHECK_INTERVAL_MS;
235+
if (this.staleRunTimer || intervalMs <= 0) return;
236+
this.staleRunTimer = setInterval(() => {
237+
void this.reconcileStaleRun().catch(() => undefined);
238+
}, intervalMs);
239+
this.staleRunTimer.unref?.();
240+
}
241+
242+
private projectedRunTurnId(): string | undefined {
243+
return (
244+
this.options.controller.snapshot().activeTurnId ??
245+
this.options.runProjection.snapshot().latestRuntimeTurnId ??
246+
this.liveTurn?.turnId
247+
);
248+
}
249+
250+
private staleRunOwnershipBlocksReconciliation(
251+
sessionId: string,
252+
projectedTurnId: string,
253+
): boolean {
254+
const snapshot = this.options.controller.snapshot();
255+
const runProjection = this.options.runProjection.snapshot();
256+
return (
257+
this.options.isStopped() ||
258+
snapshot.session?.sessionId !== sessionId ||
259+
this.projectedRunTurnId() !== projectedTurnId ||
260+
Boolean(snapshot.retiringTurnId) ||
261+
runProjection.queueHandoffPending ||
262+
Boolean(runProjection.stoppingRuntimeTurnId) ||
263+
this.options.interactionFlow.continuesTurn(sessionId, projectedTurnId)
264+
);
265+
}
266+
267+
private async checkStaleRun(): Promise<boolean> {
268+
const snapshot = this.options.controller.snapshot();
269+
const sessionId = snapshot.session?.sessionId;
270+
const projectedTurnId = this.projectedRunTurnId();
271+
if (
272+
!sessionId ||
273+
!projectedTurnId ||
274+
this.staleRunOwnershipBlocksReconciliation(sessionId, projectedTurnId)
275+
) {
276+
this.staleRunCandidate = undefined;
277+
return false;
278+
}
279+
const generation = this.staleRunGeneration;
280+
const [activeRun, queued] = await Promise.all([
281+
this.options.runtime.getActiveRun(sessionId),
282+
this.options.runtime.listQueuedMessages(sessionId),
283+
]);
284+
if (generation !== this.staleRunGeneration) return false;
285+
if (this.staleRunOwnershipBlocksReconciliation(sessionId, projectedTurnId)) {
286+
this.staleRunCandidate = undefined;
287+
return false;
288+
}
289+
const runtimeBusy =
290+
activeRun.state === 'running' ||
291+
activeRun.state === 'decision-blocked' ||
292+
queued.some((item) => item.status === 'accepted' || item.status === 'running');
293+
if (runtimeBusy) {
294+
this.staleRunCandidate = undefined;
295+
return false;
296+
}
297+
const nowMs = Date.now();
298+
const candidate = this.staleRunCandidate;
299+
if (candidate?.sessionId !== sessionId || candidate.turnId !== projectedTurnId) {
300+
this.staleRunCandidate = { sessionId, turnId: projectedTurnId, firstSeenAtMs: nowMs };
301+
return false;
302+
}
303+
const inProcessTurnId = this.options.controller.snapshot().activeTurnId;
304+
const observation = {
305+
sessionId,
306+
projectedTurnId,
307+
runtimeState: activeRun.state,
308+
queuedCount: queued.filter((item) => item.status === 'queued').length,
309+
stalledForMs: Math.max(0, nowMs - candidate.firstSeenAtMs),
310+
...(activeRun.turnId ? { turnId: activeRun.turnId } : {}),
311+
...(inProcessTurnId ? { inProcessTurnId } : {}),
312+
};
313+
if (inProcessTurnId) {
314+
if (this.reportedStaleInProcessTurnId !== inProcessTurnId) {
315+
this.reportedStaleInProcessTurnId = inProcessTurnId;
316+
this.recordRunLifecycle({ kind: 'stale-in-process-run', ...observation });
317+
this.breadcrumb('cli.run.stale_in_process', {
318+
runtimeState: activeRun.state,
319+
stalledForMs: observation.stalledForMs,
320+
});
321+
}
322+
return false;
323+
}
324+
325+
this.staleRunCandidate = undefined;
326+
const liveTurn = this.liveTurn;
327+
if (liveTurn?.sessionId === sessionId && liveTurn.turnId === projectedTurnId) {
328+
liveTurn.controller.abort();
329+
void liveTurn.task.catch(() => undefined);
330+
this.liveTurn = undefined;
331+
this.options.controller.runtimeTurnSettlement.settleProjection(projectedTurnId, 'succeeded');
332+
}
333+
this.options.runProjection.reconcileRuntimeTurn(undefined);
334+
this.recordRunLifecycle({ kind: 'stale-run-reconciled', ...observation });
335+
this.breadcrumb('cli.run.stale_reconciled', {
336+
runtimeState: activeRun.state,
337+
stalledForMs: observation.stalledForMs,
338+
});
339+
this.options.onChanged();
340+
await this.refreshRuntimeSessionProjection(sessionId, { includeDurableHistory: true }).catch(
341+
() => false,
342+
);
343+
await this.options.activeRunFlow.refresh(true).catch(() => undefined);
344+
this.options.onChanged();
345+
return true;
346+
}
347+
348+
private recordRunLifecycle(observation: TuiRunLifecycleObservation): void {
349+
try {
350+
this.options.observability?.recordRunLifecycle?.(observation);
351+
} catch {
352+
// Diagnostics must not affect Runtime event handling.
353+
}
354+
}
355+
356+
private recordTerminalNotOwned(
357+
event: TuiSessionLifecycleEvent,
358+
sessionId: string,
359+
reason: 'in-process-turn' | 'projection-mismatch',
360+
context: { readonly inProcessTurnId?: string; readonly projectedTurnId?: string },
361+
): void {
362+
// Only a terminal for a Turn other than the one the TUI holds can leave it stale.
363+
const heldTurnId = context.inProcessTurnId ?? context.projectedTurnId;
364+
if (!event.turnId || event.turnId === heldTurnId) return;
365+
this.recordRunLifecycle({
366+
kind: 'terminal-not-owned',
367+
sessionId,
368+
turnId: event.turnId,
369+
reason,
370+
...(context.inProcessTurnId ? { inProcessTurnId: context.inProcessTurnId } : {}),
371+
...(context.projectedTurnId ? { projectedTurnId: context.projectedTurnId } : {}),
372+
});
373+
}
374+
190375
applyActiveSessionProjection(): void {
191376
this.clearLlmRetry();
192377
const view = selectActiveSessionView(this.options.stateStore.snapshot());
@@ -523,6 +708,14 @@ export class TuiRuntimeEventFlow {
523708
) {
524709
return;
525710
}
711+
this.recordRunLifecycle({
712+
kind: 'runtime-turn-adopted',
713+
sessionId,
714+
turnId,
715+
...(this.options.runProjection.snapshot().latestRuntimeTurnId
716+
? { projectedTurnId: this.options.runProjection.snapshot().latestRuntimeTurnId }
717+
: {}),
718+
});
526719
this.startLiveTurn(
527720
sessionId,
528721
turnId,
@@ -751,6 +944,14 @@ export class TuiRuntimeEventFlow {
751944

752945
private handleQueuedDrainStarted(event: TuiSessionLifecycleEvent): void {
753946
this.options.runProjection.markQueueTurnStarted(event.turnId);
947+
if (event.sessionId) {
948+
this.recordRunLifecycle({
949+
kind: 'queue-turn-started',
950+
sessionId: event.sessionId,
951+
...(event.turnId ? { turnId: event.turnId } : {}),
952+
queuedCount: event.queueItemIds.length,
953+
});
954+
}
754955
for (const itemId of event.queueItemIds) {
755956
const cached = this.options.runProjection.findQueueItem(itemId);
756957
if (!cached) continue;
@@ -767,7 +968,11 @@ export class TuiRuntimeEventFlow {
767968
liveTurnDurationMs?: number,
768969
interactionHandled = false,
769970
): Promise<boolean> {
770-
if (this.options.controller.snapshot().activeTurnId) return false;
971+
const inProcessTurnId = this.options.controller.snapshot().activeTurnId;
972+
if (inProcessTurnId) {
973+
this.recordTerminalNotOwned(event, sessionId, 'in-process-turn', { inProcessTurnId });
974+
return false;
975+
}
771976

772977
const activeRun = this.options.activeRunFlow.currentSnapshot();
773978
const activeRuntimeTurnId =
@@ -787,7 +992,10 @@ export class TuiRuntimeEventFlow {
787992
!event.turnId ||
788993
projectedTurnId === event.turnId ||
789994
projectedTurnId.startsWith('session:');
790-
if (!eventMatchesProjection) return false;
995+
if (!eventMatchesProjection) {
996+
this.recordTerminalNotOwned(event, sessionId, 'projection-mismatch', { projectedTurnId });
997+
return false;
998+
}
791999

7921000
// Late or background terminal events cannot settle the foreground turn.
7931001
const ownsTerminal = Boolean(

‎packages/tui/src/tui/features/transcript/panel.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import type { Component, Focusable } from '../../rendering/component.js';
2020
import { truncateToWidth, visibleWidth } from '../../rendering/text.js';
2121
import { renderTuiStructuredPreview } from '../../transcript/presentation/structured-preview.js';
2222
import { renderDeliveredAssets } from '../../transcript/delivered-assets.js';
23+
import { wrapLiteralUserText } from '../../transcript/presentation/literal-text.js';
2324
import {
2425
renderTuiActionHint,
2526
tuiChalk as chalk,
@@ -506,11 +507,15 @@ export class TuiTranscriptPanel implements TuiFeatureScreen, Component, Focusabl
506507
if (this.rawIds.has(cell.id)) {
507508
return this.renderTextSection('raw', presentation.rawText || 'No raw content', contentWidth);
508509
}
509-
if (cell.kind === 'assistant' || cell.kind === 'assistant-preamble' || cell.kind === 'user') {
510-
const content =
511-
cell.kind === 'user'
512-
? { text: cell.content, assets: [] }
513-
: projectAssistantContentForTerminal(cell.content);
510+
if (cell.kind === 'user') {
511+
// User prompts render literally here too, matching the main rail; see
512+
// `wrapLiteralUserText`.
513+
return wrapLiteralUserText(cell.content, contentWidth).map((line) =>
514+
this.detailLine(line, width),
515+
);
516+
}
517+
if (cell.kind === 'assistant' || cell.kind === 'assistant-preamble') {
518+
const content = projectAssistantContentForTerminal(cell.content);
514519
const body = new Markdown(content.text, 0, 0, markdownTheme, {
515520
color: (value) => chalk.hex(colors.text)(value),
516521
}).render(contentWidth);

0 commit comments

Comments
 (0)