Skip to content

Commit 4357fe4

Browse files
authored
fix(storage): retry contended message writes and preserve cancellation (#287)
* fix(storage): retry message write lock acquisition without replaying mutations * fix(storage): preserve aborted turn settlement when cancelling lock waits * fix(storage): preserve uncontended tool completion writes after cancellation
1 parent 432005c commit 4357fe4

11 files changed

Lines changed: 796 additions & 30 deletions

File tree

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import { setTimeout as delay } from 'node:timers/promises';
2+
import { sql } from 'drizzle-orm';
3+
import type { AppDb } from './client.js';
4+
5+
const WRITE_LOCK_BUDGET_MS = 10_000;
6+
const WRITE_LOCK_ATTEMPT_MS = 50;
7+
8+
/** Cancellation before the mutation callback starts; no write needs to be replayed. */
9+
export class WriteLockWaitAbortedError extends Error {
10+
override readonly name = 'WriteLockWaitAbortedError';
11+
12+
constructor(readonly signal: AbortSignal) {
13+
super('SQLite write lock wait was cancelled', { cause: signal.reason });
14+
}
15+
}
16+
17+
/**
18+
* Retry only transaction admission: a callback that has started is never replayed.
19+
* The signal cancels contention waits, not an immediately available write. This
20+
* lets post-cancellation tool completion and cleanup messages remain durable.
21+
*/
22+
export async function runWithWriteLock<T>(
23+
db: AppDb,
24+
mutation: (tx: AppDb) => T,
25+
options: { readonly signal?: AbortSignal; readonly timeoutMs?: number } = {},
26+
): Promise<T> {
27+
const budget = options.timeoutMs ?? WRITE_LOCK_BUDGET_MS;
28+
if (!Number.isFinite(budget) || budget <= 0)
29+
throw new RangeError('Invalid write lock budget');
30+
const deadline = performance.now() + budget;
31+
let attempt = 0;
32+
let hasContended = false;
33+
let lastBusy: unknown = new Error('SQLite write lock wait exceeded its deadline');
34+
for (;;) {
35+
if (hasContended) throwIfWaitAborted(options.signal);
36+
const remaining = deadline - performance.now();
37+
if (remaining <= 0) throw lastBusy;
38+
const previous = db.get<{ timeout: number }>(sql`PRAGMA busy_timeout`).timeout;
39+
let entered = false;
40+
try {
41+
const nativeWaitMs = options.signal?.aborted
42+
? 0
43+
: Math.ceil(Math.min(WRITE_LOCK_ATTEMPT_MS, remaining));
44+
db.run(sql.raw(`PRAGMA busy_timeout = ${nativeWaitMs}`));
45+
return db.transaction(
46+
(tx) => {
47+
entered = true;
48+
// Only lock acquisition gets a short timeout. Restore the connection's
49+
// policy before callbacks (including nested transactions) can use it.
50+
db.run(sql.raw(`PRAGMA busy_timeout = ${previous}`));
51+
return mutation(tx);
52+
},
53+
{ behavior: 'immediate' },
54+
);
55+
} catch (error) {
56+
if (entered || !isBusy(error)) throw error;
57+
lastBusy = error;
58+
} finally {
59+
// No await occurs while the shared connection has a temporary timeout.
60+
db.run(sql.raw(`PRAGMA busy_timeout = ${previous}`));
61+
}
62+
hasContended = true;
63+
throwIfWaitAborted(options.signal);
64+
const wait = Math.min(
65+
deadline - performance.now(),
66+
25 * 2 ** Math.min(attempt++, 3) + Math.random() * 25,
67+
);
68+
if (wait <= 0) throw lastBusy;
69+
try {
70+
await delay(wait, undefined, { signal: options.signal });
71+
} catch (error) {
72+
if (options.signal?.aborted && error instanceof Error && error.name === 'AbortError') {
73+
throw new WriteLockWaitAbortedError(options.signal);
74+
}
75+
throw error;
76+
}
77+
}
78+
}
79+
80+
function throwIfWaitAborted(signal: AbortSignal | undefined): void {
81+
if (signal?.aborted) throw new WriteLockWaitAbortedError(signal);
82+
}
83+
84+
function isBusy(error: unknown): boolean {
85+
return error instanceof Error && Reflect.get(error, 'code') === 'SQLITE_BUSY';
86+
}

‎packages/local-runtime-v2/src/service/session-system/agent-projection.ts‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ export interface SessionAgentEventContext {
3737
export interface SessionAgentProjectionInput {
3838
readonly context: SessionAgentEventContext;
3939
readonly event: RuntimeEvent;
40+
readonly signal?: AbortSignal;
4041
}
4142

4243
export interface SessionSystemAgentProjectionOptions {
@@ -76,19 +77,25 @@ export function createSessionSystemAgentProjection(options: SessionSystemAgentPr
7677
const message = displayMessage(input.event);
7778
if (message && !isTerminalAssistantDisplayError(message)) {
7879
const queryKey = await queryKeyForTurn(options, input.context);
79-
await options.messages.upsert({
80-
sessionId: input.context.sessionId,
81-
turnId: input.context.turnId,
82-
message: {
83-
...message,
84-
turn_id: input.context.turnId,
85-
...(queryKey ? { query_key: queryKey } : {}),
80+
await options.messages.upsert(
81+
{
82+
sessionId: input.context.sessionId,
83+
turnId: input.context.turnId,
84+
message: {
85+
...message,
86+
turn_id: input.context.turnId,
87+
...(queryKey ? { query_key: queryKey } : {}),
88+
},
89+
source: input.context.provenance?.source ?? 'agent',
90+
...(input.context.provenance?.sourceContext
91+
? { sourceContext: input.context.provenance.sourceContext }
92+
: {}),
8693
},
87-
source: input.context.provenance?.source ?? 'agent',
88-
...(input.context.provenance?.sourceContext
89-
? { sourceContext: input.context.provenance.sourceContext }
90-
: {}),
91-
});
94+
// Complete tool messages describe work already executed, including
95+
// abort cleanup. Persist these facts within the write-lock budget
96+
// even when the lease is cancelled; text-only waits may stop early.
97+
message.tool_calls?.length ? undefined : { signal: input.signal },
98+
);
9299
}
93100
await projectQueryCollapse(() => options.queryCollapse?.projectRuntimeEvent(input));
94101
},

‎packages/local-runtime-v2/src/service/session-system/messages/repo/contract.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ export interface MessageUpsertInput {
3737
readonly source?: string;
3838
readonly sourceContext?: Record<string, unknown>;
3939
}
40+
export interface MessageWriteOptions {
41+
/** Cancels lock contention waits; immediately available cleanup writes still commit. */
42+
readonly signal?: AbortSignal;
43+
}
4044
export interface UserMessageCommitInput extends MessageUpsertInput {
4145
readonly unconsumedFromTurnIds?: readonly string[];
4246
/** Trusted Queue startup lineage; the initial Host never read these rows. */
@@ -97,8 +101,11 @@ export interface MessageRepository {
97101
},
98102
): Promise<DisplayMessageRecord[]>;
99103
commitUserMessage(input: UserMessageCommitInput): Promise<UserMessageCommitResult>;
100-
upsert(input: MessageUpsertInput): Promise<NormalizedDisplayMessage>;
101-
upsertMany(inputs: readonly MessageUpsertInput[]): Promise<readonly NormalizedDisplayMessage[]>;
104+
upsert(input: MessageUpsertInput, options?: MessageWriteOptions): Promise<NormalizedDisplayMessage>;
105+
upsertMany(
106+
inputs: readonly MessageUpsertInput[],
107+
options?: MessageWriteOptions,
108+
): Promise<readonly NormalizedDisplayMessage[]>;
102109
replace(input: MessageReplaceInput): Promise<void>;
103110
replaceStream(input: MessageReplaceStreamInput): Promise<void>;
104111
rewindInclusive(input: MessageRewindInclusiveInput): Promise<MessageRewindInclusiveResult>;

‎packages/local-runtime-v2/src/service/session-system/messages/repo/drizzle.ts‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { and, asc, desc, eq, gt, gte, inArray, lt, lte, sql } from 'drizzle-orm';
22

33
import type { AppDb } from '../../../../infra/db/client.js';
4+
import { runWithWriteLock } from '../../../../infra/db/write-transaction.js';
45
import {
56
legacyMessages,
67
messageRowMigrations,
@@ -22,6 +23,7 @@ import type {
2223
MessageRewindInclusiveInput,
2324
MessageRewindInclusiveResult,
2425
MessageUpsertInput,
26+
MessageWriteOptions,
2527
DisplayMessageRecord,
2628
NormalizedDisplayMessage,
2729
UserMessageCommitInput,
@@ -205,14 +207,18 @@ class DrizzleMessageRepository implements MessageRepository {
205207
);
206208
}
207209

208-
async upsert(input: MessageUpsertInput): Promise<NormalizedDisplayMessage> {
209-
const [result] = await this.upsertMany([input]);
210+
async upsert(
211+
input: MessageUpsertInput,
212+
options?: MessageWriteOptions,
213+
): Promise<NormalizedDisplayMessage> {
214+
const [result] = await this.upsertMany([input], options);
210215
if (!result) throw new Error('Message upsert returned no row');
211216
return result;
212217
}
213218

214219
async upsertMany(
215220
inputs: readonly MessageUpsertInput[],
221+
options?: MessageWriteOptions,
216222
): Promise<readonly NormalizedDisplayMessage[]> {
217223
const sessionId = inputs[0]?.sessionId;
218224
if (!sessionId) return [];
@@ -229,11 +235,16 @@ class DrizzleMessageRepository implements MessageRepository {
229235
}),
230236
replaceProvenance: hasMessageProvenance(input),
231237
}));
232-
this.mutationTransaction(sessionId, (tx) => {
233-
normalized.forEach(({ message, replaceProvenance }) =>
234-
this.write(tx, sessionId, message, replaceProvenance),
235-
);
236-
});
238+
await runWithWriteLock(
239+
this.options.db,
240+
(tx) => {
241+
this.ensureReadyInTransaction(tx, sessionId);
242+
normalized.forEach(({ message, replaceProvenance }) =>
243+
this.write(tx, sessionId, message, replaceProvenance),
244+
);
245+
},
246+
options,
247+
);
237248
return normalized.map(({ message }) => message);
238249
}
239250

‎packages/local-runtime-v2/src/service/turn-system/agent-host/events/contracts.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ export type AgentEventResult =
3030
};
3131

3232
export interface AgentEventDelivery {
33-
handleRuntimeEvent(context: AgentEventContext, event: RuntimeEvent): Promise<AgentEventResult>;
33+
handleRuntimeEvent(
34+
context: AgentEventContext,
35+
event: RuntimeEvent,
36+
signal?: AbortSignal,
37+
): Promise<AgentEventResult>;
3438
handleHistoryCommitted(context: AgentEventContext, change: CommittedHistoryChange): Promise<void>;
3539
}

‎packages/local-runtime-v2/src/service/turn-system/agent-host/events/required-agent-event-delivery.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ type ObservationStage = 'runtime-event' | 'history-committed' | 'history-failure
2323
interface RuntimeProjectionInput {
2424
readonly context: AgentEventContext;
2525
readonly event: RuntimeEvent;
26+
/** Process-local control; excluded from semantic snapshots and replay identities. */
27+
readonly signal?: AbortSignal;
2628
}
2729

2830
interface HistoryProjectionInput {
@@ -156,7 +158,11 @@ export class RequiredAgentEventDelivery implements AgentEventDelivery, AgentHost
156158
this.historyReplays = new SemanticReplayRegistry(maximum);
157159
}
158160

159-
handleRuntimeEvent(context: AgentEventContext, event: RuntimeEvent): Promise<AgentEventResult> {
161+
handleRuntimeEvent(
162+
context: AgentEventContext,
163+
event: RuntimeEvent,
164+
signal?: AbortSignal,
165+
): Promise<AgentEventResult> {
160166
try {
161167
const snapshot = captureSemanticSnapshot({ context, event });
162168
validateRuntimeInput(snapshot.value.context, snapshot.value.event);
@@ -167,7 +173,7 @@ export class RequiredAgentEventDelivery implements AgentEventDelivery, AgentHost
167173
conflict: () => new AgentEventIdentityConflictError('runtime-event', identity),
168174
execute: () =>
169175
this.lane.run(snapshot.value.context.sessionId, () =>
170-
this.projectRuntime(snapshot.value.context, snapshot.value.event),
176+
this.projectRuntime(snapshot.value.context, snapshot.value.event, signal),
171177
),
172178
});
173179
} catch (error) {
@@ -223,6 +229,7 @@ export class RequiredAgentEventDelivery implements AgentEventDelivery, AgentHost
223229
private async projectRuntime(
224230
context: AgentEventContext,
225231
event: RuntimeEvent,
232+
signal?: AbortSignal,
226233
): Promise<AgentEventResult> {
227234
const runtimeSequence = this.validateSequence(context, event);
228235
const authoritative = await this.options.projectors.session.projectRuntimeEvent({
@@ -233,7 +240,11 @@ export class RequiredAgentEventDelivery implements AgentEventDelivery, AgentHost
233240
throw new AgentEventAcknowledgementError(terminalOutcome(event) ?? 'non-terminal', 'missing');
234241
}
235242
validateAcknowledgement(event, authoritative);
236-
await this.options.projectors.messages.projectRuntimeEvent({ context, event });
243+
await this.options.projectors.messages.projectRuntimeEvent({
244+
context,
245+
event,
246+
...(signal ? { signal } : {}),
247+
});
237248
await this.options.projectors.stream.projectRuntimeEvent({ context, event });
238249
await this.options.projectors.turnFacts.projectRuntimeEvent({ context, event });
239250
this.commitSequence(context, runtimeSequence);

‎packages/local-runtime-v2/src/service/turn-system/agent-host/events/turn-commit-pipeline.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import { TurnCommittedHistoryState } from '../history/turn-committed-history-sta
3434
import { CanonicalUserMessageIdentityLane } from '../history/canonical-user-message-identities.js';
3535
import { AgentEventAssociationError } from '../preparation/turn-preflight.js';
3636
import type { UserMessageId } from '../../../session-system/index.js';
37+
import { WriteLockWaitAbortedError } from '../../../../infra/db/write-transaction.js';
3738

3839
export class AgentTerminalConfirmationError extends Error {
3940
override readonly name = 'AgentTerminalConfirmationError';
@@ -147,11 +148,26 @@ export class TurnCommitPipeline {
147148
}
148149
await this.lane.enqueue(async () => {
149150
validateRuntimeAssociation(this.dependencies.context, event);
150-
const result = await this.dependencies.events.handleRuntimeEvent(
151-
this.dependencies.context,
152-
event,
153-
);
154-
new TerminalConfirmation().observe(event, result);
151+
const signal = this.dependencies.lease.signal;
152+
try {
153+
const result = await this.dependencies.events.handleRuntimeEvent(
154+
this.dependencies.context,
155+
event,
156+
signal,
157+
);
158+
new TerminalConfirmation().observe(event, result);
159+
} catch (error) {
160+
if (
161+
error instanceof WriteLockWaitAbortedError &&
162+
error.signal === signal &&
163+
signal.aborted
164+
) {
165+
// This lease cancelled a projection before its write began. Keep the
166+
// lane available for the runner's abort reconciliation and terminal.
167+
return;
168+
}
169+
throw error;
170+
}
155171
});
156172
};
157173

‎release/public-source.json‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)