From 893b09f4a859ec7d6f5ecf21bfa24586dba58ab0 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:17:49 -0400 Subject: [PATCH 01/40] test: pin the transaction invariants #1005 left broken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight tests, all currently failing, for three defects that landed with #1005. They change no production code: each one states the invariant the fix has to restore, so the diff that repairs them is the specification being met rather than a claim about it. `releaseSavepoint` was never hardened the way `rollbackSavepoint` was in that PR. A RELEASE that throws leaves the savepoint on the stack and the transaction open with nothing that will ever commit or abort it, so every later write on the connection joins it, reports success, and vanishes on close — verbatim the failure mode #1005 documents for the other door. The driver tests sit beside their rollback counterparts so the asymmetry is visible in place. `endCrank` gets the companion case: it now settles its waiters in a `finally`, which is right, but it also leaves the savepoint listed, so the next crank numbers its savepoint `t1` against a database that still has `t0`. `#processCrankResult` does fallible work after the crank's transactional boundary has already been crossed. On the success path `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external caller, and only then can `#terminateVat` throw and have the new catch roll the crank back — so the caller keeps an answer computed from state the store discarded, and a restart delivers the message again. On the abort path the rollback ends the transaction, so `#terminateVat` and `collectGarbage` autocommit piecemeal and the second rollback the flag correctly suppresses would have had nothing left to undo either way. The invariant is stated as "the rollback is the last thing the crank asks of the store", which leaves the choice of remedy open. The wasm driver tracks `_inTx` itself rather than reading it from SQLite, so a failed abort inside the new catch is the one case that can leave it disagreeing with the database. Left true, `beginIfNeeded` is a no-op from then on and the next `createSavepoint` runs in autocommit mode, where the matching RELEASE commits (Agoric/agoric-sdk#8423, already cited two lines above the code) and no rollback can undo the delivery. The second test runs that next `createSavepoint` and asserts the BEGIN, so the corruption path is observable instead of argued. Co-Authored-By: Claude Opus 5 --- .../kernel-store/src/sqlite/nodejs.test.ts | 23 ++++ packages/kernel-store/src/sqlite/wasm.test.ts | 69 ++++++++++++ packages/ocap-kernel/src/KernelQueue.test.ts | 106 ++++++++++++++++++ .../src/store/methods/crank.test.ts | 17 +++ 4 files changed, 215 insertions(+) diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index a62392fe19..3bf0db5f68 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -360,6 +360,29 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._spStack).toStrictEqual([]); }); + // The same hazard `rollbackSavepoint` guards against, by the other door: a + // RELEASE that throws leaves the savepoint on the stack and the transaction + // open with nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. + it('releaseSavepoint discards the transaction when the release fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['point1']; + mockStatement.run.mockClear(); + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + // The abort is the only prepared statement this path runs. + expect(mockStatement.run).toHaveBeenCalledOnce(); + mockDb.inTransaction = false; + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 2cbc96d658..e0e6be1c45 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -518,6 +518,75 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); + // The same hazard `rollbackSavepoint` guards against, by the other door: a + // RELEASE that throws leaves the savepoint on the stack and the transaction + // open with nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. + it('releaseSavepoint discards the transaction when the release fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + expect(mockDb._inTx).toBe(false); + }); + + // `_inTx` is tracked here rather than read from SQLite, so a failed abort is + // the one case that can leave it disagreeing with the database. Left true, + // `beginIfNeeded` becomes a no-op forever after. + it('stops believing it is in a transaction when the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.rollbackSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + }); + + // The consequence of the above, and the reason it is worth asserting: a + // savepoint created outside a transaction autocommits when released + // (Agoric/agoric-sdk#8423), so no later rollback can undo the delivery — an + // aborted crank would silently keep its writes. + it('begins a transaction for the next savepoint after a failed abort', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + expect(() => db.rollbackSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + mockDb.exec.mockClear(); + mockStatement.step.mockClear(); + db.createSavepoint('next'); + + // BEGIN is the only prepared statement `createSavepoint` runs; the + // SAVEPOINT itself goes through `exec`. + expect(mockStatement.step).toHaveBeenCalledOnce(); + expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 4ff8a8e78a..d0e712a9f7 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -195,6 +195,112 @@ describe('KernelQueue', () => { expect(kernelStore.collectGarbage).toHaveBeenCalled(); expect(kernelStore.endCrank).toHaveBeenCalled(); }); + + // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external + // caller, reading the resolution out of the store on the way. Rolling the + // crank back afterwards un-resolves that promise in the store and restores + // the run queue item, so a restart delivers the message a second time and + // notifies every other subscriber again — while the original caller has + // already been told the first answer. + it('does not roll back a crank whose result the caller already received', async () => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp1' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + // A caller is awaiting this message's result. + const resolve = vi.fn(); + const reject = vi.fn(); + kernelQueue.subscriptions.set('kp1', { resolve, reject }); + + // The crank succeeds, so the flush hands that caller its answer... + ( + kernelStore.flushCrankBuffer as unknown as MockInstance + ).mockReturnValueOnce([ + { type: 'notify', endpointId: 'v1', kpid: 'kp1' }, + ]); + (kernelStore.getKernelPromise as unknown as MockInstance).mockReturnValue( + { + state: 'fulfilled', + value: { body: '"answer"', slots: [] }, + }, + ); + + // ...and only then does the kernel die, in work that runs after the flush. + const terminationError = new Error('vat worker already gone'); + (terminateVat as unknown as MockInstance).mockRejectedValueOnce( + terminationError, + ); + const deliver = vi.fn().mockResolvedValue({ + terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, + }); + + await expect(kernelQueue.run(deliver)).rejects.toBe(terminationError); + + expect(resolve).toHaveBeenCalledWith({ body: '"answer"', slots: [] }); + expect(kernelStore.rollbackCrank).not.toHaveBeenCalled(); + }); + + // `rollbackCrank('start')` rolls back the crank's outermost savepoint, which + // ends the transaction — so anything the crank does to the store afterwards + // autocommits piecemeal and no rollback can reach it. Whatever the ordering, + // the rollback has to be the last thing the crank asks of the store. + it.each([ + { label: 'an abort', crankResult: { abort: true } }, + { + label: 'an abort that also terminates', + crankResult: { + abort: true, + terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, + }, + }, + ])( + 'does no store work after rolling back $label', + async ({ crankResult }) => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp99' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + const storeCalls: string[] = []; + ( + kernelStore.rollbackCrank as unknown as MockInstance + ).mockImplementation(() => { + storeCalls.push('rollbackCrank'); + }); + (terminateVat as unknown as MockInstance).mockImplementation( + async () => { + storeCalls.push('terminateVat'); + }, + ); + ( + kernelStore.collectGarbage as unknown as MockInstance + ).mockImplementation(() => { + storeCalls.push('collectGarbage'); + throw new Error(STOP_RUN_LOOP); + }); + + const deliver = vi.fn().mockResolvedValue(crankResult); + await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); + + expect(storeCalls).toContain('rollbackCrank'); + expect(storeCalls.at(-1)).toBe('rollbackCrank'); + }, + ); }); describe('getRunLoopStatus', () => { diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index db3de86450..8fd6cd1fec 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -206,6 +206,23 @@ describe('crank methods', () => { expect(context.resolveCrank).toBeUndefined(); expect(await waiter).toBeUndefined(); }); + + // What `rollbackCrank` already does in its own `finally`. Settling the crank + // regardless means callers proceed, so a savepoint left listed here has the + // next crank number its savepoint `t1` while the database still has `t0`: + // from then on `releaseAllSavepoints` releases the wrong one and every + // rollback aims past the crank it meant to undo. + it('forgets its savepoints even if releasing them fails', () => { + crankMethods.startCrank(); + context.savepoints = ['test']; + vi.mocked(kdb.releaseSavepoint).mockImplementationOnce(() => { + throw new Error('database is gone'); + }); + + expect(() => crankMethods.endCrank()).toThrow('database is gone'); + + expect(context.savepoints).toStrictEqual([]); + }); }); describe('releaseAllSavepoints', () => { From 0493967fa86d36a1b0e576d3d7145088f8189ee6 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:23:28 +0200 Subject: [PATCH 02/40] fix: keep a crank's store work inside one transaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three transaction-integrity defects, all in the same family: a store call fails, and the layer above goes on as though its bookkeeping still matched the database. - `releaseSavepoint` (both SQLite drivers) discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already does when `ROLLBACK TO` fails. Left as it was, the savepoint stayed on the stack and the transaction open with nothing to ever commit or abort it, so every later write on the connection joined it, reported success, and vanished on `close()`. - `releaseAllSavepoints` forgets its savepoints even if the release throws, as `rollbackCrank` already does. A savepoint left listed had the next crank number its savepoint `t1` while the database still had `t0`, from which point every release and rollback aimed one crank past the one it meant to end. - The wasm driver stops believing it is in a transaction when an abort fails. `_inTx` is tracked in the driver rather than read from SQLite, and an abort usually fails because SQLite already rolled back on its own. Left true, `beginIfNeeded` was a no-op from then on and the next `createSavepoint` ran in autocommit mode, where its `RELEASE` commits (Agoric/agoric-sdk#8423) and no later rollback could undo the delivery. And the crank boundary itself, in two parts: - A crank now takes two savepoints. Rolling back to the outermost one discards the enclosing transaction, so the work an aborted crank still owes — terminating the vat whose delivery failed, collecting garbage — was autocommitting statement by statement, beyond the reach of any later rollback. That work has to follow the rollback, since the worker is gone and the store must not go on believing the vat is alive, so it is the rollback that spares the transaction. Releasing the outer savepoint in `endCrank` is now a crank's one commit point. - `#flushCrankBuffer` runs last, after everything that can still fail. It settles the promise `enqueueMessage` handed an external caller, reading the result out of the store; rolling the crank back after that left the caller holding an answer computed from state the store had discarded, and a restart would deliver the message again. Tests for the first three defects are Ryan's, from #1011. The two crank tests there specify the remedy as "the rollback is the last thing the crank asks of the store", which reordering the fallible work before it would satisfy — but that rollback would then undo the vat termination. They are restated here as the invariant the fix does hold. Co-authored-by: Claude Opus 5 (1M context) --- .../kernel-store/src/sqlite/nodejs.test.ts | 19 ++++ packages/kernel-store/src/sqlite/nodejs.ts | 17 +++- packages/kernel-store/src/sqlite/wasm.test.ts | 19 +++- packages/kernel-store/src/sqlite/wasm.ts | 29 +++++- packages/ocap-kernel/src/KernelQueue.test.ts | 99 ++++++++++++------- packages/ocap-kernel/src/KernelQueue.ts | 49 ++++++--- .../ocap-kernel/src/store/methods/crank.ts | 13 ++- 7 files changed, 189 insertions(+), 56 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index 3bf0db5f68..e38655d9a9 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -383,6 +383,25 @@ describe('makeSQLKernelDatabase', () => { mockDb.inTransaction = false; }); + it('releaseSavepoint reports the release failure even if the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.run.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + mockDb.inTransaction = false; + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index ec863edc7c..d0a6442442 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -321,7 +321,22 @@ export async function makeSQLKernelDatabase({ throw new Error(`No such savepoint: ${name}`); } const query = SQL_QUERIES.RELEASE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // The hazard `rollbackSavepoint` guards against, by the other door: left as + // it was, the savepoint stays on the stack and the transaction open with + // nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. There is no + // committing this transaction now, so discard it. + db._spStack.length = 0; + try { + rollbackIfNeeded(); + } catch { + // The release failure below is the one worth reporting. + } + throw error; + } db._spStack.splice(idx); if (db._spStack.length === 0) { commitIfNeeded(); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index e0e6be1c45..25403caa47 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -488,7 +488,6 @@ describe('makeSQLKernelDatabase', () => { ); expect(mockDb._spStack).toStrictEqual([]); - mockDb._inTx = false; }); it('releaseSavepoint validates savepoint exists', async () => { @@ -538,6 +537,24 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); + it('releaseSavepoint reports the release failure even if the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + }); + // `_inTx` is tracked here rather than read from SQLite, so a failed abort is // the one case that can leave it disagreeing with the database. Left true, // `beginIfNeeded` becomes a no-op forever after. diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index c0c32b8a72..beda148117 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -210,10 +210,18 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - sqlAbortTransaction.step(); - sqlAbortTransaction.reset(); + // Out of the transaction as far as this driver is concerned before the + // abort is even attempted. Unlike the nodejs driver, which reads + // `inTransaction` from SQLite, `_inTx` is tracked here — and an abort + // typically fails because SQLite already rolled back on its own as part of + // whatever went wrong. Left true, `beginIfNeeded` is a no-op from then on + // and the next `createSavepoint` runs in autocommit mode, where its + // `RELEASE` commits (see `createSavepoint`) and no later rollback can undo + // the delivery. db._inTx = false; db._spStack.length = 0; + sqlAbortTransaction.step(); + sqlAbortTransaction.reset(); } } @@ -403,7 +411,22 @@ export async function makeSQLKernelDatabase({ throw new Error(`No such savepoint: ${name}`); } const query = SQL_QUERIES.RELEASE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // The hazard `rollbackSavepoint` guards against, by the other door: left as + // it was, the savepoint stays on the stack and the transaction open with + // nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. There is no + // committing this transaction now, so discard it. + db._spStack.length = 0; + try { + rollbackIfNeeded(); + } catch { + // The release failure below is the one worth reporting. + } + throw error; + } db._spStack.splice(idx); if (db._spStack.length === 0) { commitIfNeeded(); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index d0e712a9f7..85a6d22d3c 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -92,6 +92,23 @@ describe('KernelQueue', () => { }; }; + /** + * Stop the run loop by failing the *next* crank's start, so that the crank + * under test runs to completion. Throwing from one of a crank's own store calls + * cuts it short, which hides everything the crank does after that call. + */ + const stopAfterOneCrank = (): void => { + let cranks = 0; + (kernelStore.startCrank as unknown as MockInstance).mockImplementation( + () => { + cranks += 1; + if (cranks > 1) { + throw new Error(STOP_RUN_LOOP); + } + }, + ); + }; + /** * Run a single crank whose delivery blows up, killing the run loop. * @@ -128,7 +145,8 @@ describe('KernelQueue', () => { const deliver = vi.fn().mockRejectedValue(deliverError); await expect(kernelQueue.run(deliver)).rejects.toBe(deliverError); expect(kernelStore.startCrank).toHaveBeenCalled(); - expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('start'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('crank'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('delivery'); expect(processGCActionSetSpy).toHaveBeenCalled(); expect(kernelStore.nextReapAction).toHaveBeenCalled(); expect(kernelStore.nextTerminatedVatCleanup).toHaveBeenCalled(); @@ -156,9 +174,9 @@ describe('KernelQueue', () => { }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(kernelStore.startCrank).toHaveBeenCalled(); - expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('start'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('delivery'); expect(deliver).toHaveBeenCalledWith(mockItem); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(kernelStore.collectGarbage).toHaveBeenCalled(); expect(kernelStore.endCrank).toHaveBeenCalled(); }); @@ -197,12 +215,13 @@ describe('KernelQueue', () => { }); // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external - // caller, reading the resolution out of the store on the way. Rolling the - // crank back afterwards un-resolves that promise in the store and restores - // the run queue item, so a restart delivers the message a second time and - // notifies every other subscriber again — while the original caller has - // already been told the first answer. - it('does not roll back a crank whose result the caller already received', async () => { + // caller, reading the resolution out of the store on the way. Were the crank + // rolled back after that, the store would un-resolve the promise and restore + // the run queue item, so a restart would deliver the message a second time + // and notify every other subscriber again — while the original caller had + // already been told the first answer. So the flush comes last, after + // everything that could still fail. + it('answers no caller from a crank it then rolls back', async () => { const mockItem: RunQueueItem = { type: 'send', target: 'ko123', @@ -220,7 +239,7 @@ describe('KernelQueue', () => { const reject = vi.fn(); kernelQueue.subscriptions.set('kp1', { resolve, reject }); - // The crank succeeds, so the flush hands that caller its answer... + // The delivery succeeds and its result is there for the flush to hand over... ( kernelStore.flushCrankBuffer as unknown as MockInstance ).mockReturnValueOnce([ @@ -233,7 +252,7 @@ describe('KernelQueue', () => { }, ); - // ...and only then does the kernel die, in work that runs after the flush. + // ...but the crank still has fallible work left, and it dies there. const terminationError = new Error('vat worker already gone'); (terminateVat as unknown as MockInstance).mockRejectedValueOnce( terminationError, @@ -244,26 +263,39 @@ describe('KernelQueue', () => { await expect(kernelQueue.run(deliver)).rejects.toBe(terminationError); - expect(resolve).toHaveBeenCalledWith({ body: '"answer"', slots: [] }); - expect(kernelStore.rollbackCrank).not.toHaveBeenCalled(); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); + expect(resolve).not.toHaveBeenCalled(); + // Told the result will never come, rather than left waiting on a crank the + // store no longer has any record of. + expect(reject).toHaveBeenCalledWith( + expect.objectContaining({ cause: terminationError }), + ); }); - // `rollbackCrank('start')` rolls back the crank's outermost savepoint, which - // ends the transaction — so anything the crank does to the store afterwards - // autocommits piecemeal and no rollback can reach it. Whatever the ordering, - // the rollback has to be the last thing the crank asks of the store. + // Rolling back to the crank's *outermost* savepoint discards the enclosing + // transaction (see `rollbackSavepoint`), which would leave the work an aborted + // crank still owes — terminating the vat whose delivery failed, collecting + // garbage — autocommitting statement by statement, beyond the reach of any + // later rollback. That work has to follow the rollback, since the worker is + // already gone and the store must not go on believing the vat is alive, so it + // is the rollback that has to spare the transaction. it.each([ - { label: 'an abort', crankResult: { abort: true } }, + { + label: 'an abort', + crankResult: { abort: true }, + storeOrder: ['rollbackCrank', 'collectGarbage'], + }, { label: 'an abort that also terminates', crankResult: { abort: true, terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, }, + storeOrder: ['rollbackCrank', 'terminateVat', 'collectGarbage'], }, ])( - 'does no store work after rolling back $label', - async ({ crankResult }) => { + 'keeps the crank transactional after rolling back $label', + async ({ crankResult, storeOrder }) => { const mockItem: RunQueueItem = { type: 'send', target: 'ko123', @@ -297,8 +329,13 @@ describe('KernelQueue', () => { const deliver = vi.fn().mockResolvedValue(crankResult); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(storeCalls).toContain('rollbackCrank'); - expect(storeCalls.at(-1)).toBe('rollbackCrank'); + expect(storeCalls).toStrictEqual(storeOrder); + expect( + ( + kernelStore.createCrankSavepoint as unknown as MockInstance + ).mock.calls.flat(), + ).toStrictEqual(['crank', 'delivery']); + expect(kernelStore.rollbackCrank).not.toHaveBeenCalledWith('crank'); }, ); }); @@ -393,7 +430,7 @@ describe('KernelQueue', () => { await killRunLoop(new Error('crank exploded')); // Without this, endCrank's savepoint release commits the half-finished // crank and the dequeued item is lost. - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); }); it('does not roll back when the savepoint was never created', async () => { @@ -949,7 +986,7 @@ describe('KernelQueue', () => { throw new Error(STOP_RUN_LOOP); }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(rejectSpy).toHaveBeenCalledWith(terminateInfo); expect(kernelQueue.subscriptions.has('kp99')).toBe(false); }); @@ -985,7 +1022,7 @@ describe('KernelQueue', () => { throw new Error(STOP_RUN_LOOP); }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(rejectedAfterAbort).toBe(false); expect(resolveSpy).not.toHaveBeenCalled(); expect(subscribedAfterAbort).toBe(true); @@ -1055,11 +1092,7 @@ describe('KernelQueue', () => { mockItem, ); const deliver = vi.fn().mockResolvedValue(undefined); - ( - kernelStore.collectGarbage as unknown as MockInstance - ).mockImplementation(() => { - throw new Error(STOP_RUN_LOOP); - }); + stopAfterOneCrank(); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(rejectSpy).toHaveBeenCalledWith(rejectedValue); expect(resolveSpy).not.toHaveBeenCalled(); @@ -1094,11 +1127,7 @@ describe('KernelQueue', () => { mockItem, ); const deliver = vi.fn().mockResolvedValue(undefined); - ( - kernelStore.collectGarbage as unknown as MockInstance - ).mockImplementation(() => { - throw new Error(STOP_RUN_LOOP); - }); + stopAfterOneCrank(); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(resolveSpy).toHaveBeenCalledWith(fulfilledValue); expect(rejectSpy).not.toHaveBeenCalled(); diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index b4280ec029..9086c0e961 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -130,7 +130,15 @@ export class KernelQueue { this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; try { - this.#kernelStore.createCrankSavepoint('start'); + // Two savepoints, because the crank's transaction has to outlive the + // delivery's rollback. Rolling back to the outermost savepoint discards + // the enclosing transaction (see `rollbackSavepoint`), and the work an + // aborted crank still owes — terminating the vat whose delivery failed, + // collecting garbage — would then autocommit statement by statement, + // beyond the reach of any later rollback. Only `delivery` is ever rolled + // back; releasing `crank` in `endCrank` is this crank's one commit point. + this.#kernelStore.createCrankSavepoint('crank'); + this.#kernelStore.createCrankSavepoint('delivery'); // The savepoint exists from here on, so a throw can be undone. Without // this, `endCrank`'s savepoint release commits the half-finished crank: @@ -158,7 +166,7 @@ export class KernelQueue { // savepoint" over the real error. if (!this.#crankRollbackAttempted) { try { - this.#kernelStore.rollbackCrank('start'); + this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { // The original failure stays the `cause`, since that is the root // cause an operator needs; the rollback failure is named here. @@ -304,7 +312,7 @@ export class KernelQueue { // For active vats, this allows the message to be retried in a future crank. // For terminated vats, the message will just go splat. try { - this.#kernelStore.rollbackCrank('start'); + this.#kernelStore.rollbackCrank('delivery'); } finally { // Set even when the rollback threw. `rollbackCrank` forgets the // savepoint in its own `finally`, so "attempted" and "the savepoint is @@ -333,17 +341,27 @@ export class KernelQueue { // TODO: Currently all errors terminate the vat, but instead we could // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. - } else { - // Upon on successful crank completion, enqueue buffered vat outputs for delivery. - this.#flushCrankBuffer(); } // Vat termination during delivery is triggered by an illegal syscall - // or by syscall.exit(). + // or by syscall.exit(). Its store writes have to survive the rollback above: + // the worker is already gone, so a store that still believed the vat was + // alive would relaunch it after a restart and redeliver what killed it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); + if (!crankResult?.abort) { + // The crank survived, so hand its buffered outputs on — last, once nothing + // fallible remains. The flush settles the promise `enqueueMessage` gave an + // external caller, reading the result out of the store; were the crank + // rolled back after that, the caller would keep an answer computed from + // state the store discarded, and a restart would deliver the message again. + this.#flushCrankBuffer(); + } + // After the flush, because the audit reads the run queue as ground truth + // while a buffered item's references were already counted when it was + // enqueued: audited mid-flush, every buffered item reads as a leak. this.#kernelStore.assertRefCountsIfAuditing(); } @@ -371,21 +389,24 @@ export class KernelQueue { */ #flushCrankBuffer(): void { const items = this.#kernelStore.flushCrankBuffer(); + const resolved: KRef[] = []; for (const item of items) { this.#enqueueRun(item); if (item.type === 'notify') { - // Invoke kernel subscription callback if any, reading resolution - // data from the (now committed) promise state - this.#invokeKernelSubscription(item.kpid); + resolved.push(item.kpid); } } + // Also promises resolved during this crank that don't have kernel-level + // subscribers (e.g., promises from enqueueMessage). + resolved.push(...this.#resolvedWithKernelSubscription); + this.#resolvedWithKernelSubscription = []; - // Invoke kernel subscriptions for promises resolved during this crank - // that don't have kernel-level subscribers (e.g., promises from enqueueMessage) - for (const kpid of this.#resolvedWithKernelSubscription) { + // Callbacks only once every store write is done. Each hands an external + // caller a result read out of the store, and a write that threw in between + // would have the crank rolled back underneath answers already given. + for (const kpid of resolved) { this.#invokeKernelSubscription(kpid); } - this.#resolvedWithKernelSubscription = []; } /** diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 87d2bc65b8..24957bce4b 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -77,8 +77,17 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { */ function releaseAllSavepoints(): void { if (ctx.savepoints.length > 0) { - kdb.releaseSavepoint('t0'); - ctx.savepoints.length = 0; + try { + kdb.releaseSavepoint('t0'); + } finally { + // Forget the savepoints even if the release failed, as `rollbackCrank` + // does. A failed release discards the whole transaction (see + // `releaseSavepoint`), so the database has no savepoints left either; + // leaving them listed here would have the next crank number its savepoint + // `t1`, and from then on every release and rollback would aim one crank + // past the one it meant to end. + ctx.savepoints.length = 0; + } } } From 5e6cf5d31838366adbe0bc55a406a1e30f45738f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:23:44 +0200 Subject: [PATCH 03/40] test(kernel-test): reap until the vat's GC is visible, not three times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `should trigger GC syscalls through bringOutYourDead` scheduled one reap and then ran three cranks. `scheduleReap` dedupes, so that bought one `bringOutYourDead`, not three — and an import is only reported as dropped once the engine has collected the vat's presence and run its finalizer, which the forced GC pass inside `bringOutYourDead` cannot guarantee on the first attempt. When it hadn't, no further reap was ever scheduled and the refcount stayed where it was: `expected 2 to be 1`, as on main in 31081630878. Each attempt now schedules its own reap and stops as soon as the kernel's bookkeeping catches up, so the common case is one crank rather than three. Co-authored-by: Claude Opus 5 (1M context) --- .../src/garbage-collection.test.ts | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 67055414f1..45f21f12d2 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -117,6 +117,28 @@ describe('Garbage Collection', () => { expect(parseReplyBody(useResult.body)).toBe(objectId); }); + /** + * Reap the importer vat until the kernel's bookkeeping catches up with the + * vat's own garbage collection, or the attempts run out. + * + * `bringOutYourDead` can only report an import as dropped once the engine has + * collected the vat's presence and run its finalizer, which forcing a GC pass + * does not guarantee on the first attempt. Reaping once and then cranking + * buys one attempt rather than several, because `scheduleReap` dedupes — so + * each attempt schedules its own reap, and a message to the vat wakes the run + * loop to consume it. + * + * @param settled - Whether the state under test has arrived. + */ + async function reapImporterUntil(settled: () => boolean): Promise { + const isImporter = (vatId: VatId): boolean => vatId === importerVatId; + for (let attempt = 0; attempt < 5 && !settled(); attempt += 1) { + kernel.reapVats(isImporter); + await kernel.queueMessage(importerKRef, 'noop', []); + await waitUntilQuiescent(500); + } + } + it('should trigger GC syscalls through bringOutYourDead', async () => { // Create an object in the exporter vat with a known ID const objectId = 'test-object'; @@ -161,14 +183,10 @@ describe('Garbage Collection', () => { await kernel.queueMessage(importerKRef, 'makeWeak', [objectId]); await waitUntilQuiescent(); - // Schedule reap to trigger bringOutYourDead on next crank - kernel.reapVats((vatId) => vatId === importerVatId); - - // Run 3 cranks to allow bringOutYourDead to be processed - for (let i = 0; i < 3; i++) { - await kernel.queueMessage(importerKRef, 'noop', []); - await waitUntilQuiescent(500); - } + // Reap until the importer reports the drop + await reapImporterUntil( + () => kernelStore.getObjectRefCount(createObjectRef).reachable === 1, + ); // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); @@ -180,13 +198,10 @@ describe('Garbage Collection', () => { await kernel.queueMessage(importerKRef, 'forgetImport', []); await waitUntilQuiescent(); - // Schedule another reap - kernel.reapVats((vatId) => vatId === importerVatId); - - for (let i = 0; i < 3; i++) { - await kernel.queueMessage(importerKRef, 'noop', []); - await waitUntilQuiescent(500); - } + // Reap until the importer reports the retirement + await reapImporterUntil( + () => kernelStore.getObjectRefCount(createObjectRef).recognizable === 1, + ); // Check reference counts after retireImports const afterForgetRefCounts = kernelStore.getObjectRefCount(createObjectRef); From 272f920c05f29d6e8dd9b09e6976d8a72b2600b2 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:33:20 +0200 Subject: [PATCH 04/40] test: fix rollback crank test --- .../kernel-test/src/crank-rollback.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index cd2a708aa0..dce56b3227 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -138,6 +138,46 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('second')).toBe('yes'); }); + // An aborted crank rolls its delivery back and then still has work to do — + // terminating the vat whose delivery failed, collecting garbage — whose writes + // have to survive that rollback, since the vat's worker is already gone. + it('keeps the writes a crank makes after rolling its delivery back', async () => { + const { kernelStore, kdb } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + kdb.kernelKVStore.set('delivered', 'yes'); + + kernelStore.rollbackCrank('delivery'); + kdb.kernelKVStore.set('terminated', 'yes'); + kernelStore.endCrank(); + + expect(kdb.kernelKVStore.get('delivered')).toBeUndefined(); + expect(kdb.kernelKVStore.get('terminated')).toBe('yes'); + }); + + // And they survive it *as part of the crank's transaction*, which is why the + // delivery gets a savepoint of its own rather than rolling back the crank's: + // rolling back the outermost savepoint discards the transaction, after which + // those writes would autocommit one statement at a time. Nothing in the run + // loop rolls the crank's own savepoint back — it is the only way from here to + // observe that the writes are still undoable at all. + it('holds those writes in the transaction rather than autocommitting them', async () => { + const { kernelStore, kdb } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + kernelStore.rollbackCrank('delivery'); + kdb.kernelKVStore.set('terminated', 'yes'); + + kernelStore.rollbackCrank('crank'); + kernelStore.endCrank(); + + expect(kdb.kernelKVStore.get('terminated')).toBeUndefined(); + }); + // `createCrankSavepoint` records the name only once the database has the // savepoint. Asking to roll back one that was never created must therefore say // so, rather than releasing someone else's savepoint. From eeb5652ed59a60f699fab86612b74efb6cfb4b23 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:37:12 +0200 Subject: [PATCH 05/40] fix(ocap-kernel): forget every savepoint when a crank rollback fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed `ROLLBACK TO` discards the whole transaction, taking every savepoint with it — not just the one rolled back to. `rollbackCrank` truncated `ctx.savepoints` to the rolled-back ordinal regardless, which was correct while a crank took one savepoint at ordinal 0 and cleared the list, but leaves `['crank']` listed now that the delivery sits at ordinal 1. `endCrank` then releases a `t0` the database no longer has, and throws "No such savepoint: t0" from the run loop's `finally` — replacing the failure that actually killed the kernel, with no `cause`. That is the masking this branch's own error-preservation exists to prevent. Clear the list on the throwing path, truncate to the ordinal only on success. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/store/methods/crank.test.ts | 33 ++++++++++++++++--- .../ocap-kernel/src/store/methods/crank.ts | 28 +++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 8fd6cd1fec..389afd7a03 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -153,6 +153,29 @@ describe('crank methods', () => { expect(mockCrankBuffer).toHaveLength(0); }); + + // A failed rollback discards the whole transaction, so the enclosing + // savepoints are gone from the database too — not just the one rolled back + // to. Truncating to the ordinal would leave `endCrank` releasing a `t0` the + // database no longer has, and that "No such savepoint" would be thrown from + // the run loop's `finally`, over whatever really killed the kernel. + it('forgets every savepoint when the rollback fails', () => { + context.inCrank = true; + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + 'disk I/O error', + ); + + expect(context.savepoints).toStrictEqual([]); + // The release `endCrank` would otherwise attempt, and throw over. + crankMethods.endCrank(); + expect(kdb.releaseSavepoint).not.toHaveBeenCalled(); + }); }); describe('endCrank', () => { @@ -207,11 +230,11 @@ describe('crank methods', () => { expect(await waiter).toBeUndefined(); }); - // What `rollbackCrank` already does in its own `finally`. Settling the crank - // regardless means callers proceed, so a savepoint left listed here has the - // next crank number its savepoint `t1` while the database still has `t0`: - // from then on `releaseAllSavepoints` releases the wrong one and every - // rollback aims past the crank it meant to undo. + // What `rollbackCrank` already does when its own rollback fails. Settling + // the crank regardless means callers proceed, so a savepoint left listed + // here has the next crank number its savepoint `t1` while the database still + // has `t0`: from then on `releaseAllSavepoints` releases the wrong one and + // every rollback aims past the crank it meant to undo. it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); context.savepoints = ['test']; diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 24957bce4b..b26219cbb6 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -51,13 +51,19 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { if (ctx.savepoints[ordinal] === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); - } finally { - // Forget the savepoint even if the rollback failed. Leaving it listed - // would have `endCrank`'s release commit the crank we just abandoned — - // the half-finished state this rollback exists to discard. A failed - // rollback discards the whole transaction instead (see - // `rollbackSavepoint`), which for a crank is the same boundary. + // Forget the savepoint. Leaving it listed would have `endCrank`'s + // release commit the crank we just abandoned — the half-finished state + // this rollback exists to discard. ctx.savepoints.length = ordinal; + } catch (error) { + // A failed rollback discards the whole transaction (see + // `rollbackSavepoint`), taking every savepoint with it, not just this + // one. Truncating to `ordinal` would leave the enclosing savepoints + // listed against a database that no longer has them, and `endCrank` + // would then throw "No such savepoint: t0" from the run loop's + // `finally` — burying the failure that actually killed the kernel. + ctx.savepoints.length = 0; + throw error; } // The rollback reverted DB state but in-memory caches are stale. // Recreate the run queue so its cached head/tail are re-read from DB. @@ -81,11 +87,11 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { kdb.releaseSavepoint('t0'); } finally { // Forget the savepoints even if the release failed, as `rollbackCrank` - // does. A failed release discards the whole transaction (see - // `releaseSavepoint`), so the database has no savepoints left either; - // leaving them listed here would have the next crank number its savepoint - // `t1`, and from then on every release and rollback would aim one crank - // past the one it meant to end. + // does when its own rollback fails. A failed release discards the whole + // transaction (see `releaseSavepoint`), so the database has no savepoints + // left either; leaving them listed here would have the next crank number + // its savepoint `t1`, and from then on every release and rollback would + // aim one crank past the one it meant to end. ctx.savepoints.length = 0; } } From d9971b08dac8f60c711e00b584026c4b51a0ca0d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:37:23 +0200 Subject: [PATCH 06/40] fix(kernel-store): log an abort that fails while discarding a transaction Both drivers recover from a failed savepoint operation by discarding the enclosing transaction, and swallow any error from that abort so the savepoint failure stays the one reported. That part is right, but it left the abandoned transaction entirely silent: on the nodejs driver, where `inTransaction` is read from SQLite, the next crank's `beginIfNeeded` sees the transaction still open, skips its `BEGIN`, and commits the dead crank's writes alongside the new crank's. Nothing here can repair that, so at least record it. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/src/sqlite/nodejs.ts | 22 ++++++++++++++++++---- packages/kernel-store/src/sqlite/wasm.ts | 22 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index d0a6442442..5b7e777b0c 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -298,8 +298,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The rollback failure below is the one worth reporting. + } catch (abortError) { + // The rollback failure below is the one worth reporting, but a failed + // abort leaves SQLite holding a transaction this driver has stopped + // tracking, which the next crank would silently write into. Nothing here + // can repair that, so at least say so. + logger?.error( + 'failed to discard transaction after rollback', + abortError, + ); } throw error; } @@ -332,8 +339,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The release failure below is the one worth reporting. + } catch (abortError) { + // The release failure below is the one worth reporting, but a failed + // abort leaves SQLite holding a transaction this driver has stopped + // tracking, which the next crank would silently write into. Nothing here + // can repair that, so at least say so. + logger?.error( + 'failed to discard transaction after release', + abortError, + ); } throw error; } diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index beda148117..65d9915e48 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -388,8 +388,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The rollback failure below is the one worth reporting. + } catch (abortError) { + // The rollback failure below is the one worth reporting. `_inTx` is + // already false by then, so the next `beginIfNeeded` issues its `BEGIN` + // and SQLite says loudly if it really is still in a transaction — but + // that is a crank away, and this is where the evidence is. + logger?.error( + 'failed to discard transaction after rollback', + abortError, + ); } throw error; } @@ -422,8 +429,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The release failure below is the one worth reporting. + } catch (abortError) { + // The release failure below is the one worth reporting. `_inTx` is + // already false by then, so the next `beginIfNeeded` issues its `BEGIN` + // and SQLite says loudly if it really is still in a transaction — but + // that is a crank away, and this is where the evidence is. + logger?.error( + 'failed to discard transaction after release', + abortError, + ); } throw error; } From 48d46c62a5685887867a0933e3b6d11dd6c27751 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:38:10 +0200 Subject: [PATCH 07/40] test(ocap-kernel): pin the flush's ordering against a failing enqueue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving `#invokeKernelSubscription` out of the enqueue loop and after it was the one production change on this branch with no test: reverting `#flushCrankBuffer` to its interleaved form left all 2412 ocap-kernel tests passing. Same hazard as the crank-level ordering a few tests up, one level down — `#enqueueRun` is store work and can fail part-way, so answering the first caller while the second enqueue is still ahead hands out a result the crank's rollback then discards. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelQueue.test.ts | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 85a6d22d3c..7f9a30bb82 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -272,6 +272,57 @@ describe('KernelQueue', () => { ); }); + // The same invariant one level down, inside the flush itself: moving every + // buffered item onto the run queue is store work too, and it can fail + // part-way. Answering the first caller while the second enqueue is still + // ahead would hand out a result the crank's rollback then discards. + it('answers no caller until every buffered item is enqueued', async () => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp1' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + const resolve = vi.fn(); + const reject = vi.fn(); + kernelQueue.subscriptions.set('kp1', { resolve, reject }); + + // Two resolutions to hand over, the caller's first. + ( + kernelStore.flushCrankBuffer as unknown as MockInstance + ).mockReturnValueOnce([ + { type: 'notify', endpointId: 'v1', kpid: 'kp1' }, + { type: 'notify', endpointId: 'v2', kpid: 'kp2' }, + ]); + (kernelStore.getKernelPromise as unknown as MockInstance).mockReturnValue( + { state: 'fulfilled', value: { body: '"answer"', slots: [] } }, + ); + + // The second enqueue is the write that fails. + const enqueueError = new Error('database is gone'); + let enqueued = 0; + (kernelStore.enqueueRun as unknown as MockInstance).mockImplementation( + () => { + enqueued += 1; + if (enqueued > 1) { + throw enqueueError; + } + }, + ); + + const deliver = vi.fn().mockResolvedValue(undefined); + await expect(kernelQueue.run(deliver)).rejects.toBe(enqueueError); + + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); + expect(resolve).not.toHaveBeenCalled(); + }); + // Rolling back to the crank's *outermost* savepoint discards the enclosing // transaction (see `rollbackSavepoint`), which would leave the work an aborted // crank still owes — terminating the vat whose delivery failed, collecting From dc8d1dec2755e699165b3946ba1643431923c2f2 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:38:24 +0200 Subject: [PATCH 08/40] docs: correct the transaction claims review found wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five comments on this branch asserted more than the code holds: - `wasm.ts` claimed a stale `_inTx` meant "no later rollback can undo the delivery". False: a savepoint created in autocommit mode does open a transaction, and an inner savepoint still rolls back. The real cost is that writes outside a savepoint autocommit one statement at a time, and the outermost `RELEASE` commits. The "an abort typically fails because SQLite already rolled back" premise was unsupported and isn't the reason for the reorder — the reason is simply that the abort can throw. - `#processCrankResult` said "the worker is already gone" ahead of the call that kills the worker. - The flush was described as running "once nothing fallible remains". It doesn't: `#terminateVat` resolves the dying vat's promises through `resolvePromises`, which defaults to `immediate` and invokes their kernel subscriptions before `collectGarbage`. Reachable without an abort, via a clean `exitVat`. Recorded rather than fixed — closing it changes termination semantics, not crank ordering. - "Only `delivery` is ever rolled back" is true of the run loop but not of the tests. Scoped, and the ordinal coupling it depends on is now stated: `endCrank` releases `t0` by position, so `crank` must stay first. - `reapImporterUntil` credited `scheduleReap` deduping for the old one-BOYD behaviour; it was `nextReapAction` shifting the single entry off, leaving the later cranks nothing to do. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 6 +++ packages/kernel-store/src/sqlite/wasm.ts | 16 ++++---- .../src/garbage-collection.test.ts | 15 ++++--- packages/ocap-kernel/CHANGELOG.md | 5 +++ packages/ocap-kernel/src/KernelQueue.ts | 39 ++++++++++++------- 5 files changed, 56 insertions(+), 25 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 51e5eb1116..a5c90de8e6 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` itself fails, the same way `rollbackSavepoint` already did when `ROLLBACK TO` failed ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - The same hazard by the other door: the savepoint stayed on the stack and the transaction stayed open with nothing left that would ever commit or abort it. The release failure is what gets thrown, even if aborting fails too +- The wasm driver leaves `_inTx` false when aborting a transaction throws, rather than believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - It tracks `_inTx` itself instead of reading it from SQLite, so a throwing abort was the one case that could leave the two disagreeing. Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted a statement at a time; left false, the next `BEGIN` fails loudly if SQLite really is still in a transaction. The nodejs driver reads `db.inTransaction` and was never affected +- An abort that fails while recovering from a failed savepoint operation is logged, in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - The savepoint failure is still the one thrown, but the abandoned transaction it leaves behind was previously silent ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 65d9915e48..45313869fa 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -211,13 +211,15 @@ export async function makeSQLKernelDatabase({ function rollbackIfNeeded(): void { if (db._inTx) { // Out of the transaction as far as this driver is concerned before the - // abort is even attempted. Unlike the nodejs driver, which reads - // `inTransaction` from SQLite, `_inTx` is tracked here — and an abort - // typically fails because SQLite already rolled back on its own as part of - // whatever went wrong. Left true, `beginIfNeeded` is a no-op from then on - // and the next `createSavepoint` runs in autocommit mode, where its - // `RELEASE` commits (see `createSavepoint`) and no later rollback can undo - // the delivery. + // abort is even attempted, because the abort can throw. Unlike the nodejs + // driver, which reads `inTransaction` from SQLite, `_inTx` is tracked here, + // so a throw is the one thing that can leave the two disagreeing. Left + // true, `beginIfNeeded` is a no-op from then on: writes outside a savepoint + // autocommit one statement at a time, and the outermost `RELEASE` of a + // savepoint created in autocommit mode commits rather than nesting (see + // `createSavepoint`). Setting it false instead means the next `BEGIN` + // throws if SQLite really is still in a transaction, which is the failure + // worth having. db._inTx = false; db._spStack.length = 0; sqlAbortTransaction.step(); diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 45f21f12d2..c081955853 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -122,13 +122,18 @@ describe('Garbage Collection', () => { * vat's own garbage collection, or the attempts run out. * * `bringOutYourDead` can only report an import as dropped once the engine has - * collected the vat's presence and run its finalizer, which forcing a GC pass + * collected the vat's presence and run its finalizer, which `gcAndFinalize` * does not guarantee on the first attempt. Reaping once and then cranking - * buys one attempt rather than several, because `scheduleReap` dedupes — so - * each attempt schedules its own reap, and a message to the vat wakes the run - * loop to consume it. + * repeatedly buys one attempt rather than several, because `nextReapAction` + * shifts the single scheduled entry off and the later cranks find nothing to + * do — so each attempt has to schedule its own reap, with a message to the vat + * to wake the run loop and consume it. * - * @param settled - Whether the state under test has arrived. + * Gives up after five attempts and lets the caller's assertion report the + * failure, which names the refcount that never arrived. + * + * @param settled - Predicate answering whether the state under test has + * arrived. */ async function reapImporterUntil(settled: () => boolean): Promise { const isImporter = (vatId: VatId): boolean => vatId === importerVatId; diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 1393fc21a5..78b13012ce 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -70,6 +70,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly +- Keep a crank's store work inside one transaction, so the writes an aborted crank still owes are committed or undone as a unit ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost savepoint ends the enclosing transaction, so terminating the vat and collecting garbage — which follow the rollback, and must survive it — were autocommitting a statement at a time, beyond the reach of any later rollback + - Buffered vat outputs are flushed after that work rather than before it. The flush settles the promise `queueMessage` handed an external caller, so a later failure used to roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A failed rollback or release discards the whole transaction, taking every savepoint with it. Keeping them listed had the next crank number its savepoint `t1` while the database had none, and had `endCrank` throw `No such savepoint: t0` from the run loop's `finally` — over whatever really killed the kernel - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 9086c0e961..57409717c0 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -135,8 +135,11 @@ export class KernelQueue { // the enclosing transaction (see `rollbackSavepoint`), and the work an // aborted crank still owes — terminating the vat whose delivery failed, // collecting garbage — would then autocommit statement by statement, - // beyond the reach of any later rollback. Only `delivery` is ever rolled - // back; releasing `crank` in `endCrank` is this crank's one commit point. + // beyond the reach of any later rollback. The run loop only ever rolls + // back `delivery`; `crank` is released by `endCrank`, which is this + // crank's one commit point. That release names the *first* savepoint + // created here, by ordinal — see `releaseAllSavepoints` — so `crank` has + // to stay first. this.#kernelStore.createCrankSavepoint('crank'); this.#kernelStore.createCrankSavepoint('delivery'); @@ -342,21 +345,30 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // Vat termination during delivery is triggered by an illegal syscall - // or by syscall.exit(). Its store writes have to survive the rollback above: - // the worker is already gone, so a store that still believed the vat was - // alive would relaunch it after a restart and redeliver what killed it. + // Vat termination during delivery is triggered by an illegal syscall or by + // syscall.exit(). This call is what kills the worker, and its store writes + // have to survive the rollback above: once the worker is gone, a store that + // still believed the vat was alive would relaunch it after a restart and + // redeliver what killed it. Hence the rollback goes only as far as + // `delivery`, leaving these writes inside the crank's transaction. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { - // The crank survived, so hand its buffered outputs on — last, once nothing - // fallible remains. The flush settles the promise `enqueueMessage` gave an - // external caller, reading the result out of the store; were the crank - // rolled back after that, the caller would keep an answer computed from - // state the store discarded, and a restart would deliver the message again. + // The crank survived, so hand its buffered outputs on — after the store + // work above, which can still fail. The flush settles the promise + // `enqueueMessage` gave an external caller, reading the result out of the + // store; were the crank rolled back after that, the caller would keep an + // answer computed from state the store discarded, and a restart would + // deliver the message again. + // + // Not airtight: `#terminateVat` resolves the promises the dying vat was + // deciding via `resolvePromises`, which defaults to `immediate` and so + // invokes their kernel subscriptions before `collectGarbage` runs. Closing + // that would mean deferring those too, which is a change to termination + // semantics rather than to crank ordering. this.#flushCrankBuffer(); } // After the flush, because the audit reads the run queue as ground truth @@ -396,8 +408,9 @@ export class KernelQueue { resolved.push(item.kpid); } } - // Also promises resolved during this crank that don't have kernel-level - // subscribers (e.g., promises from enqueueMessage). + // Plus promises resolved during this crank that produced no notify of their + // own — nothing in the store was subscribed to them — but that the kernel + // itself is waiting on (e.g., promises from `enqueueMessage`). resolved.push(...this.#resolvedWithKernelSubscription); this.#resolvedWithKernelSubscription = []; From dc6eef44f6144d4ed162ecfeb1d0d389167a5b88 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:50:32 +0200 Subject: [PATCH 09/40] docs: cut the padding from this branch's comments and changelogs Comment the non-obvious why, in the shortest form that carries it. The two-savepoint rationale was re-argued in full in four places; the tests now point at `#runLoop` and `#processCrankResult` instead of restating them, and the hazard block duplicated across both driver test files is a line. No reasoning removed, only the retelling. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 11 ++- .../kernel-store/src/sqlite/nodejs.test.ts | 5 +- packages/kernel-store/src/sqlite/nodejs.ts | 10 +-- packages/kernel-store/src/sqlite/wasm.test.ts | 17 ++-- packages/kernel-store/src/sqlite/wasm.ts | 30 +++---- .../kernel-test/src/crank-rollback.test.ts | 14 ++- .../src/garbage-collection.test.ts | 16 ++-- packages/ocap-kernel/CHANGELOG.md | 8 +- packages/ocap-kernel/src/KernelQueue.test.ts | 24 ++--- packages/ocap-kernel/src/KernelQueue.ts | 88 +++++++------------ .../src/store/methods/crank.test.ts | 16 ++-- .../ocap-kernel/src/store/methods/crank.ts | 25 +++--- 12 files changed, 98 insertions(+), 166 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index a5c90de8e6..fa0c284ba4 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,12 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` itself fails, the same way `rollbackSavepoint` already did when `ROLLBACK TO` failed ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - The same hazard by the other door: the savepoint stayed on the stack and the transaction stayed open with nothing left that would ever commit or abort it. The release failure is what gets thrown, even if aborting fails too -- The wasm driver leaves `_inTx` false when aborting a transaction throws, rather than believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - It tracks `_inTx` itself instead of reading it from SQLite, so a throwing abort was the one case that could leave the two disagreeing. Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted a statement at a time; left false, the next `BEGIN` fails loudly if SQLite really is still in a transaction. The nodejs driver reads `db.inTransaction` and was never affected -- An abort that fails while recovering from a failed savepoint operation is logged, in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - The savepoint failure is still the one thrown, but the abandoned transaction it leaves behind was previously silent +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown +- The wasm driver clears `_inTx` when aborting a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted. The nodejs driver reads `db.inTransaction` and was never affected +- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index e38655d9a9..270f018ae6 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -360,10 +360,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._spStack).toStrictEqual([]); }); - // The same hazard `rollbackSavepoint` guards against, by the other door: a - // RELEASE that throws leaves the savepoint on the stack and the transaction - // open with nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. + // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb.inTransaction = true; diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index 5b7e777b0c..fa754bb88f 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -300,9 +300,8 @@ export async function makeSQLKernelDatabase({ rollbackIfNeeded(); } catch (abortError) { // The rollback failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction this driver has stopped - // tracking, which the next crank would silently write into. Nothing here - // can repair that, so at least say so. + // abort leaves SQLite holding a transaction the next crank would + // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -341,9 +340,8 @@ export async function makeSQLKernelDatabase({ rollbackIfNeeded(); } catch (abortError) { // The release failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction this driver has stopped - // tracking, which the next crank would silently write into. Nothing here - // can repair that, so at least say so. + // abort leaves SQLite holding a transaction the next crank would + // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 25403caa47..8e82a00bc9 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -517,10 +517,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // The same hazard `rollbackSavepoint` guards against, by the other door: a - // RELEASE that throws leaves the savepoint on the stack and the transaction - // open with nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. + // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -555,9 +552,8 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // `_inTx` is tracked here rather than read from SQLite, so a failed abort is - // the one case that can leave it disagreeing with the database. Left true, - // `beginIfNeeded` becomes a no-op forever after. + // A failed abort is the one case that can leave `_inTx` disagreeing with the + // database. Left true, `beginIfNeeded` is a no-op forever after. it('stops believing it is in a transaction when the abort fails too', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -576,10 +572,9 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // The consequence of the above, and the reason it is worth asserting: a - // savepoint created outside a transaction autocommits when released - // (Agoric/agoric-sdk#8423), so no later rollback can undo the delivery — an - // aborted crank would silently keep its writes. + // Why that matters: a savepoint created outside a transaction commits when + // released (Agoric/agoric-sdk#8423), so an aborted crank would keep its + // writes. it('begins a transaction for the next savepoint after a failed abort', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 45313869fa..a43092732b 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -210,16 +210,12 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - // Out of the transaction as far as this driver is concerned before the - // abort is even attempted, because the abort can throw. Unlike the nodejs - // driver, which reads `inTransaction` from SQLite, `_inTx` is tracked here, - // so a throw is the one thing that can leave the two disagreeing. Left - // true, `beginIfNeeded` is a no-op from then on: writes outside a savepoint - // autocommit one statement at a time, and the outermost `RELEASE` of a - // savepoint created in autocommit mode commits rather than nesting (see - // `createSavepoint`). Setting it false instead means the next `BEGIN` - // throws if SQLite really is still in a transaction, which is the failure - // worth having. + // Cleared before the abort is attempted, because the abort can throw and + // `_inTx` is tracked here rather than read from SQLite as the nodejs driver + // does. Left true, `beginIfNeeded` is a no-op forever after and writes + // autocommit one statement at a time (see `createSavepoint`). Cleared, a + // still-open transaction surfaces as a failed `BEGIN` — the louder + // failure. db._inTx = false; db._spStack.length = 0; sqlAbortTransaction.step(); @@ -391,10 +387,9 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The rollback failure below is the one worth reporting. `_inTx` is - // already false by then, so the next `beginIfNeeded` issues its `BEGIN` - // and SQLite says loudly if it really is still in a transaction — but - // that is a crank away, and this is where the evidence is. + // The rollback failure below is the one worth reporting. The next + // `BEGIN` will fail if SQLite really is still in a transaction, but that + // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -432,10 +427,9 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The release failure below is the one worth reporting. `_inTx` is - // already false by then, so the next `beginIfNeeded` issues its `BEGIN` - // and SQLite says loudly if it really is still in a transaction — but - // that is a crank away, and this is where the evidence is. + // The release failure below is the one worth reporting. The next + // `BEGIN` will fail if SQLite really is still in a transaction, but that + // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index dce56b3227..edb4fd0e61 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -138,9 +138,8 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('second')).toBe('yes'); }); - // An aborted crank rolls its delivery back and then still has work to do — - // terminating the vat whose delivery failed, collecting garbage — whose writes - // have to survive that rollback, since the vat's worker is already gone. + // An aborted crank still owes work after the rollback — terminating the vat, + // collecting garbage — whose writes have to survive it. it('keeps the writes a crank makes after rolling its delivery back', async () => { const { kernelStore, kdb } = await makeStore(); @@ -157,12 +156,9 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBe('yes'); }); - // And they survive it *as part of the crank's transaction*, which is why the - // delivery gets a savepoint of its own rather than rolling back the crank's: - // rolling back the outermost savepoint discards the transaction, after which - // those writes would autocommit one statement at a time. Nothing in the run - // loop rolls the crank's own savepoint back — it is the only way from here to - // observe that the writes are still undoable at all. + // And survive it *inside the crank's transaction*, not as autocommitted + // statements. Rolling back `crank` is the only way to observe that from here; + // the run loop never does it. it('holds those writes in the transaction rather than autocommitting them', async () => { const { kernelStore, kdb } = await makeStore(); diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index c081955853..2fc1dc1605 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -121,19 +121,15 @@ describe('Garbage Collection', () => { * Reap the importer vat until the kernel's bookkeeping catches up with the * vat's own garbage collection, or the attempts run out. * - * `bringOutYourDead` can only report an import as dropped once the engine has + * `bringOutYourDead` reports an import as dropped only once the engine has * collected the vat's presence and run its finalizer, which `gcAndFinalize` - * does not guarantee on the first attempt. Reaping once and then cranking - * repeatedly buys one attempt rather than several, because `nextReapAction` - * shifts the single scheduled entry off and the later cranks find nothing to - * do — so each attempt has to schedule its own reap, with a message to the vat - * to wake the run loop and consume it. + * does not guarantee on the first attempt. Each attempt needs its own reap — + * `nextReapAction` shifts the one scheduled entry off, so cranking again finds + * nothing to do — plus a message to wake the run loop and consume it. * - * Gives up after five attempts and lets the caller's assertion report the - * failure, which names the refcount that never arrived. + * Gives up after five attempts; the caller's assertion reports the failure. * - * @param settled - Predicate answering whether the state under test has - * arrived. + * @param settled - Whether the state under test has arrived yet. */ async function reapImporterUntil(settled: () => boolean): Promise { const isImporter = (vatId: VatId): boolean => vatId === importerVatId; diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 78b13012ce..84a230284a 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -70,11 +70,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction, so the writes an aborted crank still owes are committed or undone as a unit ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost savepoint ends the enclosing transaction, so terminating the vat and collecting garbage — which follow the rollback, and must survive it — were autocommitting a statement at a time, beyond the reach of any later rollback - - Buffered vat outputs are flushed after that work rather than before it. The flush settles the promise `queueMessage` handed an external caller, so a later failure used to roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time + - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - A failed rollback or release discards the whole transaction, taking every savepoint with it. Keeping them listed had the next crank number its savepoint `t1` while the database had none, and had `endCrank` throw `No such savepoint: t0` from the run loop's `finally` — over whatever really killed the kernel + - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 7f9a30bb82..b70d5a488c 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -214,13 +214,8 @@ describe('KernelQueue', () => { expect(kernelStore.endCrank).toHaveBeenCalled(); }); - // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external - // caller, reading the resolution out of the store on the way. Were the crank - // rolled back after that, the store would un-resolve the promise and restore - // the run queue item, so a restart would deliver the message a second time - // and notify every other subscriber again — while the original caller had - // already been told the first answer. So the flush comes last, after - // everything that could still fail. + // Why the flush comes after the crank's fallible work: see + // `#processCrankResult`. Here the terminate is what fails. it('answers no caller from a crank it then rolls back', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -272,10 +267,8 @@ describe('KernelQueue', () => { ); }); - // The same invariant one level down, inside the flush itself: moving every - // buffered item onto the run queue is store work too, and it can fail - // part-way. Answering the first caller while the second enqueue is still - // ahead would hand out a result the crank's rollback then discards. + // The same invariant inside the flush: `#enqueueRun` is store work and can + // fail part-way, so no caller may be answered until all of it lands. it('answers no caller until every buffered item is enqueued', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -323,13 +316,8 @@ describe('KernelQueue', () => { expect(resolve).not.toHaveBeenCalled(); }); - // Rolling back to the crank's *outermost* savepoint discards the enclosing - // transaction (see `rollbackSavepoint`), which would leave the work an aborted - // crank still owes — terminating the vat whose delivery failed, collecting - // garbage — autocommitting statement by statement, beyond the reach of any - // later rollback. That work has to follow the rollback, since the worker is - // already gone and the store must not go on believing the vat is alive, so it - // is the rollback that has to spare the transaction. + // Why two savepoints: see `#runLoop`. This pins that the rollback spares the + // transaction, so the work an aborted crank still owes stays inside it. it.each([ { label: 'an abort', diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 57409717c0..358df83aef 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -56,14 +56,10 @@ export class KernelQueue { /** * Whether this crank's savepoint has already been handed to `rollbackCrank`. - * Attempted, not necessarily succeeded: `rollbackCrank` forgets the savepoint - * whether or not the database call throws, so after either outcome a second - * attempt can only report "no such savepoint" over the real error. - * - * This has to be recorded at the moment of the attempt rather than returned - * from `#processCrankResult`, because that method can throw after rolling back - * (`#terminateVat`, `collectGarbage`), and the catch below must still know not - * to ask twice. + * Attempted, not necessarily succeeded: it is forgotten either way, so a second + * attempt could only report "no such savepoint" over the real error. Recorded + * at the attempt rather than returned, because `#processCrankResult` can throw + * after rolling back. */ #crankRollbackAttempted: boolean = false; @@ -130,24 +126,18 @@ export class KernelQueue { this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; try { - // Two savepoints, because the crank's transaction has to outlive the - // delivery's rollback. Rolling back to the outermost savepoint discards - // the enclosing transaction (see `rollbackSavepoint`), and the work an - // aborted crank still owes — terminating the vat whose delivery failed, - // collecting garbage — would then autocommit statement by statement, - // beyond the reach of any later rollback. The run loop only ever rolls - // back `delivery`; `crank` is released by `endCrank`, which is this - // crank's one commit point. That release names the *first* savepoint - // created here, by ordinal — see `releaseAllSavepoints` — so `crank` has - // to stay first. + // Two savepoints, because rolling back the outermost one discards the + // enclosing transaction (see `rollbackSavepoint`) and an aborted crank + // still has writes to make. Only `delivery` is ever rolled back; + // releasing `crank` in `endCrank` is this crank's one commit point. + // `releaseAllSavepoints` names it by ordinal, so `crank` must stay first. this.#kernelStore.createCrankSavepoint('crank'); this.#kernelStore.createCrankSavepoint('delivery'); // The savepoint exists from here on, so a throw can be undone. Without - // this, `endCrank`'s savepoint release commits the half-finished crank: - // the item this crank dequeued is gone for good, refcount increments - // stick, and promises resolved during it stay resolved while their - // notifies die unflushed. A restart would resume from that. + // this, `endCrank`'s release commits the half-finished crank: the + // dequeued item is gone for good, refcount increments stick, and + // resolved promises keep their unflushed notifies. try { const queueItem = this.#getNextRunQueueItem(); if (queueItem) { @@ -164,15 +154,14 @@ export class KernelQueue { wakeUpPromise = promise; } } catch (error) { - // An aborted crank already asked, and `rollbackCrank` discards the - // savepoint either way; asking again could only throw "no such - // savepoint" over the real error. + // An aborted crank already asked; asking again could only throw "no + // such savepoint" over the real error. if (!this.#crankRollbackAttempted) { try { this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { - // The original failure stays the `cause`, since that is the root - // cause an operator needs; the rollback failure is named here. + // The original failure stays the `cause`; the rollback failure is + // named here. throw new Error( `Run loop died and its crank could not be rolled back: ${String(rollbackError)}`, { cause: error }, @@ -317,11 +306,9 @@ export class KernelQueue { try { this.#kernelStore.rollbackCrank('delivery'); } finally { - // Set even when the rollback threw. `rollbackCrank` forgets the - // savepoint in its own `finally`, so "attempted" and "the savepoint is - // gone" now coincide exactly — and a second attempt from the run loop's - // catch would report a missing savepoint as the reason the kernel died, - // burying the database error that actually killed it. + // Set even when the rollback threw: the savepoint is gone either way, so + // a second attempt would report a missing savepoint as the reason the + // kernel died, burying the error that actually killed it. this.#crankRollbackAttempted = true; } // Discard kernel subscriptions that were queued for invocation @@ -345,30 +332,23 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // Vat termination during delivery is triggered by an illegal syscall or by - // syscall.exit(). This call is what kills the worker, and its store writes - // have to survive the rollback above: once the worker is gone, a store that - // still believed the vat was alive would relaunch it after a restart and - // redeliver what killed it. Hence the rollback goes only as far as - // `delivery`, leaving these writes inside the crank's transaction. + // This call kills the worker, so its writes must outlive the rollback above: + // a store that still believed the vat was alive would relaunch it after a + // restart and redeliver what killed it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { - // The crank survived, so hand its buffered outputs on — after the store - // work above, which can still fail. The flush settles the promise - // `enqueueMessage` gave an external caller, reading the result out of the - // store; were the crank rolled back after that, the caller would keep an - // answer computed from state the store discarded, and a restart would - // deliver the message again. + // After the fallible work above, not before it. The flush settles the + // promise `enqueueMessage` gave an external caller, so a later rollback + // would discard the state that answer was computed from. // - // Not airtight: `#terminateVat` resolves the promises the dying vat was - // deciding via `resolvePromises`, which defaults to `immediate` and so - // invokes their kernel subscriptions before `collectGarbage` runs. Closing - // that would mean deferring those too, which is a change to termination - // semantics rather than to crank ordering. + // Not airtight: `#terminateVat` resolves the dying vat's promises through + // `resolvePromises`, which defaults to `immediate` and invokes their + // subscriptions before `collectGarbage`. Deferring those too would change + // termination semantics, not crank ordering. this.#flushCrankBuffer(); } // After the flush, because the audit reads the run queue as ground truth @@ -408,15 +388,13 @@ export class KernelQueue { resolved.push(item.kpid); } } - // Plus promises resolved during this crank that produced no notify of their - // own — nothing in the store was subscribed to them — but that the kernel - // itself is waiting on (e.g., promises from `enqueueMessage`). + // Plus promises with no vat subscriber to notify, which the kernel is + // nonetheless waiting on (e.g. from `enqueueMessage`). resolved.push(...this.#resolvedWithKernelSubscription); this.#resolvedWithKernelSubscription = []; - // Callbacks only once every store write is done. Each hands an external - // caller a result read out of the store, and a write that threw in between - // would have the crank rolled back underneath answers already given. + // Callbacks only once every `#enqueueRun` is done: one that threw partway + // would roll the crank back underneath answers already given. for (const kpid of resolved) { this.#invokeKernelSubscription(kpid); } diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 389afd7a03..08a1e49254 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -154,11 +154,9 @@ describe('crank methods', () => { expect(mockCrankBuffer).toHaveLength(0); }); - // A failed rollback discards the whole transaction, so the enclosing - // savepoints are gone from the database too — not just the one rolled back - // to. Truncating to the ordinal would leave `endCrank` releasing a `t0` the - // database no longer has, and that "No such savepoint" would be thrown from - // the run loop's `finally`, over whatever really killed the kernel. + // A failed rollback discards every savepoint, not just this one. Truncating + // to the ordinal would have `endCrank` release a `t0` the database lacks and + // throw over whatever really killed the kernel. it('forgets every savepoint when the rollback fails', () => { context.inCrank = true; crankMethods.createCrankSavepoint('crank'); @@ -230,11 +228,9 @@ describe('crank methods', () => { expect(await waiter).toBeUndefined(); }); - // What `rollbackCrank` already does when its own rollback fails. Settling - // the crank regardless means callers proceed, so a savepoint left listed - // here has the next crank number its savepoint `t1` while the database still - // has `t0`: from then on `releaseAllSavepoints` releases the wrong one and - // every rollback aims past the crank it meant to undo. + // As `rollbackCrank` does. Left listed, the next crank numbers its savepoint + // `t1` against a database that has none, and every later release and rollback + // aims one crank past its target. it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); context.savepoints = ['test']; diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index b26219cbb6..887ef2bee1 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -51,17 +51,14 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { if (ctx.savepoints[ordinal] === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); - // Forget the savepoint. Leaving it listed would have `endCrank`'s - // release commit the crank we just abandoned — the half-finished state - // this rollback exists to discard. + // Left listed, `endCrank`'s release would commit the crank we just + // abandoned. ctx.savepoints.length = ordinal; } catch (error) { - // A failed rollback discards the whole transaction (see - // `rollbackSavepoint`), taking every savepoint with it, not just this - // one. Truncating to `ordinal` would leave the enclosing savepoints - // listed against a database that no longer has them, and `endCrank` - // would then throw "No such savepoint: t0" from the run loop's - // `finally` — burying the failure that actually killed the kernel. + // A failed rollback discards the whole transaction, so every savepoint + // is gone, not just this one. Truncating to `ordinal` would have + // `endCrank` release a `t0` the database lacks and throw over whatever + // really killed the kernel. ctx.savepoints.length = 0; throw error; } @@ -86,12 +83,10 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { try { kdb.releaseSavepoint('t0'); } finally { - // Forget the savepoints even if the release failed, as `rollbackCrank` - // does when its own rollback fails. A failed release discards the whole - // transaction (see `releaseSavepoint`), so the database has no savepoints - // left either; leaving them listed here would have the next crank number - // its savepoint `t1`, and from then on every release and rollback would - // aim one crank past the one it meant to end. + // A failed release discards the transaction too, so the database has no + // savepoints left either. Left listed, the next crank would number its + // savepoint `t1` and aim every later release and rollback one crank past + // the one it meant to end. ctx.savepoints.length = 0; } } From 323dd189620909ad3ae66d6a06cf6a066d4b5c49 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 10 Aug 2026 12:51:54 +0200 Subject: [PATCH 10/40] fix(ocap-kernel): revert cached values and GC candidates on crank rollback A database rollback cannot reach two pieces of state, so `rollbackCrank` now reverts both itself. Every `provideCachedStoredValue` answers reads from a closure and only writes through to kv. Reverting the database therefore left the closure holding the abandoned crank's value, and the next `set` persisted it. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright rather than retrying it. `reapQueue` was exposed the same way. `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of the decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop. No live bug either way: every `abort` `#deliverGCAction` returns is paired with a `terminate`, which is what made losing the action harmless. The comment there claimed the rollback restored the action, which is the thing a future reader would trust when adding an abort path that isn't paired with a termination; it now states the real causality. The cached values are declared once so that initialization and the refresher cannot disagree about which ones exist. Co-Authored-By: Claude Opus 5 (1M context) --- .../kernel-test/src/crank-rollback.test.ts | 64 +++++++++++++ packages/ocap-kernel/CHANGELOG.md | 4 + packages/ocap-kernel/src/store/index.ts | 89 ++++++++++++------- .../src/store/methods/crank.test.ts | 2 + .../ocap-kernel/src/store/methods/crank.ts | 54 +++++++++-- packages/ocap-kernel/src/store/types.ts | 1 + 6 files changed, 173 insertions(+), 41 deletions(-) diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index edb4fd0e61..36131a8a16 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -174,6 +174,70 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBeUndefined(); }); + // Every `provideCachedStoredValue` keeps its value in a closure and writes + // through to kv, so a rollback that only reverts the database leaves the cache + // holding the abandoned crank's value — and the next `set` persists it. The GC + // action set is the case that matters: `processGCActionSet` consumes an action + // before delivering it, so losing the rollback loses the action outright. + it('restores the GC action set consumed by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + kernelStore.addGCActions(['v1 dropExport ko1']); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + // Consume the action the way `processGCActionSet` does. + kernelStore.setGCActions(new Set()); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + expect([...kernelStore.getGCActions()]).toStrictEqual([ + 'v1 dropExport ko1', + ]); + }); + + // Same closure, same failure: a reap scheduled and then consumed by a crank + // that rolls back must still be pending afterwards. + it('restores the reap queue consumed by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + kernelStore.scheduleReap('v1'); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + expect(kernelStore.nextReapAction()).toBeDefined(); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + expect(kernelStore.nextReapAction()).toBeDefined(); + }); + + // `maybeFreeKrefs` is RAM-only, so nothing rolls it back. Left populated, the + // next crank's `collectGarbage` visits krefs whose decrements were undone — + // and `getKernelPromise` throws outright for one the rollback deleted, which + // kills the run loop. + it('discards GC candidates accumulated by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + // Born at 1, so this drops it to 0 and leaves `kpid` in `maybeFreeKrefs` + // while the rollback removes the promise record it names. + const kpid = kernelStore.initKernelPromise()[0]; + kernelStore.decrementRefCount(kpid, 'test'); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + expect(() => kernelStore.collectGarbage()).not.toThrow(); + kernelStore.endCrank(); + }); + // `createCrankSavepoint` records the name only once the database has the // savepoint. Asking to roll back one that was never created must therefore say // so, rather than releasing someone else's savepoint. diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 84a230284a..d24ebf8de9 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -75,6 +75,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel +- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A cached stored value answers reads from a closure and only writes through to the database, so reverting the database left it holding the abandoned crank's value and the next write persisted that. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright instead of retrying it + - `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop + - A failed rollback discards the whole transaction, moving the database back at least as far as a successful rollback would have — so reverting only on success left exactly the state that is least able to tolerate it - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 75f946eed6..f2257af545 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -92,7 +92,7 @@ import { getRevocationMethods } from './methods/revocation.ts'; import { getSubclusterMethods } from './methods/subclusters.ts'; import { getTranslators } from './methods/translators.ts'; import { getVatMethods } from './methods/vat.ts'; -import type { StoreContext } from './types.ts'; +import type { StoreContext, StoredValue } from './types.ts'; /** * The prefix shared by the issuance count of every object an ocap URL names, @@ -127,6 +127,48 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { const { getPrefixedKeys, provideCachedStoredValue, provideStoredQueue } = getBaseMethods(kv); + /** + * Every cached stored value the context holds, as `field: [key, initial]`. + * Declared once so that initialization and `refreshCachedValues` cannot + * disagree about which values exist: adding one here does both. + */ + const CACHED_VALUES = { + /** Counter for allocating kernel object IDs */ + nextObjectId: ['nextObjectId', '1'], + /** Counter for allocating kernel promise IDs */ + nextPromiseId: ['nextPromiseId', '1'], + /** Counter for allocating VatIDs */ + nextVatId: ['nextVatId', '1'], + /** Counter for allocating RemoteIDs */ + nextRemoteId: ['nextRemoteId', '1'], + // Garbage collection + gcActions: ['gcActions', '[]'], + reapQueue: ['reapQueue', '[]'], + terminatedVats: ['vats.terminated', '[]'], + // Subclusters + subclusters: ['subclusters', '[]'], + nextSubclusterId: ['nextSubclusterId', '1'], + vatToSubclusterMap: ['vatToSubclusterMap', '{}'], + } as const satisfies Record; + + /** + * Provide a fresh stored value for each of {@link CACHED_VALUES}, reading its + * current setting out of the database. + * + * @returns The stored values, keyed by the context field that holds each. + */ + function provideCachedValues(): Record< + keyof typeof CACHED_VALUES, + StoredValue + > { + return Object.fromEntries( + Object.entries(CACHED_VALUES).map(([field, [key, init]]) => [ + field, + provideCachedStoredValue(key, init), + ]), + ) as Record; + } + const context: StoreContext = { kv, /** The kernel's run queue. */ @@ -137,14 +179,16 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { refreshRunQueue: () => { context.runQueue = provideStoredQueue('run', true); }, - /** Counter for allocating kernel object IDs */ - nextObjectId: provideCachedStoredValue('nextObjectId', '1'), - /** Counter for allocating kernel promise IDs */ - nextPromiseId: provideCachedStoredValue('nextPromiseId', '1'), - /** Counter for allocating VatIDs */ - nextVatId: provideCachedStoredValue('nextVatId', '1'), - /** Counter for allocating RemoteIDs */ - nextRemoteId: provideCachedStoredValue('nextRemoteId', '1'), + ...provideCachedValues(), + /** + * Re-read every cached stored value from the database. Each one closes over + * the last value written through it (see `provideCachedStoredValue`), so + * reverting the database alone is not enough: the closure would still hold + * the abandoned value and the next `set` would write it straight back. + */ + refreshCachedValues: () => { + Object.assign(context, provideCachedValues()); + }, // As refcounts are decremented, we accumulate a set of krefs for which // action might need to be taken: // * promises which are now resolved and unreferenced can be deleted @@ -156,17 +200,9 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { // the change, else removals might be lost (not performed during the next // replay). maybeFreeKrefs: new Set(), - // Garbage collection - gcActions: provideCachedStoredValue('gcActions', '[]'), - reapQueue: provideCachedStoredValue('reapQueue', '[]'), - terminatedVats: provideCachedStoredValue('vats.terminated', '[]'), inCrank: false, savepoints: [], crankBuffer: [], - // Subclusters - subclusters: provideCachedStoredValue('subclusters', '[]'), - nextSubclusterId: provideCachedStoredValue('nextSubclusterId', '1'), - vatToSubclusterMap: provideCachedStoredValue('vatToSubclusterMap', '{}'), auditRefCounts: false, // Logging logger: logger?.subLogger({ tags: ['kernel-store'] }), @@ -226,23 +262,8 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { })); kdb.clear(); context.maybeFreeKrefs.clear(); - context.runQueue = provideStoredQueue('run', true); - context.gcActions = provideCachedStoredValue('gcActions', '[]'); - context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); - context.terminatedVats = provideCachedStoredValue('vats.terminated', '[]'); - context.nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - context.nextVatId = provideCachedStoredValue('nextVatId', '1'); - context.nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - context.subclusters = provideCachedStoredValue('subclusters', '[]'); - context.nextSubclusterId = provideCachedStoredValue( - 'nextSubclusterId', - '1', - ); - context.vatToSubclusterMap = provideCachedStoredValue( - 'vatToSubclusterMap', - '{}', - ); + context.refreshRunQueue(); + context.refreshCachedValues(); crank.releaseAllSavepoints(); context.crankBuffer.length = 0; preservedState?.forEach(({ key, value }) => { diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 08a1e49254..630444e850 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -17,6 +17,8 @@ describe('crank methods', () => { savepoints: [], crankBuffer: mockCrankBuffer, refreshRunQueue: vi.fn(), + refreshCachedValues: vi.fn(), + maybeFreeKrefs: new Set(), } as unknown as StoreContext; kdb = { diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 887ef2bee1..a3bba1956b 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -60,21 +60,61 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { // `endCrank` release a `t0` the database lacks and throw over whatever // really killed the kernel. ctx.savepoints.length = 0; + // Before the rethrow, and not only on the path below. A failed + // rollback discards the whole transaction, so the database has moved + // back at least as far as a successful rollback would have taken it + // and these caches are at least as stale. Rethrowing ahead of this + // would leave the dying crank holding the GC action it consumed and + // the freed krefs it was about to collect. + revertStateBeneathRollback(error); throw error; } - // The rollback reverted DB state but in-memory caches are stale. - // Recreate the run queue so its cached head/tail are re-read from DB. - ctx.refreshRunQueue(); - // Invalidate the run queue length cache so it's recalculated from - // the database on next access, since the rollback may have restored - // dequeued items. - ctx.runQueueLengthCache = -1; + revertStateBeneathRollback(); return; } } Fail`no such savepoint as "${q(savepoint)}"`; } + /** + * Revert what a database rollback cannot reach: the in-memory caches built + * over the abandoned crank's writes. + * + * @param rollbackError - The error the rollback threw, if it threw. Kept as + * the `cause` should reverting fail too, since it is the root cause an + * operator needs. + */ + function revertStateBeneathRollback(rollbackError?: unknown): void { + try { + // Recreate the run queue so its cached head/tail are re-read from the + // database, and invalidate the length cache, since the rollback may have + // restored dequeued items. + ctx.refreshRunQueue(); + ctx.runQueueLengthCache = -1; + // Same staleness, worse consequence: a cached value reads from its + // closure and only writes through to kv, so one this crank consumed stays + // consumed and the next `set` persists that. `processGCActionSet` takes an + // action out of the set before delivering it, so an action not restored + // here is lost rather than retried. + ctx.refreshCachedValues(); + // Nothing rolls back RAM. These krefs are collection candidates only + // because this crank decremented them, and that is precisely what was just + // undone. Left in place, `collectGarbage` throws on a later crank for any + // promise this one created — killing the run loop over work that no longer + // exists. Correct only while every rollback discards the whole delivery, + // which is all any caller asks for. + ctx.maybeFreeKrefs.clear(); + } catch (revertError) { + if (rollbackError === undefined) { + throw revertError; + } + throw new Error( + `Crank rollback failed and its caches could not be reverted: ${String(revertError)}`, + { cause: rollbackError }, + ); + } + } + /** * Release all savepoints. */ diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index 3bf54862fa..b9886c174f 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -11,6 +11,7 @@ export type StoreContext = { runQueue: StoredQueue; // Holds RunAction[] runQueueLengthCache: number; // Holds number refreshRunQueue: () => void; + refreshCachedValues: () => void; nextObjectId: StoredValue; // Holds string nextPromiseId: StoredValue; // Holds string nextVatId: StoredValue; // Holds string From 6c28b9631399658cec5aae85ca5f13a3b238bbbd Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:51:25 -0400 Subject: [PATCH 11/40] test(kernel-store): pin the failed COMMIT that wedges `_inTx` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue `rollbackIfNeeded` was corrected in #1012 to clear `_inTx` *before* stepping the abort, because the abort can throw and `_inTx` is tracked in the driver rather than read from SQLite. `commitIfNeeded` has the identical shape and was left alone: function commitIfNeeded(): void { if (db._inTx && db._spStack.length === 0) { sqlCommitTransaction.step(); // can throw sqlCommitTransaction.reset(); db._inTx = false; // ...so this never runs } } A COMMIT that throws leaves `_inTx` true against a database that may hold no transaction. `beginIfNeeded` is then a no-op forever after, so the next `createSavepoint` issues its SAVEPOINT outside a transaction — and a savepoint taken outside a transaction commits when it is released (Agoric/agoric-sdk#8423). That is the hazard the whole `beginIfNeeded` dance exists to prevent, and `commitIfNeeded` is reached from `releaseSavepoint`, which is the crank's commit point. The writes that leak are a whole crank's. The nodejs driver is unaffected, for the same reason it was unaffected by the abort case: it reads `db.inTransaction` live from SQLite. Worth noting that the comment introduced above `stops believing it is in a transaction when the abort fails too` asserts that a failed abort is "the one case that can leave `_inTx` disagreeing with the database". This is the second case, so that comment needs correcting along with the code. ## What we hope to see instead `releaseSavepoint` still throws the COMMIT failure, but `_inTx` is false afterwards, so the next `createSavepoint` opens a transaction of its own instead of creating a bare savepoint. Same two-line reorder as `rollbackIfNeeded`, and the "one case" comment updated. ## Current failure AssertionError: expected true to be false packages/kernel-store/src/sqlite/wasm.test.ts > stops believing it is in a transaction when the commit fails Co-Authored-By: Claude Opus 5 --- packages/kernel-store/src/sqlite/wasm.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 8e82a00bc9..fa4e2c8a80 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -599,6 +599,44 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); + // FAILING REPRO — see the commit message for this test. + // + // `commitIfNeeded` still steps the COMMIT before clearing `_inTx`, the exact + // ordering `rollbackIfNeeded` was corrected to avoid. A COMMIT that throws + // therefore leaves `_inTx` true against a database that may hold no + // transaction, `beginIfNeeded` is a no-op forever after, and the next + // savepoint is created outside a transaction — which commits when released + // (Agoric/agoric-sdk#8423). This is the crank's commit point, so the writes + // that leak are a whole crank's. + // + // The comment above `stops believing it is in a transaction when the abort + // fails too` calls a failed abort "the one case that can leave `_inTx` + // disagreeing with the database". This is the second case. + it('stops believing it is in a transaction when the commit fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + // The RELEASE goes through `exec` and succeeds; COMMIT is the first + // prepared statement this path steps, and it is what fails. + mockStatement.step.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + + // And so the next savepoint gets a transaction of its own rather than + // being created bare. + mockDb.exec.mockClear(); + mockStatement.step.mockClear(); + db.createSavepoint('next'); + expect(mockStatement.step).toHaveBeenCalledOnce(); + expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); From 3d3fe3e21d6f270bb06e8b97b618f50ce8ac49d6 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:52:51 -0400 Subject: [PATCH 12/40] test(ocap-kernel): pin the endCrank failure that buries the real error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 fixes one error-masking path at the start of a dying crank and opens another at its end. Before the two-savepoint scheme, `rollbackCrank('start')` emptied `ctx.savepoints`, so `endCrank` -> `releaseAllSavepoints` was a guaranteed no-op on the dying path: nothing to release, nothing that could throw. Now `rollbackCrank('delivery')` truncates to the ordinal and leaves `['crank']` behind (crank.ts:56, deliberately — that is what keeps the transaction open for the work an aborted crank still owes). So `endCrank` issues a real `RELEASE t0`, which commits, which can fail. `#runLoop` calls it from a bare `finally`: } finally { this.#kernelStore.endCrank(); ... } A throw there replaces the pending exception. The disk error that actually killed the kernel is discarded — not demoted to `cause`, discarded — and `run()` rejects with the release failure instead. `#failRunLoop` records that, so `getRunLoopStatus().detail` loses the root cause too, and `onRunLoopFailure` — what the daemon logs as fatal — gets the wrong error. A/B against origin/main with the same repro: main reports `crank exploded`, this branch reports `database is gone` with `cause: undefined`. This is the same class of bug as the `No such savepoint: t0` masking that 82b88ce62 fixes, and the same class the `reports both failures when the rollback also fails` test above already guards on the other path. ## What we hope to see instead Whatever names the release failure, the error that killed the crank stays reachable. The rollback path already has the shape to copy: throw new Error( `Run loop died and its crank could not be rolled back: ${...}`, { cause: error }, ); The assertion is deliberately fix-agnostic — it walks the `cause` chain — so either wrapping `endCrank`'s failure with the original as `cause`, or reporting it and rethrowing the original, will satisfy it. ## Current failure AssertionError: expected [ Error: database is gone ] to include Error: crank exploded packages/ocap-kernel/src/KernelQueue.test.ts > reports both failures when endCrank also fails Co-Authored-By: Claude Opus 5 --- packages/ocap-kernel/src/KernelQueue.test.ts | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index b70d5a488c..6e24915fd3 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -25,6 +25,24 @@ vi.mock('@endo/promise-kit', () => ({ */ const STOP_RUN_LOOP = 'test: stop run loop'; +/** + * Collect an error and every error reachable through its `cause` chain, so that + * a test can assert a root cause survived without pinning how its reporter + * chose to wrap it. + * + * @param error - The error to walk. + * @returns The chain, outermost first. + */ +const causeChain = (error: unknown): unknown[] => { + const chain: unknown[] = []; + let current = error; + while (current instanceof Error) { + chain.push(current); + current = current.cause; + } + return chain; +}; + describe('KernelQueue', () => { let kernelStore: KernelStore; let kernelQueue: KernelQueue; @@ -540,6 +558,43 @@ describe('KernelQueue', () => { }); }); + // FAILING REPRO — see the commit message for this test. + // + // The companion of the case above, at the other end of the crank. Since the + // delivery rollback now spares `crank`, `endCrank`'s release is a real + // RELEASE + COMMIT on the dying path where it used to be a no-op, and + // `#runLoop` calls it from a bare `finally` — so when it throws it replaces + // the error that killed the kernel instead of being reported alongside it. + it('reports both failures when endCrank also fails', async () => { + ( + kernelStore.runQueueLength as unknown as MockInstance + ).mockReturnValueOnce(1); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce({ + type: 'send', + target: 'ko123', + message: {} as KernelMessage, + }); + (kernelStore.endCrank as unknown as MockInstance).mockImplementation( + () => { + throw new Error('database is gone'); + }, + ); + const crankError = new Error('crank exploded'); + const deliver = vi.fn().mockRejectedValue(crankError); + + const failure = await kernelQueue + .run(deliver) + .catch((error: unknown) => error); + + // However the release failure is named, the error that actually killed the + // kernel has to stay reachable — as the rollback path already manages. + expect(causeChain(failure)).toContain(crankError); + expect(kernelQueue.getRunLoopStatus()).toMatchObject({ + state: 'failed', + detail: expect.stringContaining('crank exploded'), + }); + }); + // `rollbackCrank` discards the savepoint even when its database call throws, // so a second attempt could only report a missing savepoint. Without the // `finally` that records the attempt, the abort path leaves the flag unset, From f1e0ed0851b05b805ee20cc05212e721fcac334d Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:53:48 -0400 Subject: [PATCH 13/40] test(kernel-node-runtime): pin the kernel store's missing logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 replaces four silently-swallowed aborts with `logger?.error(...)` in the SQLite drivers, and its description says: "Four swallowed aborts were silent. Now logged." They are not. No production call site passes a `logger` to `makeSQLKernelDatabase`, so every one of those calls is dead code: packages/kernel-node-runtime/src/kernel/make-kernel.ts:63 packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts:47 packages/kernel-test-local/src/lms-chat.ts:30 packages/kernel-node-runtime/test/helpers/remote-comms.ts:172 `make-kernel.ts` is the clearest case: it builds a `rootLogger` and hands sub-loggers to `NodejsPlatformServices` and to `Kernel.make`, then constructs the store with `{ dbFilename }` alone. The store is the one collaborator that gets no logger. Nor does any test pass one, which is why the gap survived review. This matters more than a missing log line. On the nodejs driver a failed abort leaves `db.inTransaction` true with nothing that will ever commit or abort it, so later writes on that connection join a transaction that vanishes on close. The driver's own comment concedes "Nothing here can repair that" — the log is the entire remedy, and it does not reach anyone. `logger?.error` is the right convention for this package; the injection is what is missing. ## What we hope to see instead `makeKernel` passes a tagged sub-logger to `makeSQLKernelDatabase`, as it already does for its other collaborators — something like `rootLogger.subLogger({ tags: ['store'] })`. The other three call sites want the same treatment, and are worth covering once this one is fixed. ## Current failure AssertionError: expected "vi.fn()" to be called with arguments: [ ObjectContaining{…} ] - "logger": Any, packages/kernel-node-runtime/src/kernel/make-kernel.test.ts > gives the kernel store a logger Co-Authored-By: Claude Opus 5 --- .../src/kernel/make-kernel.test.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts index 57b0293d65..98490dd65f 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts @@ -1,3 +1,5 @@ +import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; +import { Logger } from '@metamask/logger'; import { Kernel } from '@metamask/ocap-kernel'; import { describe, expect, it, vi } from 'vitest'; @@ -8,7 +10,9 @@ vi.mock('@metamask/kernel-store/sqlite/nodejs', async () => { '../../../ocap-kernel/test/storage.ts' ); return { - makeSQLKernelDatabase: makeMapKernelDatabase, + // Wrapped so that a test can see what the database was constructed with, + // while still getting a real store back. + makeSQLKernelDatabase: vi.fn(makeMapKernelDatabase), }; }); @@ -18,4 +22,18 @@ describe('makeKernel', () => { expect(kernel).toBeInstanceOf(Kernel); }); + + // FAILING REPRO — see the commit message for this test. + // + // The kernel store is the only collaborator `makeKernel` builds without + // handing it a logger, so every `logger?.` call inside the SQLite driver is + // dead code in production — including the four abort failures #1012 added + // logging for. + it('gives the kernel store a logger', async () => { + await makeKernel({}); + + expect(makeSQLKernelDatabase).toHaveBeenCalledWith( + expect.objectContaining({ logger: expect.any(Logger) }), + ); + }); }); From c10b5b98bcc5b92c28fb63002f75ec888572b45d Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:56:04 -0400 Subject: [PATCH 14/40] test(ocap-kernel): pin the release failure lost at the remote savepoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 hardens `releaseSavepoint` so that a failed `RELEASE` discards the enclosing transaction, clearing the driver's `_spStack` on the way. Two callers it does not touch depend on the old behaviour, and both are now worse off than before the change. `RemoteHandle.handleRemoteMessage` releases inside the `try` and rolls back in the `catch`: this.#kernelStore.setRemoteHighestReceivedSeq(this.remoteId, seq); this.#kernelStore.releaseSavepoint(savepointName); // fails } catch (error) { this.#kernelStore.rollbackSavepoint(savepointName); // "No such savepoint" throw error; // never reached } Since the release already cleared the stack, the rollback throws `No such savepoint: receive_r0_1`, which escapes the `catch` and replaces the real failure. Not demoted to `cause` — replaced. `RemoteManager` has the same shape at its `peerIncarnation_*` savepoint. A/B verified against origin/main with a real driver: main's rollback succeeds and `database or disk is full` propagates; on this branch the caller gets the missing-savepoint error instead. So the PR description's "the release failure still propagates" holds for the crank path it fixed and not for these two. `crank.ts:57-63` shows the author recognised exactly this hazard — a stale savepoint list producing `No such savepoint` over the real error — and fixed it for the crank only. The remote paths were missed because nothing exercised them. Note the secondary effect these tests don't reach: `ctx.savepoints` still lists the crank's own savepoints after this, so the next `endCrank` throws `No such savepoint: t0` over whatever is left of the failure. ## What we hope to see instead The failure the database reported is what reaches the caller. Any of these does it, and the assertion doesn't care which: - move the release out of the `try`, so a release failure isn't followed by a rollback attempt at all - have the `catch` tolerate a rollback that reports a savepoint already discarded, rethrowing the original either way - make the driver's discard leave the name rollback-able as a no-op The mock models the drivers' bookkeeping rather than the expected outcome, so it is `RemoteHandle`'s error handling under test, not the mock's. ## Current failure AssertionError: expected Error: No such savepoint: receive_r0_1 to be Error: database or disk is full packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts > reports the release failure rather than a missing savepoint Co-Authored-By: Claude Opus 5 --- .../src/remotes/kernel/RemoteHandle.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index 4f53572c0d..ea201ada21 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -220,6 +220,49 @@ describe('RemoteHandle', () => { }); }); + // FAILING REPRO — see the commit message for this test. + // + // `handleRemoteMessage` releases its savepoint inside the `try` and rolls + // back in the `catch`. #1012 made a failed `RELEASE` discard the whole + // savepoint stack, so that rollback now reports a savepoint that no longer + // exists, and it throws out of the `catch` in place of the failure that + // brought it there. + it('reports the release failure rather than a missing savepoint', async () => { + // The drivers' bookkeeping as #1012 leaves it, verified against both: a + // failed RELEASE clears `_spStack`, and rolling back a name that is not on + // it throws `No such savepoint`. + const savepoints: string[] = []; + const releaseFailure = new Error('database or disk is full'); + mockKernelStore = { + ...mockKernelStore, + createSavepoint: (name: string) => { + savepoints.push(name); + }, + releaseSavepoint: () => { + savepoints.length = 0; + throw releaseFailure; + }, + rollbackSavepoint: (name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }, + } as KernelStore; + const remote = makeRemote(); + + const delivery = JSON.stringify({ + seq: 1, + method: 'deliver', + params: ['bringOutYourDead'], + }); + + // The error an operator needs is the one the database gave, not the + // bookkeeping artefact of trying to clean up after it. + await expect(remote.handleRemoteMessage(delivery)).rejects.toBe( + releaseFailure, + ); + }); + // A dead run loop will never deliver the message, and `handleRemoteMessage` // rolls back without advancing the received sequence number, so the peer // retries and gives up rather than being acknowledged by a black hole. From 7ed40cde20e37d74e3c724fb73f16c55f3d1e4b3 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:09:39 -0400 Subject: [PATCH 15/40] test: tighten the four repros after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No change to what any of them proves; all four still fail for the reasons their own commits describe. - `RemoteHandle`: assert the rollback is still *attempted*. Without this, deleting the rollback from the `catch` outright would turn the test green, which is not the fix — a `RELEASE` that failed for a reason of its own may well have left the savepoint standing. - `RemoteHandle`: drop an unnecessary `as KernelStore` cast, and say why the store is replaced wholesale rather than having its methods assigned over (`makeKernelStore` hardens what it returns). - `make-kernel`: note that `kernel-worker.ts` omits the logger too, so the wasm driver's pair of `logger?.error` calls stays dead even once this test passes. Use `vi.mocked`, as the sibling `make-kernel-options.test.ts` does. - `causeChain` returns `Error[]`; every element is already narrowed by the loop guard. - Drop "see the commit message for this test" from the four comment blocks: each stands alone, and the reference would not survive a squash-merge. Restate the claim the wasm comment made by citing a neighbouring test's title, which would have broken silently on rename. Co-Authored-By: Claude Opus 5 --- .../src/kernel/make-kernel.test.ts | 7 +++--- packages/kernel-store/src/sqlite/wasm.test.ts | 8 +++---- packages/ocap-kernel/src/KernelQueue.test.ts | 6 ++--- .../src/remotes/kernel/RemoteHandle.test.ts | 23 ++++++++++++------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts index 98490dd65f..672313a02a 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts @@ -23,16 +23,17 @@ describe('makeKernel', () => { expect(kernel).toBeInstanceOf(Kernel); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // The kernel store is the only collaborator `makeKernel` builds without // handing it a logger, so every `logger?.` call inside the SQLite driver is // dead code in production — including the four abort failures #1012 added - // logging for. + // logging for. `kernel-worker.ts` omits it too, which keeps the wasm driver's + // pair dead even once this passes. it('gives the kernel store a logger', async () => { await makeKernel({}); - expect(makeSQLKernelDatabase).toHaveBeenCalledWith( + expect(vi.mocked(makeSQLKernelDatabase)).toHaveBeenCalledWith( expect.objectContaining({ logger: expect.any(Logger) }), ); }); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index fa4e2c8a80..1cf496dc95 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -599,7 +599,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // `commitIfNeeded` still steps the COMMIT before clearing `_inTx`, the exact // ordering `rollbackIfNeeded` was corrected to avoid. A COMMIT that throws @@ -609,9 +609,9 @@ describe('makeSQLKernelDatabase', () => { // (Agoric/agoric-sdk#8423). This is the crank's commit point, so the writes // that leak are a whole crank's. // - // The comment above `stops believing it is in a transaction when the abort - // fails too` calls a failed abort "the one case that can leave `_inTx` - // disagreeing with the database". This is the second case. + // A failed abort is therefore not, as the abort case above claims, the one + // case that can leave `_inTx` disagreeing with the database. This is the + // second. it('stops believing it is in a transaction when the commit fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 6e24915fd3..d9c4626d03 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -33,8 +33,8 @@ const STOP_RUN_LOOP = 'test: stop run loop'; * @param error - The error to walk. * @returns The chain, outermost first. */ -const causeChain = (error: unknown): unknown[] => { - const chain: unknown[] = []; +const causeChain = (error: unknown): Error[] => { + const chain: Error[] = []; let current = error; while (current instanceof Error) { chain.push(current); @@ -558,7 +558,7 @@ describe('KernelQueue', () => { }); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // The companion of the case above, at the other end of the crank. Since the // delivery rollback now spares `crank`, `endCrank`'s release is a real diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index ea201ada21..cb0142e064 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -220,7 +220,7 @@ describe('RemoteHandle', () => { }); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // `handleRemoteMessage` releases its savepoint inside the `try` and rolls // back in the `catch`. #1012 made a failed `RELEASE` discard the whole @@ -230,9 +230,16 @@ describe('RemoteHandle', () => { it('reports the release failure rather than a missing savepoint', async () => { // The drivers' bookkeeping as #1012 leaves it, verified against both: a // failed RELEASE clears `_spStack`, and rolling back a name that is not on - // it throws `No such savepoint`. + // it throws `No such savepoint`. Replacing the store wholesale rather than + // assigning over its methods because `makeKernelStore` hardens what it + // returns. const savepoints: string[] = []; const releaseFailure = new Error('database or disk is full'); + const rollbackSavepoint = vi.fn((name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }); mockKernelStore = { ...mockKernelStore, createSavepoint: (name: string) => { @@ -242,12 +249,8 @@ describe('RemoteHandle', () => { savepoints.length = 0; throw releaseFailure; }, - rollbackSavepoint: (name: string) => { - if (!savepoints.includes(name)) { - throw new Error(`No such savepoint: ${name}`); - } - }, - } as KernelStore; + rollbackSavepoint, + }; const remote = makeRemote(); const delivery = JSON.stringify({ @@ -261,6 +264,10 @@ describe('RemoteHandle', () => { await expect(remote.handleRemoteMessage(delivery)).rejects.toBe( releaseFailure, ); + // Still attempted, so that abandoning the rollback is not a way to pass + // this test: a release that failed for a reason of its own may well have + // left the savepoint standing. + expect(rollbackSavepoint).toHaveBeenCalledWith('receive_r0_1'); }); // A dead run loop will never deliver the message, and `handleRemoteMessage` From afc664fdabc01629c19784cf8eb990712cf708cb Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:17 +0200 Subject: [PATCH 16/40] fix(kernel-store): clear `_inTx` before the COMMIT, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rollbackIfNeeded` was corrected for this ordering already; `commitIfNeeded` still stepped the COMMIT first. `_inTx` is tracked in the driver rather than read from SQLite, so a throwing COMMIT wedged it true: `beginIfNeeded` became a permanent no-op, and the next savepoint was created bare — where its RELEASE commits (Agoric/agoric-sdk#8423) and no later rollback could undo the delivery. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 4 ++-- packages/kernel-store/src/sqlite/wasm.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index fa0c284ba4..7a7df9aa85 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -14,8 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The rollback failure is still what gets thrown, even if aborting the transaction fails too - `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted. The nodejs driver reads `db.inTransaction` and was never affected +- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected - An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index a43092732b..a047d6cbcf 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -199,9 +199,13 @@ export async function makeSQLKernelDatabase({ */ function commitIfNeeded(): void { if (db._inTx && db._spStack.length === 0) { + // Cleared before the commit is attempted, for the reason `rollbackIfNeeded` + // gives: a throwing COMMIT would otherwise wedge `_inTx` true, and every + // later savepoint would be created bare — where its RELEASE commits + // (Agoric/agoric-sdk#8423) and no rollback can undo the delivery. + db._inTx = false; sqlCommitTransaction.step(); sqlCommitTransaction.reset(); - db._inTx = false; } } From 10b042a5733493bfd6484a97a5fa31d2a302c5ff Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:28 +0200 Subject: [PATCH 17/40] fix(ocap-kernel): stop `endCrank` burying the error that killed the run loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path, where it used to be a no-op. `#runLoop` called it from a bare `finally`, so a failing one silently replaced whatever killed the kernel — and only `error.message` crosses the wire, so the real failure reached neither `getStatus` nor the daemon log. Report it with the crank's failure as the `cause`, the shape the rollback path already uses. The in-flight error is boxed rather than left `undefined`, so a crank that threw `undefined` stays distinguishable from one that did not throw. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelQueue.ts | 33 ++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 358df83aef..a3fac5b778 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -122,6 +122,9 @@ export class KernelQueue { ): Promise { for (;;) { let wakeUpPromise: Promise | undefined; + // Boxed rather than left `undefined`, so that a crank which threw + // `undefined` is still distinguishable from one that did not throw. + let crankFailure: { error: unknown } | undefined; this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; @@ -170,8 +173,11 @@ export class KernelQueue { } throw error; } + } catch (error) { + crankFailure = { error }; + throw error; } finally { - this.#kernelStore.endCrank(); + this.#endCrank(crankFailure); if (wakeUpPromise) { await wakeUpPromise; } @@ -179,6 +185,31 @@ export class KernelQueue { } } + /** + * End the crank without losing the error that is already unwinding. Since the + * delivery rollback now spares `crank`, `endCrank`'s release is a real RELEASE + * and COMMIT on the dying path, where it used to be a no-op — and from a bare + * `finally` a failing one would silently replace whatever killed the kernel. + * + * @param crankFailure - The error already in flight, if the crank threw. + * @param crankFailure.error - That error. + */ + #endCrank(crankFailure?: { error: unknown }): void { + try { + this.#kernelStore.endCrank(); + } catch (endCrankError) { + if (!crankFailure) { + throw endCrankError; + } + // The original failure stays the `cause`, as on the rollback path; the + // release failure is named here. + throw new Error( + `Run loop died and its crank could not be ended: ${String(endCrankError)}`, + { cause: crankFailure.error }, + ); + } + } + /** * Record the death of the run loop and fail the kernel's own message-result * subscriptions, which would otherwise hang forever. Kernel promises in the From 54705540cddcf8d65b837a1c3ab078c746dbca69 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:40 +0200 Subject: [PATCH 18/40] fix(ocap-kernel): report the remote release failure, not a missing savepoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RemoteHandle.handleRemoteMessage` releases its savepoint inside the `try` and rolls back in the `catch`. Now that a failed RELEASE discards the whole savepoint stack, that rollback names a savepoint that is already gone and threw `No such savepoint` out of the `catch` in place of the database failure that brought it there — not even as `cause`. Log the rollback failure instead of throwing it. The rollback is still attempted, because a release that failed for a reason of its own may well have left the savepoint standing. `RemoteManager`'s `peerIncarnation_*` savepoint has the identical shape and had no coverage of it at all, so a fix applied here and forgotten there would have left its suite green. Fixed alike, and the savepoint-stack model both tests drive the drivers with is now shared. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/remotes/kernel/RemoteHandle.test.ts | 35 +++--------- .../src/remotes/kernel/RemoteHandle.ts | 14 ++++- .../src/remotes/kernel/RemoteManager.test.ts | 40 ++++++++++++++ .../src/remotes/kernel/RemoteManager.ts | 11 +++- packages/ocap-kernel/test/savepoint-stack.ts | 53 +++++++++++++++++++ 5 files changed, 123 insertions(+), 30 deletions(-) create mode 100644 packages/ocap-kernel/test/savepoint-stack.ts diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index cb0142e064..90f4ef3bc5 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { RemoteHandle } from './RemoteHandle.ts'; import { createMockRemotesFactory } from '../../../test/remotes-mocks.ts'; +import { withFailingSavepointRelease } from '../../../test/savepoint-stack.ts'; import type { KernelQueue } from '../../KernelQueue.ts'; import type { KernelStore } from '../../store/index.ts'; import { parseRef } from '../../store/utils/parse-ref.ts'; @@ -220,37 +221,15 @@ describe('RemoteHandle', () => { }); }); - // FAILING REPRO. - // // `handleRemoteMessage` releases its savepoint inside the `try` and rolls // back in the `catch`. #1012 made a failed `RELEASE` discard the whole - // savepoint stack, so that rollback now reports a savepoint that no longer - // exists, and it throws out of the `catch` in place of the failure that - // brought it there. + // savepoint stack, so that rollback reports a savepoint that no longer + // exists, and left unguarded it throws out of the `catch` in place of the + // failure that brought it there. it('reports the release failure rather than a missing savepoint', async () => { - // The drivers' bookkeeping as #1012 leaves it, verified against both: a - // failed RELEASE clears `_spStack`, and rolling back a name that is not on - // it throws `No such savepoint`. Replacing the store wholesale rather than - // assigning over its methods because `makeKernelStore` hardens what it - // returns. - const savepoints: string[] = []; - const releaseFailure = new Error('database or disk is full'); - const rollbackSavepoint = vi.fn((name: string) => { - if (!savepoints.includes(name)) { - throw new Error(`No such savepoint: ${name}`); - } - }); - mockKernelStore = { - ...mockKernelStore, - createSavepoint: (name: string) => { - savepoints.push(name); - }, - releaseSavepoint: () => { - savepoints.length = 0; - throw releaseFailure; - }, - rollbackSavepoint, - }; + const failing = withFailingSavepointRelease(mockKernelStore); + const { releaseFailure, rollbackSavepoint } = failing; + mockKernelStore = failing.kernelStore; const remote = makeRemote(); const delivery = JSON.stringify({ diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts index d92359b5c0..34181df661 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts @@ -1032,7 +1032,19 @@ export class RemoteHandle implements EndpointHandle { this.#kernelStore.releaseSavepoint(savepointName); } catch (error) { // Rollback on any error - in-memory state unchanged since we didn't update it yet - this.#kernelStore.rollbackSavepoint(savepointName); + try { + this.#kernelStore.rollbackSavepoint(savepointName); + } catch (rollbackError) { + // The release above is inside the `try`, and a failed RELEASE discards + // the whole savepoint stack — so this rollback reports a savepoint that + // is already gone, over the database failure an operator actually needs. + // Still attempted, because a release that failed for a reason of its own + // may well have left the savepoint standing. + this.#logger.error( + `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, + rollbackError, + ); + } throw error; } diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index 29ca0b2da7..c3257d85df 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -4,6 +4,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import * as remoteComms from './remote-comms.ts'; import { RemoteManager } from './RemoteManager.ts'; import { createMockRemotesFactory } from '../../../test/remotes-mocks.ts'; +import { withFailingSavepointRelease } from '../../../test/savepoint-stack.ts'; import { makeMapKernelDatabase } from '../../../test/storage.ts'; import type { KernelQueue } from '../../KernelQueue.ts'; import { makeKernelStore } from '../../store/index.ts'; @@ -948,5 +949,44 @@ describe('RemoteManager', () => { // mutations would otherwise drift from the rolled-back kv view. expect(finalizeSpy).not.toHaveBeenCalled(); }); + + // The same shape `RemoteHandle.handleRemoteMessage` has: the release sits + // inside the `try` and the rollback in the `catch`, so a failed RELEASE — + // which discards the whole savepoint stack — leaves the rollback naming a + // savepoint that is gone. + it('reports the release failure rather than a missing savepoint', async () => { + const peerId = 'peer-whose-release-fails'; + const { + kernelStore: failingStore, + releaseFailure, + rollbackSavepoint, + } = withFailingSavepointRelease(kernelStore); + remoteManager = new RemoteManager({ + platformServices: mockPlatformServices, + kernelStore: failingStore, + kernelQueue: mockKernelQueue, + logger, + }); + remoteManager.setMessageHandler(vi.fn()); + await remoteManager.initRemoteComms(); + const onIncarnationChange = vi + .mocked(remoteComms.initRemoteComms) + .mock.calls.at(-1)?.[8] as ( + peerId: string, + observedIncarnation: string, + ) => Promise; + + // The error an operator needs is the one the database gave, not the + // bookkeeping artefact of trying to clean up after it. + await expect(onIncarnationChange(peerId, 'incarnation-A')).rejects.toBe( + releaseFailure, + ); + // Still attempted, so that abandoning the rollback is not a way to pass: + // a release that failed for a reason of its own may well have left the + // savepoint standing. + expect(rollbackSavepoint).toHaveBeenCalledWith( + `peerIncarnation_${peerId}`, + ); + }); }); }); diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts index d9b7c3c946..9a477cf398 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts @@ -261,7 +261,16 @@ export class RemoteManager { this.#kernelStore.setPeerIncarnation(peerId, observedIncarnation); this.#kernelStore.releaseSavepoint(savepoint); } catch (error) { - this.#kernelStore.rollbackSavepoint(savepoint); + try { + this.#kernelStore.rollbackSavepoint(savepoint); + } catch (rollbackError) { + // The release above is inside the `try`, and a failed RELEASE discards + // the whole savepoint stack — so this rollback reports a savepoint that + // is already gone, over the database failure an operator actually needs. + // Still attempted, because a release that failed for a reason of its own + // may well have left the savepoint standing. + this.#logger?.error(`Rollback of ${savepoint} failed`, rollbackError); + } throw error; } diff --git a/packages/ocap-kernel/test/savepoint-stack.ts b/packages/ocap-kernel/test/savepoint-stack.ts new file mode 100644 index 0000000000..c4b7e587fc --- /dev/null +++ b/packages/ocap-kernel/test/savepoint-stack.ts @@ -0,0 +1,53 @@ +import type { MockedFunction } from 'vitest'; +import { vi } from 'vitest'; + +import type { KernelStore } from '../src/store/index.ts'; + +export type FailingReleaseStore = { + /** The store to hand the subject under test. */ + kernelStore: KernelStore; + /** The error every `releaseSavepoint` throws. */ + releaseFailure: Error; + /** Exposed so a test can assert the rollback was still attempted. */ + rollbackSavepoint: MockedFunction<(name: string) => void>; +}; + +/** + * Wrap a kernel store so that `releaseSavepoint` fails the way a full disk does, + * modelling the drivers' bookkeeping as of #1012 and verified against both: a + * failed RELEASE clears the savepoint stack, and rolling back a name that is no + * longer on it throws `No such savepoint`. What is under test is therefore the + * caller's error handling, not an expected outcome baked into the mock. + * + * Replaces the store wholesale rather than assigning over its methods, because + * `makeKernelStore` hardens what it returns. + * + * @param kernelStore - The store to wrap. + * @returns The wrapped store and the handles a test needs to assert against. + */ +export function withFailingSavepointRelease( + kernelStore: KernelStore, +): FailingReleaseStore { + const savepoints: string[] = []; + const releaseFailure = new Error('database or disk is full'); + const rollbackSavepoint = vi.fn((name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }); + return { + kernelStore: { + ...kernelStore, + createSavepoint: (name: string) => { + savepoints.push(name); + }, + releaseSavepoint: () => { + savepoints.length = 0; + throw releaseFailure; + }, + rollbackSavepoint, + }, + releaseFailure, + rollbackSavepoint, + }; +} From 726d06304cb1ff09645600e42f7ee5c6218bf029 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:54 +0200 Subject: [PATCH 19/40] test(ocap-kernel): pin the in-memory revert against a failed crank rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rollbackCrank` gained two pieces of work that compose the wrong way round if the failure path simply rethrows: forgetting every savepoint, and reverting the caches a database rollback cannot reach. A failed rollback discards the whole transaction, so the database has moved back at least as far as a successful rollback would have taken it and those caches are at least as stale — the one case where skipping the revert leaves the consumed GC action lost and krefs queued for a collection that then kills the run loop. The second test pins the other direction: reverting must not become a way to lose the database error either. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 4 ++ .../src/store/methods/crank.test.ts | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index d24ebf8de9..48b63bd639 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -73,6 +73,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path where it used to be a no-op — and the run loop called it from a bare `finally` +- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Both release inside the `try` and roll back in the `catch`, so once a failed RELEASE discarded the whole savepoint stack, the rollback reported a savepoint that no longer existed in place of the real error. The rollback is still attempted: a release that failed for a reason of its own may well have left the savepoint standing - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel - `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 630444e850..426890d209 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -176,6 +176,48 @@ describe('crank methods', () => { crankMethods.endCrank(); expect(kdb.releaseSavepoint).not.toHaveBeenCalled(); }); + + // The two halves of this function compose the wrong way round if the + // failure path simply rethrows: a failed rollback discards the whole + // transaction, so the database has moved back at least as far as a + // successful rollback would have taken it and the caches it left behind are + // at least as stale. + it('reverts the caches the database cannot reach even when the rollback fails', () => { + context.inCrank = true; + context.maybeFreeKrefs.add('kp1'); + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + 'disk I/O error', + ); + + expect(context.refreshCachedValues).toHaveBeenCalled(); + expect(context.refreshRunQueue).toHaveBeenCalled(); + expect(context.runQueueLengthCache).toBe(-1); + expect([...context.maybeFreeKrefs]).toStrictEqual([]); + }); + + // Reverting must not become a way to lose the database error either. + it('keeps the rollback failure as the cause when reverting also fails', () => { + context.inCrank = true; + const rollbackFailure = new Error('disk I/O error'); + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw rollbackFailure; + }); + vi.mocked(context.refreshCachedValues).mockImplementationOnce(() => { + throw new Error('database is gone'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + expect.objectContaining({ cause: rollbackFailure }), + ); + }); }); describe('endCrank', () => { From daae217261f4a3e8cff45c3614462a3d15e0c42f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:02:03 +0200 Subject: [PATCH 20/40] fix(runtimes): give the kernel store a logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No production call site passed one, so every `logger?.` call in the SQLite drivers was dead code — including the aborts they report while discarding a transaction, which fire on exactly the path where the kernel is already dying and a diagnostic is worth most. The browser worker has a module-level `Logger` already, so both drivers are covered rather than only the nodejs one. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 1 + .../src/kernel-worker/kernel-worker.ts | 5 ++++- packages/kernel-node-runtime/CHANGELOG.md | 1 + packages/kernel-node-runtime/src/kernel/make-kernel.ts | 5 ++++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index 473a868c64..be2534ff9d 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts b/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts index 832a57a906..afffa24e32 100644 --- a/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts +++ b/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts @@ -44,7 +44,10 @@ async function main(): Promise { isJsonRpcMessage, ), PlatformServicesClient.make(globalThis as PostMessageTarget), - makeSQLKernelDatabase({ dbFilename: DB_FILENAME }), + makeSQLKernelDatabase({ + dbFilename: DB_FILENAME, + logger: logger.subLogger({ tags: ['kernel-store'] }), + }), ]); setupConsoleForwarding({ diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index 51b675e9d4..9f85296e5e 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.ts index 901e79982a..529eb7af2d 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.ts @@ -60,7 +60,10 @@ export async function makeKernel({ }); // Initialize kernel store. - const kernelDatabase = await makeSQLKernelDatabase({ dbFilename }); + const kernelDatabase = await makeSQLKernelDatabase({ + dbFilename, + logger: rootLogger.subLogger({ tags: ['kernel-store'] }), + }); // Create and start kernel. const kernel = await Kernel.make(platformServicesClient, kernelDatabase, { From 8e0676b1fd508be6cf58724937616ab39a9527c2 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:28:46 +0200 Subject: [PATCH 21/40] chore: cite this PR in the changelogs The crank-transaction and rollback work landed here rather than in #1012, which is closed and replaced. #1021 is a placeholder until the PR exists. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 2 +- packages/kernel-node-runtime/CHANGELOG.md | 2 +- packages/kernel-store/CHANGELOG.md | 6 +++--- packages/ocap-kernel/CHANGELOG.md | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index be2534ff9d..be6e9bbf51 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index 9f85296e5e..a2ba64ef1b 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 7a7df9aa85..fd82e2c1e8 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,11 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected -- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 48b63bd639..ffff6c1ec6 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -70,16 +70,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- Keep a crank's store work inside one transaction ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately -- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path where it used to be a no-op — and the run loop called it from a bare `finally` -- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Both release inside the `try` and roll back in the `catch`, so once a failed RELEASE discarded the whole savepoint stack, the rollback reported a savepoint that no longer existed in place of the real error. The rollback is still attempted: a release that failed for a reason of its own may well have left the savepoint standing -- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel -- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A cached stored value answers reads from a closure and only writes through to the database, so reverting the database left it holding the abandoned crank's value and the next write persisted that. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright instead of retrying it - `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop - A failed rollback discards the whole transaction, moving the database back at least as far as a successful rollback would have — so reverting only on success left exactly the state that is least able to tolerate it From 2724dff1baafdedcea3eed408dce59b3a4d28c70 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 20 Aug 2026 16:10:44 +0200 Subject: [PATCH 22/40] test(kernel-store): pin the transaction a failed COMMIT abandons `commitIfNeeded` clears `_inTx` before stepping the COMMIT, so by the time the failure reaches `safeMutate` the flag it recovers on is already false and `rollbackIfNeeded` no-ops. A COMMIT can fail with the transaction still open (classically `SQLITE_BUSY`), and nothing is then left to end it, so every later write on the connection joins it, reports success, and is lost at close. Both assertions count steps rather than naming the statement, because every prepared statement in this file shares one mock. Also replaces the stale `FAILING REPRO` marker on the neighbouring test, whose comment still described the ordering bug fixed in 86d16dc49 as present, with a pointer to the rationale in `commitIfNeeded`. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/src/sqlite/wasm.test.ts | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 1cf496dc95..69385ae94c 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -599,19 +599,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); - // FAILING REPRO. - // - // `commitIfNeeded` still steps the COMMIT before clearing `_inTx`, the exact - // ordering `rollbackIfNeeded` was corrected to avoid. A COMMIT that throws - // therefore leaves `_inTx` true against a database that may hold no - // transaction, `beginIfNeeded` is a no-op forever after, and the next - // savepoint is created outside a transaction — which commits when released - // (Agoric/agoric-sdk#8423). This is the crank's commit point, so the writes - // that leak are a whole crank's. - // - // A failed abort is therefore not, as the abort case above claims, the one - // case that can leave `_inTx` disagreeing with the database. This is the - // second. + // Why the ordering matters: see `commitIfNeeded`. it('stops believing it is in a transaction when the commit fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -637,6 +625,58 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); + // Why the COMMIT path discards too: see `commitIfNeeded`. + it('releaseSavepoint discards the transaction when the commit fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockStatement.step.mockClear(); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + // The RELEASE goes through `exec` and succeeds; COMMIT is the first + // prepared statement this path steps, and it is what fails. + mockStatement.step.mockImplementationOnce(() => { + throw new Error('database is locked'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'database is locked', + ); + + // Every statement shares one mock, so the ABORT is only visible as a + // second step. + expect(mockStatement.step).toHaveBeenCalledTimes(2); + expect(mockDb._inTx).toBe(false); + }); + + it('releaseSavepoint reports the commit failure even if the abort fails too', async () => { + const logger = { + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + subLogger: vi.fn(() => logger), + } as unknown as Logger; + const db = await makeSQLKernelDatabase({ logger }); + mockStatement.step.mockClear(); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockStatement.step + .mockImplementationOnce(() => { + throw new Error('database is locked'); + }) + .mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'database is locked', + ); + + expect(logger.error).toHaveBeenCalledWith( + 'failed to discard transaction after commit', + expect.objectContaining({ message: 'cannot rollback' }), + ); + expect(mockDb._inTx).toBe(false); + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); From eab46813a365fb46662abd20f59f7f62bf53482b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 20 Aug 2026 16:13:03 +0200 Subject: [PATCH 23/40] fix(kernel-store): discard the transaction a failed COMMIT leaves open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 86d16dc49 cleared `_inTx` before stepping the COMMIT so a throwing one could not wedge the flag true. That fixed the wedge but left nothing to end the transaction: SQLite can fail a COMMIT with it still open, and `safeMutate`'s recovery calls `rollbackIfNeeded`, which reads the now-false flag and no-ops. `releaseSavepoint` reaches `commitIfNeeded` outside any try/catch at all. So the transaction stayed open with no owner, took every later write on the connection, reported success, and lost them at `close()` — the same hazard `rollbackSavepoint` and `releaseSavepoint` already discard the transaction to avoid, by a third door. `commitIfNeeded` now does the same. Reported by Cursor Bugbot. The nodejs driver reads `db.inTransaction` from SQLite rather than caching it, so it has no equivalent gap; that asymmetry is #1013. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 2 +- packages/kernel-store/src/sqlite/wasm.ts | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index fd82e2c1e8..1a92cfca56 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The rollback failure is still what gets thrown, even if aborting the transaction fails too - `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The wasm driver clears `_inTx` when aborting or committing a transaction throws, and discards the transaction that a failed COMMIT can leave open ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected - An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index a047d6cbcf..534184fd38 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -204,8 +204,24 @@ export async function makeSQLKernelDatabase({ // later savepoint would be created bare — where its RELEASE commits // (Agoric/agoric-sdk#8423) and no rollback can undo the delivery. db._inTx = false; - sqlCommitTransaction.step(); - sqlCommitTransaction.reset(); + try { + sqlCommitTransaction.step(); + sqlCommitTransaction.reset(); + } catch (error) { + // Discarded for the reason `releaseSavepoint` gives: a failed COMMIT can + // leave the transaction open too. Stepped directly because `_inTx` is + // already false above, which makes `rollbackIfNeeded` a no-op. + try { + sqlAbortTransaction.step(); + sqlAbortTransaction.reset(); + } catch (abortError) { + logger?.error( + 'failed to discard transaction after commit', + abortError, + ); + } + throw error; + } } } From cf07c281d575f5c0c8dacdd2b81b5c488360ded1 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 27 Aug 2026 13:23:14 +0200 Subject: [PATCH 24/40] docs: trim changelog entries and comments The changelog entries for this PR explained mechanism where a consumer only needs effect; the comments repeated what the code says or said it twice across duplicated call sites. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 2 +- packages/kernel-node-runtime/CHANGELOG.md | 2 +- packages/kernel-store/CHANGELOG.md | 9 ++-- packages/kernel-store/src/sqlite/nodejs.ts | 13 +----- packages/kernel-store/src/sqlite/wasm.ts | 32 ++++---------- packages/ocap-kernel/CHANGELOG.md | 15 ++----- packages/ocap-kernel/src/KernelQueue.ts | 43 ++++++------------- .../src/remotes/kernel/RemoteHandle.ts | 7 +-- .../src/remotes/kernel/RemoteManager.ts | 6 +-- packages/ocap-kernel/src/store/index.ts | 20 +++------ .../ocap-kernel/src/store/methods/crank.ts | 39 ++--------------- 11 files changed, 45 insertions(+), 143 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index be6e9bbf51..bd09c572be 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index a2ba64ef1b..643a7042d1 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 1a92cfca56..a2504faf83 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,11 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting or committing a transaction throws, and discards the transaction that a failed COMMIT can leave open ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected -- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO`; the release failure is still what gets thrown ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The wasm driver no longer considers itself in a transaction after a commit or abort throws, and discards the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - Otherwise later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work. The nodejs driver reads `db.inTransaction` and was never affected +- Both drivers log an abort that fails while recovering from a failed savepoint operation ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index fa754bb88f..d841997142 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -299,9 +299,6 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The rollback failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction the next crank would - // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -330,18 +327,12 @@ export async function makeSQLKernelDatabase({ try { db.exec(query); } catch (error) { - // The hazard `rollbackSavepoint` guards against, by the other door: left as - // it was, the savepoint stays on the stack and the transaction open with - // nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. There is no - // committing this transaction now, so discard it. + // The hazard `rollbackSavepoint` guards against, by the other door, and + // there is no committing this transaction now. db._spStack.length = 0; try { rollbackIfNeeded(); } catch (abortError) { - // The release failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction the next crank would - // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 534184fd38..55c7594e2f 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -199,18 +199,14 @@ export async function makeSQLKernelDatabase({ */ function commitIfNeeded(): void { if (db._inTx && db._spStack.length === 0) { - // Cleared before the commit is attempted, for the reason `rollbackIfNeeded` - // gives: a throwing COMMIT would otherwise wedge `_inTx` true, and every - // later savepoint would be created bare — where its RELEASE commits - // (Agoric/agoric-sdk#8423) and no rollback can undo the delivery. + // Cleared first, for the reason `rollbackIfNeeded` gives. db._inTx = false; try { sqlCommitTransaction.step(); sqlCommitTransaction.reset(); } catch (error) { - // Discarded for the reason `releaseSavepoint` gives: a failed COMMIT can - // leave the transaction open too. Stepped directly because `_inTx` is - // already false above, which makes `rollbackIfNeeded` a no-op. + // A failed COMMIT can leave the transaction open. Stepped directly + // because `_inTx` above already made `rollbackIfNeeded` a no-op. try { sqlAbortTransaction.step(); sqlAbortTransaction.reset(); @@ -230,12 +226,9 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - // Cleared before the abort is attempted, because the abort can throw and - // `_inTx` is tracked here rather than read from SQLite as the nodejs driver - // does. Left true, `beginIfNeeded` is a no-op forever after and writes - // autocommit one statement at a time (see `createSavepoint`). Cleared, a - // still-open transaction surfaces as a failed `BEGIN` — the louder - // failure. + // Cleared before the abort, which can throw: left true, `beginIfNeeded` is + // a no-op forever after and writes autocommit one statement at a time (see + // `createSavepoint`). db._inTx = false; db._spStack.length = 0; sqlAbortTransaction.step(); @@ -407,9 +400,6 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The rollback failure below is the one worth reporting. The next - // `BEGIN` will fail if SQLite really is still in a transaction, but that - // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -438,18 +428,12 @@ export async function makeSQLKernelDatabase({ try { db.exec(query); } catch (error) { - // The hazard `rollbackSavepoint` guards against, by the other door: left as - // it was, the savepoint stays on the stack and the transaction open with - // nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. There is no - // committing this transaction now, so discard it. + // The hazard `rollbackSavepoint` guards against, by the other door, and + // there is no committing this transaction now. db._spStack.length = 0; try { rollbackIfNeeded(); } catch (abortError) { - // The release failure below is the one worth reporting. The next - // `BEGIN` will fail if SQLite really is still in a transaction, but that - // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index ffff6c1ec6..c5d80480b5 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -70,19 +70,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time +- Keep a crank's store work inside one transaction, so terminating the vat and collecting garbage after an aborted delivery are committed with the crank instead of autocommitting a statement at a time ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately -- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path where it used to be a no-op — and the run loop called it from a bare `finally` +- `rollbackCrank` now also reverts the state a database rollback cannot reach, whether or not the rollback itself succeeded: cached stored values are re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - Otherwise a cache kept the abandoned crank's value and the next write persisted it, a GC action taken out of the set for an aborted delivery was lost rather than retried, and a later `collectGarbage` killed the run loop on a promise the rollback had deleted +- Cleanup after a dead crank no longer replaces the error that killed the run loop: a failing `endCrank` reports it as `cause`, and neither `endCrank` nor `rollbackCrank` retries a savepoint that a failed rollback or release has already discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - Both release inside the `try` and roll back in the `catch`, so once a failed RELEASE discarded the whole savepoint stack, the rollback reported a savepoint that no longer existed in place of the real error. The rollback is still attempted: a release that failed for a reason of its own may well have left the savepoint standing -- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel -- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - A cached stored value answers reads from a closure and only writes through to the database, so reverting the database left it holding the abandoned crank's value and the next write persisted that. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright instead of retrying it - - `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop - - A failed rollback discards the whole transaction, moving the database back at least as far as a successful rollback would have — so reverting only on success left exactly the state that is least able to tolerate it - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index a3fac5b778..4e8def6e5b 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -56,10 +56,8 @@ export class KernelQueue { /** * Whether this crank's savepoint has already been handed to `rollbackCrank`. - * Attempted, not necessarily succeeded: it is forgotten either way, so a second - * attempt could only report "no such savepoint" over the real error. Recorded - * at the attempt rather than returned, because `#processCrankResult` can throw - * after rolling back. + * Attempted, not necessarily succeeded: it is forgotten either way, so asking + * twice could only report "no such savepoint" over the real error. */ #crankRollbackAttempted: boolean = false; @@ -122,8 +120,8 @@ export class KernelQueue { ): Promise { for (;;) { let wakeUpPromise: Promise | undefined; - // Boxed rather than left `undefined`, so that a crank which threw - // `undefined` is still distinguishable from one that did not throw. + // Boxed, so a crank that threw `undefined` stays distinguishable from one + // that did not throw. let crankFailure: { error: unknown } | undefined; this.#kernelStore.startCrank(); @@ -133,7 +131,6 @@ export class KernelQueue { // enclosing transaction (see `rollbackSavepoint`) and an aborted crank // still has writes to make. Only `delivery` is ever rolled back; // releasing `crank` in `endCrank` is this crank's one commit point. - // `releaseAllSavepoints` names it by ordinal, so `crank` must stay first. this.#kernelStore.createCrankSavepoint('crank'); this.#kernelStore.createCrankSavepoint('delivery'); @@ -163,8 +160,6 @@ export class KernelQueue { try { this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { - // The original failure stays the `cause`; the rollback failure is - // named here. throw new Error( `Run loop died and its crank could not be rolled back: ${String(rollbackError)}`, { cause: error }, @@ -186,10 +181,9 @@ export class KernelQueue { } /** - * End the crank without losing the error that is already unwinding. Since the - * delivery rollback now spares `crank`, `endCrank`'s release is a real RELEASE - * and COMMIT on the dying path, where it used to be a no-op — and from a bare - * `finally` a failing one would silently replace whatever killed the kernel. + * End the crank without losing the error that is already unwinding. Now that + * the delivery rollback spares `crank`, `endCrank` is a real RELEASE and + * COMMIT on the dying path where it used to be a no-op. * * @param crankFailure - The error already in flight, if the crank threw. * @param crankFailure.error - That error. @@ -201,8 +195,6 @@ export class KernelQueue { if (!crankFailure) { throw endCrankError; } - // The original failure stays the `cause`, as on the rollback path; the - // release failure is named here. throw new Error( `Run loop died and its crank could not be ended: ${String(endCrankError)}`, { cause: crankFailure.error }, @@ -337,9 +329,7 @@ export class KernelQueue { try { this.#kernelStore.rollbackCrank('delivery'); } finally { - // Set even when the rollback threw: the savepoint is gone either way, so - // a second attempt would report a missing savepoint as the reason the - // kernel died, burying the error that actually killed it. + // Set even when the rollback threw: the savepoint is gone either way. this.#crankRollbackAttempted = true; } // Discard kernel subscriptions that were queued for invocation @@ -363,28 +353,21 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // This call kills the worker, so its writes must outlive the rollback above: - // a store that still believed the vat was alive would relaunch it after a - // restart and redeliver what killed it. + // This kills the worker, so its writes must outlive the rollback above: a + // store that still believed the vat was alive would relaunch it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { - // After the fallible work above, not before it. The flush settles the + // After the fallible work above, not before it: the flush settles the // promise `enqueueMessage` gave an external caller, so a later rollback // would discard the state that answer was computed from. - // - // Not airtight: `#terminateVat` resolves the dying vat's promises through - // `resolvePromises`, which defaults to `immediate` and invokes their - // subscriptions before `collectGarbage`. Deferring those too would change - // termination semantics, not crank ordering. this.#flushCrankBuffer(); } - // After the flush, because the audit reads the run queue as ground truth - // while a buffered item's references were already counted when it was - // enqueued: audited mid-flush, every buffered item reads as a leak. + // After the flush: a buffered item's references were counted when it was + // enqueued, so audited mid-flush every one of them reads as a leak. this.#kernelStore.assertRefCountsIfAuditing(); } diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts index 34181df661..ddb6d0d64a 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts @@ -1035,11 +1035,8 @@ export class RemoteHandle implements EndpointHandle { try { this.#kernelStore.rollbackSavepoint(savepointName); } catch (rollbackError) { - // The release above is inside the `try`, and a failed RELEASE discards - // the whole savepoint stack — so this rollback reports a savepoint that - // is already gone, over the database failure an operator actually needs. - // Still attempted, because a release that failed for a reason of its own - // may well have left the savepoint standing. + // A failed RELEASE above discards the whole savepoint stack, so this + // rollback would report a savepoint already gone over the real error. this.#logger.error( `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, rollbackError, diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts index 9a477cf398..2ac1aa5aca 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts @@ -264,11 +264,7 @@ export class RemoteManager { try { this.#kernelStore.rollbackSavepoint(savepoint); } catch (rollbackError) { - // The release above is inside the `try`, and a failed RELEASE discards - // the whole savepoint stack — so this rollback reports a savepoint that - // is already gone, over the database failure an operator actually needs. - // Still attempted, because a release that failed for a reason of its own - // may well have left the savepoint standing. + // Same reasoning as `RemoteHandle.handleRemoteMessage`. this.#logger?.error(`Rollback of ${savepoint} failed`, rollbackError); } throw error; diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index f2257af545..0960bbbe43 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -130,7 +130,7 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { /** * Every cached stored value the context holds, as `field: [key, initial]`. * Declared once so that initialization and `refreshCachedValues` cannot - * disagree about which values exist: adding one here does both. + * disagree about which values exist. */ const CACHED_VALUES = { /** Counter for allocating kernel object IDs */ @@ -152,8 +152,7 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { } as const satisfies Record; /** - * Provide a fresh stored value for each of {@link CACHED_VALUES}, reading its - * current setting out of the database. + * Provide a fresh stored value for each of {@link CACHED_VALUES}. * * @returns The stored values, keyed by the context field that holds each. */ @@ -181,10 +180,10 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { }, ...provideCachedValues(), /** - * Re-read every cached stored value from the database. Each one closes over - * the last value written through it (see `provideCachedStoredValue`), so - * reverting the database alone is not enough: the closure would still hold - * the abandoned value and the next `set` would write it straight back. + * Re-read every cached stored value from the database. Each closes over the + * last value written through it (see `provideCachedStoredValue`), so after a + * rollback the closure would otherwise still hold the abandoned value and + * the next `set` would write it straight back. */ refreshCachedValues: () => { Object.assign(context, provideCachedValues()); @@ -434,9 +433,6 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { return Number(kv.get(`${OCAP_URL_PREFIX}${kref}`) ?? 0); }, retainForOcapURL(kref: KRef): void { - // Refuse to mint a token for something already collected: the URL would - // name a capability that can never be delivered to. Refusing here names - // what was attempted, and does it before anything has been written. this.kernelRefExists(kref) || Fail`cannot issue an ocap URL for deleted kref ${kref}`; const issuances = this.getOcapURLIssuanceCount(kref); @@ -451,16 +447,12 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { kv.set(`${OCAP_URL_PREFIX}${kref}`, `${issuances - 1}`); return; } - // The last issuance, or none at all: either way, what is left of the - // retention is exactly what a release drops. this.releaseOcapURLRetentions(kref); }, releaseOcapURLRetentions(kref: KRef): void { if (this.getOcapURLIssuanceCount(kref) === 0) { return; } - // Spend the pin before forgetting the retention, so a failure to release - // it leaves a ledger that still says the retention is held. this.unpinObject(kref); kv.delete(`${OCAP_URL_PREFIX}${kref}`); }, diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index a3bba1956b..df4b02a665 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -32,9 +32,6 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { function createCrankSavepoint(name: string): void { ctx.inCrank || Fail`createCrankSavepoint outside of crank`; const ordinal = ctx.savepoints.length; - // Record the name only once the database has the savepoint. Recording it - // first would leave `endCrank` trying to release a savepoint that was never - // created, and that error would replace whatever really went wrong. kdb.createSavepoint(`t${ordinal}`); ctx.savepoints.push(name); } @@ -51,21 +48,9 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { if (ctx.savepoints[ordinal] === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); - // Left listed, `endCrank`'s release would commit the crank we just - // abandoned. ctx.savepoints.length = ordinal; } catch (error) { - // A failed rollback discards the whole transaction, so every savepoint - // is gone, not just this one. Truncating to `ordinal` would have - // `endCrank` release a `t0` the database lacks and throw over whatever - // really killed the kernel. ctx.savepoints.length = 0; - // Before the rethrow, and not only on the path below. A failed - // rollback discards the whole transaction, so the database has moved - // back at least as far as a successful rollback would have taken it - // and these caches are at least as stale. Rethrowing ahead of this - // would leave the dying crank holding the GC action it consumed and - // the freed krefs it was about to collect. revertStateBeneathRollback(error); throw error; } @@ -80,29 +65,15 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { * Revert what a database rollback cannot reach: the in-memory caches built * over the abandoned crank's writes. * - * @param rollbackError - The error the rollback threw, if it threw. Kept as - * the `cause` should reverting fail too, since it is the root cause an - * operator needs. + * @param rollbackError - The error the rollback threw, if it threw. */ function revertStateBeneathRollback(rollbackError?: unknown): void { try { - // Recreate the run queue so its cached head/tail are re-read from the - // database, and invalidate the length cache, since the rollback may have - // restored dequeued items. ctx.refreshRunQueue(); ctx.runQueueLengthCache = -1; - // Same staleness, worse consequence: a cached value reads from its - // closure and only writes through to kv, so one this crank consumed stays - // consumed and the next `set` persists that. `processGCActionSet` takes an - // action out of the set before delivering it, so an action not restored - // here is lost rather than retried. ctx.refreshCachedValues(); - // Nothing rolls back RAM. These krefs are collection candidates only - // because this crank decremented them, and that is precisely what was just - // undone. Left in place, `collectGarbage` throws on a later crank for any - // promise this one created — killing the run loop over work that no longer - // exists. Correct only while every rollback discards the whole delivery, - // which is all any caller asks for. + // Clearing all of them is correct only while a rollback discards the whole + // delivery, which is all any caller asks for. ctx.maybeFreeKrefs.clear(); } catch (revertError) { if (rollbackError === undefined) { @@ -123,10 +94,6 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { try { kdb.releaseSavepoint('t0'); } finally { - // A failed release discards the transaction too, so the database has no - // savepoints left either. Left listed, the next crank would number its - // savepoint `t1` and aim every later release and rollback one crank past - // the one it meant to end. ctx.savepoints.length = 0; } } From d8318d5108f5427ac67474777da85de9f9698c9d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 27 Aug 2026 13:36:15 +0200 Subject: [PATCH 25/40] docs: cut the leftover narration from this branch's tests Stale TDD markers ("FAILING REPRO") that now describe bugs the branch fixed, and per-test preambles restating what the test title and the source comment already say. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/kernel/make-kernel.test.ts | 9 ---- .../kernel-store/src/sqlite/nodejs.test.ts | 2 - packages/kernel-store/src/sqlite/wasm.test.ts | 18 ++----- .../kernel-test/src/crank-rollback.test.ts | 50 +++---------------- packages/ocap-kernel/src/KernelQueue.test.ts | 17 ------- .../src/remotes/kernel/RemoteHandle.test.ts | 11 +--- .../src/remotes/kernel/RemoteManager.test.ts | 10 +--- .../src/store/methods/crank.test.ts | 13 ----- 8 files changed, 13 insertions(+), 117 deletions(-) diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts index 672313a02a..eb80137e45 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts @@ -10,8 +10,6 @@ vi.mock('@metamask/kernel-store/sqlite/nodejs', async () => { '../../../ocap-kernel/test/storage.ts' ); return { - // Wrapped so that a test can see what the database was constructed with, - // while still getting a real store back. makeSQLKernelDatabase: vi.fn(makeMapKernelDatabase), }; }); @@ -23,13 +21,6 @@ describe('makeKernel', () => { expect(kernel).toBeInstanceOf(Kernel); }); - // FAILING REPRO. - // - // The kernel store is the only collaborator `makeKernel` builds without - // handing it a logger, so every `logger?.` call inside the SQLite driver is - // dead code in production — including the four abort failures #1012 added - // logging for. `kernel-worker.ts` omits it too, which keeps the wasm driver's - // pair dead even once this passes. it('gives the kernel store a logger', async () => { await makeKernel({}); diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index 270f018ae6..189fe0cf16 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -308,7 +308,6 @@ describe('makeSQLKernelDatabase', () => { ); expect(mockDb._spStack).toStrictEqual([]); - // The abort is the only prepared statement this path runs. expect(mockStatement.run).toHaveBeenCalledOnce(); mockDb.inTransaction = false; }); @@ -360,7 +359,6 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._spStack).toStrictEqual([]); }); - // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb.inTransaction = true; diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 69385ae94c..f67455e272 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -517,7 +517,6 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -552,8 +551,6 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // A failed abort is the one case that can leave `_inTx` disagreeing with the - // database. Left true, `beginIfNeeded` is a no-op forever after. it('stops believing it is in a transaction when the abort fails too', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -572,9 +569,6 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // Why that matters: a savepoint created outside a transaction commits when - // released (Agoric/agoric-sdk#8423), so an aborted crank would keep its - // writes. it('begins a transaction for the next savepoint after a failed abort', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -599,13 +593,12 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); - // Why the ordering matters: see `commitIfNeeded`. it('stops believing it is in a transaction when the commit fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; mockDb._spStack = ['point1']; - // The RELEASE goes through `exec` and succeeds; COMMIT is the first - // prepared statement this path steps, and it is what fails. + // The RELEASE goes through `exec`; COMMIT is the first prepared statement + // this path steps. mockStatement.step.mockImplementationOnce(() => { throw new Error('disk I/O error'); }); @@ -616,8 +609,6 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); - // And so the next savepoint gets a transaction of its own rather than - // being created bare. mockDb.exec.mockClear(); mockStatement.step.mockClear(); db.createSavepoint('next'); @@ -625,14 +616,13 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); - // Why the COMMIT path discards too: see `commitIfNeeded`. it('releaseSavepoint discards the transaction when the commit fails', async () => { const db = await makeSQLKernelDatabase({}); mockStatement.step.mockClear(); mockDb._inTx = true; mockDb._spStack = ['point1']; - // The RELEASE goes through `exec` and succeeds; COMMIT is the first - // prepared statement this path steps, and it is what fails. + // The RELEASE goes through `exec`; COMMIT is the first prepared statement + // this path steps. mockStatement.step.mockImplementationOnce(() => { throw new Error('database is locked'); }); diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index 36131a8a16..cc71c5cbc5 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -5,12 +5,9 @@ import type { RunQueueItem } from '@metamask/ocap-kernel'; import { describe, it, expect } from 'vitest'; /** - * The run loop rolls back the crank it died in, so that a restart resumes from a - * consistent boundary rather than from a half-finished crank. `KernelQueue`'s own - * tests mock the store, so they prove only that `rollbackCrank` is *called* - * correctly. These exercise what it actually does against real SQLite: the - * savepoint, the run queue and its length cache, and the release that `endCrank` - * performs afterwards. + * `KernelQueue`'s own tests mock the store, so they prove only that + * `rollbackCrank` is *called* correctly. These exercise what it does against + * real SQLite. */ /** @@ -34,9 +31,6 @@ const makeItem = (target: string): RunQueueItem => }) as unknown as RunQueueItem; describe('crank rollback against a real database', () => { - // The claim the changelog makes: because the killing crank is rolled back, the - // item it dequeued is still there to be re-dequeued after a restart. With the - // store mocked this is unobservable. it('returns the dequeued item to the run queue', async () => { const { kernelStore } = await makeStore(); kernelStore.enqueueRun(makeItem('ko1')); @@ -54,9 +48,6 @@ describe('crank rollback against a real database', () => { expect(kernelStore.dequeueRun()).toStrictEqual(dequeued); }); - // `rollbackCrank` invalidates the length cache precisely because the rollback - // restored rows the cache no longer knows about. Reading the length *before* - // the rollback primes that cache, which is what makes the invalidation matter. it('recomputes the run queue length after a rollback', async () => { const { kernelStore } = await makeStore(); kernelStore.enqueueRun(makeItem('ko1')); @@ -75,10 +66,6 @@ describe('crank rollback against a real database', () => { expect(kernelStore.runQueueLength()).toBe(2); }); - // `endCrank` releases savepoints unconditionally, and releasing the savepoint a - // rollback abandoned would commit the crank being discarded. It cannot, because - // `rollbackCrank` forgets the savepoint — but that reasoning is about SQLite's - // savepoint stack, so it is worth pinning against a real one. it('does not commit the abandoned crank when endCrank releases afterwards', async () => { const { kernelStore, kdb } = await makeStore(); kdb.kernelKVStore.set('before', 'yes'); @@ -95,8 +82,6 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('before')).toBe('yes'); }); - // A rollback that left the transaction open would swallow every later write on - // the connection, including the ones `Kernel.stop()` makes on the way out. it('leaves the database writable after a rolled-back crank', async () => { const { kernelStore, kdb } = await makeStore(); @@ -109,8 +94,7 @@ describe('crank rollback against a real database', () => { kdb.kernelKVStore.set('after', 'yes'); expect(kdb.kernelKVStore.get('after')).toBe('yes'); - // Survives the commit boundary a subsequent crank draws, so the write really - // landed rather than sitting in a transaction that never resolves. + // Survives the commit boundary a later crank draws. kernelStore.startCrank(); kernelStore.createCrankSavepoint('start'); kernelStore.endCrank(); @@ -118,8 +102,6 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('discarded')).toBeUndefined(); }); - // The abort path rolls back mid-crank and the run loop then keeps going, so the - // next crank has to be able to create its own savepoint and commit normally. it('commits a later crank after an earlier one rolled back', async () => { const { kernelStore, kdb } = await makeStore(); @@ -138,8 +120,6 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('second')).toBe('yes'); }); - // An aborted crank still owes work after the rollback — terminating the vat, - // collecting garbage — whose writes have to survive it. it('keeps the writes a crank makes after rolling its delivery back', async () => { const { kernelStore, kdb } = await makeStore(); @@ -156,9 +136,8 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBe('yes'); }); - // And survive it *inside the crank's transaction*, not as autocommitted - // statements. Rolling back `crank` is the only way to observe that from here; - // the run loop never does it. + // Rolling back `crank` is the only way to observe this from here; the run loop + // never does it. it('holds those writes in the transaction rather than autocommitting them', async () => { const { kernelStore, kdb } = await makeStore(); @@ -174,11 +153,6 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBeUndefined(); }); - // Every `provideCachedStoredValue` keeps its value in a closure and writes - // through to kv, so a rollback that only reverts the database leaves the cache - // holding the abandoned crank's value — and the next `set` persists it. The GC - // action set is the case that matters: `processGCActionSet` consumes an action - // before delivering it, so losing the rollback loses the action outright. it('restores the GC action set consumed by a rolled-back crank', async () => { const { kernelStore } = await makeStore(); kernelStore.addGCActions(['v1 dropExport ko1']); @@ -197,8 +171,6 @@ describe('crank rollback against a real database', () => { ]); }); - // Same closure, same failure: a reap scheduled and then consumed by a crank - // that rolls back must still be pending afterwards. it('restores the reap queue consumed by a rolled-back crank', async () => { const { kernelStore } = await makeStore(); kernelStore.scheduleReap('v1'); @@ -214,18 +186,13 @@ describe('crank rollback against a real database', () => { expect(kernelStore.nextReapAction()).toBeDefined(); }); - // `maybeFreeKrefs` is RAM-only, so nothing rolls it back. Left populated, the - // next crank's `collectGarbage` visits krefs whose decrements were undone — - // and `getKernelPromise` throws outright for one the rollback deleted, which - // kills the run loop. it('discards GC candidates accumulated by a rolled-back crank', async () => { const { kernelStore } = await makeStore(); kernelStore.startCrank(); kernelStore.createCrankSavepoint('crank'); kernelStore.createCrankSavepoint('delivery'); - // Born at 1, so this drops it to 0 and leaves `kpid` in `maybeFreeKrefs` - // while the rollback removes the promise record it names. + // Born at 1, so this drops it to 0 and leaves `kpid` in `maybeFreeKrefs`. const kpid = kernelStore.initKernelPromise()[0]; kernelStore.decrementRefCount(kpid, 'test'); @@ -238,9 +205,6 @@ describe('crank rollback against a real database', () => { kernelStore.endCrank(); }); - // `createCrankSavepoint` records the name only once the database has the - // savepoint. Asking to roll back one that was never created must therefore say - // so, rather than releasing someone else's savepoint. it('refuses to roll back a savepoint that was never created', async () => { const { kernelStore } = await makeStore(); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index d9c4626d03..d835137dc7 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -232,8 +232,6 @@ describe('KernelQueue', () => { expect(kernelStore.endCrank).toHaveBeenCalled(); }); - // Why the flush comes after the crank's fallible work: see - // `#processCrankResult`. Here the terminate is what fails. it('answers no caller from a crank it then rolls back', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -278,15 +276,11 @@ describe('KernelQueue', () => { expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(resolve).not.toHaveBeenCalled(); - // Told the result will never come, rather than left waiting on a crank the - // store no longer has any record of. expect(reject).toHaveBeenCalledWith( expect.objectContaining({ cause: terminationError }), ); }); - // The same invariant inside the flush: `#enqueueRun` is store work and can - // fail part-way, so no caller may be answered until all of it lands. it('answers no caller until every buffered item is enqueued', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -334,8 +328,6 @@ describe('KernelQueue', () => { expect(resolve).not.toHaveBeenCalled(); }); - // Why two savepoints: see `#runLoop`. This pins that the rollback spares the - // transaction, so the work an aborted crank still owes stays inside it. it.each([ { label: 'an abort', @@ -558,13 +550,6 @@ describe('KernelQueue', () => { }); }); - // FAILING REPRO. - // - // The companion of the case above, at the other end of the crank. Since the - // delivery rollback now spares `crank`, `endCrank`'s release is a real - // RELEASE + COMMIT on the dying path where it used to be a no-op, and - // `#runLoop` calls it from a bare `finally` — so when it throws it replaces - // the error that killed the kernel instead of being reported alongside it. it('reports both failures when endCrank also fails', async () => { ( kernelStore.runQueueLength as unknown as MockInstance @@ -586,8 +571,6 @@ describe('KernelQueue', () => { .run(deliver) .catch((error: unknown) => error); - // However the release failure is named, the error that actually killed the - // kernel has to stay reachable — as the rollback path already manages. expect(causeChain(failure)).toContain(crankError); expect(kernelQueue.getRunLoopStatus()).toMatchObject({ state: 'failed', diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index 90f4ef3bc5..e1e219958c 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -221,11 +221,6 @@ describe('RemoteHandle', () => { }); }); - // `handleRemoteMessage` releases its savepoint inside the `try` and rolls - // back in the `catch`. #1012 made a failed `RELEASE` discard the whole - // savepoint stack, so that rollback reports a savepoint that no longer - // exists, and left unguarded it throws out of the `catch` in place of the - // failure that brought it there. it('reports the release failure rather than a missing savepoint', async () => { const failing = withFailingSavepointRelease(mockKernelStore); const { releaseFailure, rollbackSavepoint } = failing; @@ -238,14 +233,10 @@ describe('RemoteHandle', () => { params: ['bringOutYourDead'], }); - // The error an operator needs is the one the database gave, not the - // bookkeeping artefact of trying to clean up after it. await expect(remote.handleRemoteMessage(delivery)).rejects.toBe( releaseFailure, ); - // Still attempted, so that abandoning the rollback is not a way to pass - // this test: a release that failed for a reason of its own may well have - // left the savepoint standing. + // Still attempted: abandoning the rollback is not a way to pass this. expect(rollbackSavepoint).toHaveBeenCalledWith('receive_r0_1'); }); diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index c3257d85df..b6b9dd08a9 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -950,10 +950,6 @@ describe('RemoteManager', () => { expect(finalizeSpy).not.toHaveBeenCalled(); }); - // The same shape `RemoteHandle.handleRemoteMessage` has: the release sits - // inside the `try` and the rollback in the `catch`, so a failed RELEASE — - // which discards the whole savepoint stack — leaves the rollback naming a - // savepoint that is gone. it('reports the release failure rather than a missing savepoint', async () => { const peerId = 'peer-whose-release-fails'; const { @@ -976,14 +972,10 @@ describe('RemoteManager', () => { observedIncarnation: string, ) => Promise; - // The error an operator needs is the one the database gave, not the - // bookkeeping artefact of trying to clean up after it. await expect(onIncarnationChange(peerId, 'incarnation-A')).rejects.toBe( releaseFailure, ); - // Still attempted, so that abandoning the rollback is not a way to pass: - // a release that failed for a reason of its own may well have left the - // savepoint standing. + // Still attempted: abandoning the rollback is not a way to pass this. expect(rollbackSavepoint).toHaveBeenCalledWith( `peerIncarnation_${peerId}`, ); diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 426890d209..7df88392c1 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -156,9 +156,6 @@ describe('crank methods', () => { expect(mockCrankBuffer).toHaveLength(0); }); - // A failed rollback discards every savepoint, not just this one. Truncating - // to the ordinal would have `endCrank` release a `t0` the database lacks and - // throw over whatever really killed the kernel. it('forgets every savepoint when the rollback fails', () => { context.inCrank = true; crankMethods.createCrankSavepoint('crank'); @@ -172,16 +169,10 @@ describe('crank methods', () => { ); expect(context.savepoints).toStrictEqual([]); - // The release `endCrank` would otherwise attempt, and throw over. crankMethods.endCrank(); expect(kdb.releaseSavepoint).not.toHaveBeenCalled(); }); - // The two halves of this function compose the wrong way round if the - // failure path simply rethrows: a failed rollback discards the whole - // transaction, so the database has moved back at least as far as a - // successful rollback would have taken it and the caches it left behind are - // at least as stale. it('reverts the caches the database cannot reach even when the rollback fails', () => { context.inCrank = true; context.maybeFreeKrefs.add('kp1'); @@ -201,7 +192,6 @@ describe('crank methods', () => { expect([...context.maybeFreeKrefs]).toStrictEqual([]); }); - // Reverting must not become a way to lose the database error either. it('keeps the rollback failure as the cause when reverting also fails', () => { context.inCrank = true; const rollbackFailure = new Error('disk I/O error'); @@ -272,9 +262,6 @@ describe('crank methods', () => { expect(await waiter).toBeUndefined(); }); - // As `rollbackCrank` does. Left listed, the next crank numbers its savepoint - // `t1` against a database that has none, and every later release and rollback - // aims one crank past its target. it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); context.savepoints = ['test']; From 29744e4c9423ddd988a3e870705256a417d92072 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:38:31 +0200 Subject: [PATCH 26/40] test(kernel-store): pin the transaction a failed COMMIT leaves open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two invariants the crank layer relies on: a failed `ROLLBACK TO` discards the whole transaction, so truncating `ctx.savepoints` to zero still matches the database; and `commitIfNeeded` leaves no transaction behind. Both drivers catch and log an abort that fails while discarding a transaction, so the first holds only when that abort succeeds. This driver reads `db.inTransaction` from SQLite rather than caching it, which prevents a wedged flag but does not end a transaction nothing owns. The mock tracks the transaction the way SQLite does — a statement that throws changes nothing, one that succeeds opens or closes it — because a mock whose `inTransaction` never clears can only ever model a wedged connection, which is the state under test and so the one thing that must not be assumed. Co-Authored-By: Dimitris Marlagkoutsos Co-Authored-By: Claude Opus 5 (1M context) --- .../nodejs.transaction-survival.test.ts | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts diff --git a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts new file mode 100644 index 0000000000..ba9a442597 --- /dev/null +++ b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +import { makeSQLKernelDatabase } from './nodejs.ts'; + +/** + * Two invariants the crank layer relies on: + * + * - a failed `ROLLBACK TO` discards the whole transaction, so truncating + * `ctx.savepoints` to zero still matches the database; + * - `commitIfNeeded` leaves no transaction behind. + * + * Both drivers catch and log an abort that fails while discarding a transaction, + * so "the whole transaction is discarded" holds only when that abort succeeds. + * This driver has no `_inTx` flag, reading `db.inTransaction` from SQLite + * instead, which prevents a wedged flag but does not by itself end an ownerless + * transaction. + */ + +/** Every statement and exec call, in order. */ +let issued: string[] = []; +/** SQL that throws when next run. */ +let failOnce: Set = new Set(); + +/** What SQLite would report through `db.inTransaction`. */ +let inTransaction = false; + +/** + * Run one statement against the mock, tracking the transaction the way SQLite + * does: a statement that throws changes nothing, and one that succeeds opens or + * closes the transaction. Without the latter the mock can only ever model a + * connection that is wedged, which is the state under test here and so exactly + * the state that must not be assumed. + * + * @param text - The SQL being run. + */ +function runSql(text: string): void { + issued.push(text); + if (failOnce.delete(text)) { + throw new Error(`SQLITE_IOERR: ${text}`); + } + if (text === 'BEGIN TRANSACTION') { + inTransaction = true; + } else if (text === 'COMMIT TRANSACTION' || text === 'ROLLBACK TRANSACTION') { + inTransaction = false; + } +} + +const makeStatement = (text: string): Record => ({ + run: () => { + runSql(text); + return undefined; + }, + get: () => undefined, + all: () => [], + pluck: () => undefined, + iterate: () => [], +}); + +const mockDb = { + prepare: vi.fn((text: string) => makeStatement(text)), + transaction: vi.fn((fn: () => void) => fn), + exec: vi.fn(runSql), + get inTransaction(): boolean { + return inTransaction; + }, + set inTransaction(value: boolean) { + inTransaction = value; + }, + _spStack: [] as string[], + close: vi.fn(), +}; + +vi.mock('better-sqlite3', () => ({ + default: vi.fn(function () { + return mockDb; + }), +})); +vi.mock('node:fs/promises', () => ({ mkdir: vi.fn() })); +vi.mock('node:os', () => ({ tmpdir: vi.fn(() => '/mock-tmpdir') })); + +describe('the nodejs driver after a failure it tolerates', () => { + beforeEach(() => { + issued = []; + failOnce = new Set(); + mockDb.inTransaction = false; + mockDb._spStack = []; + }); + + it('discards the transaction when the rollback fails and the abort fails too', async () => { + const kdb = await makeSQLKernelDatabase({}); + // A crank in progress: SAVEPOINT t0, SAVEPOINT t1. + mockDb.inTransaction = true; + mockDb._spStack = ['t0', 't1']; + issued = []; + + // The disk fills. `ROLLBACK TO SAVEPOINT t1` fails, and so does the + // `ROLLBACK TRANSACTION` meant to discard the transaction instead. The + // driver logs that second failure and rethrows the first, so SQLite is + // still in a transaction with t0 and t1 on its stack. + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + expect(mockDb._spStack).toStrictEqual([]); + expect(mockDb.inTransaction).toBe(true); + + // `_spStack` now says "no savepoints, nothing to commit or abort" while + // SQLite says otherwise. Teardown still runs after the run loop dies -- + // `reset`, a peer incarnation change, a remote message -- and takes a + // savepoint. + issued = []; + kdb.createSavepoint('teardown'); + kdb.releaseSavepoint('teardown'); + + // The abandoned transaction is retried and discarded before the new + // savepoint can join it, so what the COMMIT makes durable is `teardown`'s + // own transaction and not the crank that was thrown away. + expect(issued).toStrictEqual([ + 'ROLLBACK TRANSACTION', + 'BEGIN TRANSACTION', + 'SAVEPOINT teardown', + 'RELEASE SAVEPOINT teardown', + 'COMMIT TRANSACTION', + ]); + }); + + it('refuses to write once the transaction cannot be discarded at all', async () => { + const kdb = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['t0', 't1']; + issued = []; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + // The retry fails too, so there is no honest way to carry on: a savepoint + // taken now would be released into the abandoned transaction, committing + // the crank the abort was trying to discard. + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.createSavepoint('teardown')).toThrow( + 'refusing further writes', + ); + expect(issued).not.toContain('SAVEPOINT teardown'); + }); + + it('discards the transaction when the commit fails', async () => { + const kdb = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['t0']; + issued = []; + + // endCrank: RELEASE SAVEPOINT t0 succeeds, the COMMIT it triggers does not. + failOnce.add('COMMIT TRANSACTION'); + expect(() => kdb.releaseSavepoint('t0')).toThrow('SQLITE_IOERR'); + + // A failed COMMIT can leave the transaction open, and `_spStack` was + // already spliced empty, so nothing else would have ended it. + expect(issued).toContain('ROLLBACK TRANSACTION'); + expect(mockDb.inTransaction).toBe(false); + }); +}); From 198380abc8bd39185c1c777cd497d0b05b3d2d56 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Sep 2026 18:38:47 +0200 Subject: [PATCH 27/40] fix(kernel-store): refuse to commit a transaction an abort could not end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two doors onto the same hazard, one per driver. `releaseSavepoint` in the nodejs driver reaches `commitIfNeeded` outside any try of its own, so a throwing COMMIT left the transaction open with nothing to end it — the gap b90e7a5e5 closed in the wasm driver. Reading `db.inTransaction` prevents a wedged flag but does not end a transaction nothing owns; that asymmetry was #1013. And when the compensating abort itself failed, both drivers emptied the savepoint list while SQLite still held the transaction. The next `createSavepoint` then saw `inTransaction`, skipped BEGIN, nested inside the abandoned transaction, and its RELEASE committed the crank the abort had been trying to throw away. Both now retry the abort and, if the transaction survives it, throw rather than return: reporting that the connection can no longer persist anything is the only honest answer left, where returning normally tells the caller its write landed. Checking the flag before `_inTx` in the wasm driver also makes that guard reachable — `abortTransaction` clears `_inTx` on its way to failing, so gating on it made the branch dead code. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/src/sqlite/nodejs.ts | 92 +++++++++--- packages/kernel-store/src/sqlite/wasm.test.ts | 9 +- packages/kernel-store/src/sqlite/wasm.ts | 132 ++++++++++++------ 3 files changed, 164 insertions(+), 69 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index d841997142..ec17e1c117 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -150,12 +150,20 @@ export async function makeSQLKernelDatabase({ const sqlCommitTransaction = db.prepare(SQL_QUERIES.COMMIT_TRANSACTION); const sqlAbortTransaction = db.prepare(SQL_QUERIES.ABORT_TRANSACTION); + // Set when an abort meant to discard a transaction fails. Reading + // `db.inTransaction` keeps this driver from wedging a flag the way a cached + // one can, but it cannot end a transaction nothing owns: the writes of the + // crank we gave up on are still in it, and a savepoint taken inside it would + // be released into it. + let txAbandoned = false; + /** * Begin a transaction if not already in one * * @returns True if a new transaction was started, false if already in one */ function beginIfNeeded(): boolean { + assertNotAbandoned(); if (db.inTransaction) { return false; } @@ -163,12 +171,44 @@ export async function makeSQLKernelDatabase({ return true; } + /** + * Refuse to touch a transaction an earlier abort could not end. A savepoint + * created inside one is released into it, committing the crank that abort was + * discarding, and a COMMIT makes those writes durable outright. Retried once + * first, since the failure may have been transient. + * + * @throws If the transaction is still there afterwards, because reporting + * that this connection can no longer persist anything is the only honest + * answer left — returning normally would tell the caller its write landed. + */ + function assertNotAbandoned(): void { + if (!txAbandoned) { + return; + } + discardTransaction('abandonment'); + if (txAbandoned) { + throw new Error( + 'transaction cannot be ended; refusing further writes on this connection', + ); + } + } + /** * Commit a transaction if one is active and no savepoints remain */ function commitIfNeeded(): void { - if (db.inTransaction && db._spStack.length === 0) { + assertNotAbandoned(); + if (!db.inTransaction || db._spStack.length > 0) { + return; + } + try { sqlCommitTransaction.run(); + } catch (error) { + // A failed COMMIT can leave the transaction open, and `releaseSavepoint` + // reaches here outside any try of its own — the same hazard the savepoint + // paths below discard the transaction to avoid, by a third door. + discardTransaction('commit'); + throw error; } } @@ -176,9 +216,35 @@ export async function makeSQLKernelDatabase({ * Rollback a transaction */ function rollbackIfNeeded(): void { - if (db.inTransaction) { + if (!db.inTransaction) { + txAbandoned = false; + return; + } + try { sqlAbortTransaction.run(); - db._spStack.length = 0; + } catch (error) { + txAbandoned = true; + throw error; + } + db._spStack.length = 0; + // Normally false now. If SQLite still reports a transaction, it is still + // not ours to commit. + txAbandoned = db.inTransaction; + } + + /** + * Discard the transaction after a failure that leaves it unowned, keeping the + * error that got us here rather than the abort's. + * + * @param after - What failed, completing "failed to discard transaction after + * ...". + */ + function discardTransaction(after: string): void { + db._spStack.length = 0; + try { + rollbackIfNeeded(); + } catch (error) { + logger?.error(`failed to discard transaction after ${after}`, error); } } @@ -295,15 +361,7 @@ export async function makeSQLKernelDatabase({ // connection joins it, reports success, and vanishes on close. Discarding // the whole transaction is safe: it begins with the outermost savepoint, so // it holds only what this rollback was abandoning anyway. - db._spStack.length = 0; - try { - rollbackIfNeeded(); - } catch (abortError) { - logger?.error( - 'failed to discard transaction after rollback', - abortError, - ); - } + discardTransaction('rollback'); throw error; } db._spStack.splice(idx); @@ -329,15 +387,7 @@ export async function makeSQLKernelDatabase({ } catch (error) { // The hazard `rollbackSavepoint` guards against, by the other door, and // there is no committing this transaction now. - db._spStack.length = 0; - try { - rollbackIfNeeded(); - } catch (abortError) { - logger?.error( - 'failed to discard transaction after release', - abortError, - ); - } + discardTransaction('release'); throw error; } db._spStack.splice(idx); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index f67455e272..921c3f6430 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -569,7 +569,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - it('begins a transaction for the next savepoint after a failed abort', async () => { + it('retries the abort before beginning the next savepoint', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; mockDb._spStack = ['point1']; @@ -587,9 +587,12 @@ describe('makeSQLKernelDatabase', () => { mockStatement.step.mockClear(); db.createSavepoint('next'); - // BEGIN is the only prepared statement `createSavepoint` runs; the + // The abort that failed above may have left SQLite holding a transaction + // `_inTx` no longer accounts for, and BEGIN inside one throws. So the + // abort is retried first: two prepared statements, that and BEGIN. The // SAVEPOINT itself goes through `exec`. - expect(mockStatement.step).toHaveBeenCalledOnce(); + expect(mockStatement.step).toHaveBeenCalledTimes(2); + expect(mockDb._inTx).toBe(true); expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 55c7594e2f..a9a5db1d29 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -179,12 +179,21 @@ export async function makeSQLKernelDatabase({ const sqlCommitTransaction = db.prepare(SQL_QUERIES.COMMIT_TRANSACTION); const sqlAbortTransaction = db.prepare(SQL_QUERIES.ABORT_TRANSACTION); + // Set when an abort meant to discard a transaction fails, which leaves SQLite + // holding one `_inTx` no longer accounts for: nothing will try to end it + // again, and a savepoint taken inside it would be released into it. The + // nodejs driver reads `db.inTransaction` instead of caching it and so cannot + // wedge the flag, but it has the same unowned-transaction gap; both drivers + // close it the same way. + let txAbandoned = false; + /** * Begin a transaction if not already in one * * @returns True if a new transaction was started, false if already in one */ function beginIfNeeded(): boolean { + assertNotAbandoned(); if (db._inTx) { return false; } @@ -194,31 +203,71 @@ export async function makeSQLKernelDatabase({ return true; } + /** + * Refuse to touch a transaction an earlier abort could not end. A savepoint + * created inside one is released into it, committing the crank that abort was + * discarding, and a COMMIT makes those writes durable outright. Retried once + * first, since the failure may have been transient. + * + * Checked before `_inTx` rather than after: `abortTransaction` clears the flag + * on its way to failing, so the abandoned transaction is exactly the one + * `_inTx` no longer admits to. + * + * @throws If the transaction is still there afterwards, because reporting + * that this connection can no longer persist anything is the only honest + * answer left — returning normally would tell the caller its write landed. + */ + function assertNotAbandoned(): void { + if (!txAbandoned) { + return; + } + discardTransaction('abandonment'); + if (txAbandoned) { + throw new Error( + 'transaction cannot be ended; refusing further writes on this connection', + ); + } + } + /** * Commit a transaction if one is active and no savepoints remain */ function commitIfNeeded(): void { - if (db._inTx && db._spStack.length === 0) { - // Cleared first, for the reason `rollbackIfNeeded` gives. - db._inTx = false; - try { - sqlCommitTransaction.step(); - sqlCommitTransaction.reset(); - } catch (error) { - // A failed COMMIT can leave the transaction open. Stepped directly - // because `_inTx` above already made `rollbackIfNeeded` a no-op. - try { - sqlAbortTransaction.step(); - sqlAbortTransaction.reset(); - } catch (abortError) { - logger?.error( - 'failed to discard transaction after commit', - abortError, - ); - } - throw error; - } + assertNotAbandoned(); + if (!db._inTx || db._spStack.length > 0) { + return; + } + // Cleared first, for the reason `rollbackIfNeeded` gives. + db._inTx = false; + try { + sqlCommitTransaction.step(); + sqlCommitTransaction.reset(); + } catch (error) { + // A failed COMMIT can leave the transaction open, and `releaseSavepoint` + // reaches here outside any try of its own. + discardTransaction('commit'); + throw error; + } + } + + /** + * Abort the transaction, whatever `_inTx` currently says — the callers that + * most need this have already cleared it. + */ + function abortTransaction(): void { + // Cleared before the abort, which can throw: left true, `beginIfNeeded` is + // a no-op forever after and writes autocommit one statement at a time (see + // `createSavepoint`). + db._inTx = false; + db._spStack.length = 0; + try { + sqlAbortTransaction.step(); + sqlAbortTransaction.reset(); + } catch (error) { + txAbandoned = true; + throw error; } + txAbandoned = false; } /** @@ -226,13 +275,22 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - // Cleared before the abort, which can throw: left true, `beginIfNeeded` is - // a no-op forever after and writes autocommit one statement at a time (see - // `createSavepoint`). - db._inTx = false; - db._spStack.length = 0; - sqlAbortTransaction.step(); - sqlAbortTransaction.reset(); + abortTransaction(); + } + } + + /** + * Discard the transaction after a failure that leaves it unowned, keeping the + * error that got us here rather than the abort's. + * + * @param after - What failed, completing "failed to discard transaction after + * ...". + */ + function discardTransaction(after: string): void { + try { + abortTransaction(); + } catch (error) { + logger?.error(`failed to discard transaction after ${after}`, error); } } @@ -396,15 +454,7 @@ export async function makeSQLKernelDatabase({ // connection joins it, reports success, and vanishes on close. Discarding // the whole transaction is safe: it begins with the outermost savepoint, so // it holds only what this rollback was abandoning anyway. - db._spStack.length = 0; - try { - rollbackIfNeeded(); - } catch (abortError) { - logger?.error( - 'failed to discard transaction after rollback', - abortError, - ); - } + discardTransaction('rollback'); throw error; } db._spStack.splice(idx); @@ -430,15 +480,7 @@ export async function makeSQLKernelDatabase({ } catch (error) { // The hazard `rollbackSavepoint` guards against, by the other door, and // there is no committing this transaction now. - db._spStack.length = 0; - try { - rollbackIfNeeded(); - } catch (abortError) { - logger?.error( - 'failed to discard transaction after release', - abortError, - ); - } + discardTransaction('release'); throw error; } db._spStack.splice(idx); From 64ffca1e5ed7a21d2771b7e1a68d3b94ec595ab5 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:39:00 +0200 Subject: [PATCH 28/40] test(ocap-kernel): pin the audit that runs after the crank answers its callers The flush is last of the crank's own work so that no external caller is answered before the fallible work is done, and the reference count audit runs after the flush so that buffered items are not read as leaks. The audit is itself fallible, so the two orderings contradict each other. Co-Authored-By: Dimitris Marlagkoutsos Co-Authored-By: Claude Opus 5 (1M context) --- .../src/KernelQueue.audit-ordering.test.ts | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts diff --git a/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts b/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts new file mode 100644 index 0000000000..278cdc49e7 --- /dev/null +++ b/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts @@ -0,0 +1,101 @@ +import type { CapData } from '@endo/marshal'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +import { KernelQueue } from './KernelQueue.ts'; +import type { KernelStore } from './store/index.ts'; +import type { CrankResult, KRef, RunQueueItem } from './types.ts'; + +vi.mock('./garbage-collection/garbage-collection.ts', () => ({ + processGCActionSet: vi.fn().mockReturnValue(null), +})); + +/** + * `#processCrankResult` flushes the crank buffer last of the crank's own work, + * "after the fallible work above, not before it", because the flush settles the + * promise `enqueueMessage` gave an external caller and a later rollback would + * discard the state that answer was computed from. + * + * `assertRefCountsIfAuditing` has to run after that flush too, because a + * buffered item's references were counted at enqueue time and so read as a leak + * if the audit runs mid-flush. Both orderings were individually justified and + * they contradicted each other: the audit is fallible, it ran after the answers + * had gone out, and a crank that throws is rolled back by the run loop's catch. + * + * The audit now runs after `endCrank` instead, outside anything a rollback can + * reach, so a violation kills the run loop rather than retracting a delivery the + * caller was already answered from. + */ +describe('a reference count audit that fails after the flush', () => { + let kernelStore: KernelStore; + let kernelQueue: KernelQueue; + let resolveSubscription: (value: CapData) => void; + + const AUDIT_FAILED = 'reference count invariant violated'; + + beforeEach(() => { + resolveSubscription = vi.fn(); + + kernelStore = { + startCrank: vi.fn(), + endCrank: vi.fn(), + beginOutOfCrank: vi.fn().mockResolvedValue(undefined), + endOutOfCrank: vi.fn(), + outOfCrankWorkPending: vi.fn().mockReturnValue(undefined), + createCrankSavepoint: vi.fn(), + rollbackCrank: vi.fn(), + nextTerminatedVatCleanup: vi.fn(), + nextReapAction: vi.fn().mockReturnValue(null), + runQueueLength: vi.fn().mockReturnValue(0), + dequeueRun: vi.fn(), + enqueueRun: vi.fn(), + incrementRefCount: vi.fn(), + collectGarbage: vi.fn(), + // The buffered notify a successful crank flushes. + flushCrankBuffer: vi + .fn() + .mockReturnValue([{ type: 'notify', endpointId: 'v1', kpid: 'kp1' }]), + getKernelPromise: vi.fn().mockReturnValue({ + state: 'fulfilled', + value: { body: '{}', slots: [] }, + }), + // The audit the `auditRefCounts` option turns on, finding drift. + assertRefCountsIfAuditing: vi.fn(() => { + throw new Error(AUDIT_FAILED); + }), + } as unknown as KernelStore; + + kernelQueue = new KernelQueue( + kernelStore, + vi.fn().mockResolvedValue(undefined), + ); + + // An external caller waiting on `kp1`, as `enqueueMessage` leaves one. + kernelQueue.subscriptions.set('kp1' as KRef, { + resolve: resolveSubscription, + reject: vi.fn(), + }); + }); + + it('keeps the delivery it already answered the caller from', async () => { + const item: RunQueueItem = { + type: 'notify', + endpointId: 'v1', + kpid: 'kp1' as KRef, + } as RunQueueItem; + vi.mocked(kernelStore.runQueueLength).mockReturnValueOnce(1); + vi.mocked(kernelStore.dequeueRun).mockReturnValueOnce(item); + + const deliver = vi + .fn<(queueItem: RunQueueItem) => Promise>() + .mockResolvedValue(undefined); + + await expect(kernelQueue.run(deliver)).rejects.toThrow(AUDIT_FAILED); + + // The flush invoked the subscription, so the caller has its answer... + expect(resolveSubscription).toHaveBeenCalled(); + // ...and the audit's failure takes the run loop down without retracting the + // delivery that answer was computed from. + expect(kernelStore.rollbackCrank).not.toHaveBeenCalledWith('delivery'); + expect(kernelStore.endCrank).toHaveBeenCalled(); + }); +}); From be4f40685cac44b0ff8c3c78fd979c7dc0cc3592 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Sep 2026 18:39:17 +0200 Subject: [PATCH 29/40] fix(ocap-kernel): audit reference counts after the crank commits, not inside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assertRefCountsIfAuditing` has to run after the crank buffer flush, because a buffered item's references were counted at enqueue time and so read as a leak mid-flush. But the flush is also what settles the promise `enqueueMessage` gave an external caller, and the audit is fallible: a violation threw into the run loop's catch, which rolled back the delivery the caller had already been answered from. Moved past `endCrank`, where a violation still kills the run loop — which is what an audit failure means — without pretending to undo a crank that has landed. Guarded on the crank having delivered something, so an idle crank about to sleep is not audited. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelQueue.test.ts | 99 ++++++++++++++++++++ packages/ocap-kernel/src/KernelQueue.ts | 36 ++++++- 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index d835137dc7..f66e79b05a 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -74,6 +74,9 @@ describe('KernelQueue', () => { getGCActions: vi.fn().mockReturnValue([]), startCrank: vi.fn(), endCrank: vi.fn(), + beginOutOfCrank: vi.fn().mockResolvedValue(undefined), + endOutOfCrank: vi.fn(), + outOfCrankWorkPending: vi.fn().mockReturnValue(undefined), createCrankSavepoint: vi.fn(), rollbackCrank: vi.fn(), waitForCrank: vi.fn(), @@ -1139,6 +1142,102 @@ describe('KernelQueue', () => { }); }); + describe('out-of-crank work', () => { + it('waits for a caller holding the store before starting a crank', async () => { + let releaseHolder!: () => void; + const held = new Promise((resolve) => { + releaseHolder = resolve; + }); + let cranksWhileHeld = 0; + // Return values rather than an implementation: the contract is that + // nothing waiting is `undefined` and not a resolved promise, and an + // `async` mock cannot express that. + (kernelStore.outOfCrankWorkPending as unknown as MockInstance) + .mockReturnValueOnce(held) + .mockReturnValue(undefined); + (kernelStore.startCrank as unknown as MockInstance).mockImplementation( + () => { + cranksWhileHeld += 1; + throw new Error(STOP_RUN_LOOP); + }, + ); + + const runLoop = kernelQueue.run(vi.fn()); + await Promise.resolve(); + expect(cranksWhileHeld).toBe(0); + + releaseHolder(); + await expect(runLoop).rejects.toThrow(STOP_RUN_LOOP); + expect(cranksWhileHeld).toBe(1); + }); + + it('starts the crank without yielding when nothing is waiting', async () => { + stopAfterOneCrank(); + (kernelStore.runQueueLength as unknown as MockInstance).mockReturnValue( + 0, + ); + + await expect(kernelQueue.run(vi.fn())).rejects.toThrow(STOP_RUN_LOOP); + expect(kernelStore.outOfCrankWorkPending).toHaveBeenCalled(); + }); + + // A caller registers synchronously, so one can arrive in a microtask queued + // ahead of this loop's resumption. Checking once would leave `startCrank` + // refusing, which kills the kernel outright. + it('keeps yielding while callers keep arriving', async () => { + const pending = + kernelStore.outOfCrankWorkPending as unknown as MockInstance; + // Two callers in a row, then nothing. `undefined` rather than a resolved + // promise is the contract; an `async` mock would loop here forever. + pending + .mockReturnValueOnce(Promise.resolve()) + .mockReturnValueOnce(Promise.resolve()) + .mockReturnValue(undefined); + let checksBeforeCrank = -1; + (kernelStore.startCrank as unknown as MockInstance).mockImplementation( + () => { + checksBeforeCrank = pending.mock.calls.length; + throw new Error(STOP_RUN_LOOP); + }, + ); + + await expect(kernelQueue.run(vi.fn())).rejects.toThrow(STOP_RUN_LOOP); + + // Both gates awaited, and the check that found nothing waiting is the one + // immediately before the crank starts. + expect(checksBeforeCrank).toBe(3); + }); + }); + + describe('reference count audit', () => { + it('audits a crank that delivered something', async () => { + stopAfterOneCrank(); + ( + kernelStore.runQueueLength as unknown as MockInstance + ).mockReturnValueOnce(1); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce({ + type: 'send', + target: 'ko123', + message: {} as KernelMessage, + }); + + await expect( + kernelQueue.run(vi.fn().mockResolvedValue(undefined)), + ).rejects.toThrow(STOP_RUN_LOOP); + expect(kernelStore.assertRefCountsIfAuditing).toHaveBeenCalled(); + }); + + it('leaves an idle crank alone', async () => { + stopAfterOneCrank(); + (kernelStore.runQueueLength as unknown as MockInstance).mockReturnValue( + 0, + ); + + await expect(kernelQueue.run(vi.fn())).rejects.toThrow(STOP_RUN_LOOP); + expect(kernelStore.assertRefCountsIfAuditing).not.toHaveBeenCalled(); + }); + }); + describe('invokeKernelSubscription', () => { it('calls reject for rejected promises', async () => { const rejectSpy = vi.fn(); diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 4e8def6e5b..24839b7273 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -123,6 +123,25 @@ export class KernelQueue { // Boxed, so a crank that threw `undefined` stays distinguishable from one // that did not throw. let crankFailure: { error: unknown } | undefined; + let delivered = false; + + // Work that has to be its own transaction — an inbound remote message, a + // peer restart — can only run while no crank is open, and this loop is + // synchronous from `endCrank` to here. Without this yield its only gap is + // the sleep below, so such work would wait for the queue to drain or, as + // it used to, nest a savepoint inside a crank whose rollback would + // discard it. Undefined on the usual path, which keeps that path + // synchronous. + // Re-checked rather than awaited once: a caller registers synchronously, + // so one arriving in a microtask queued ahead of this loop's resumption + // would otherwise meet `startCrank`'s refusal and kill the kernel over + // ordinary concurrent remote traffic. The last check and `startCrank` + // have no await between them, which is what makes the handoff airtight. + let outOfCrankWork = this.#kernelStore.outOfCrankWorkPending(); + while (outOfCrankWork) { + await outOfCrankWork; + outOfCrankWork = this.#kernelStore.outOfCrankWorkPending(); + } this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; @@ -141,6 +160,7 @@ export class KernelQueue { try { const queueItem = this.#getNextRunQueueItem(); if (queueItem) { + delivered = true; this.#kernelStore.nextTerminatedVatCleanup(); const crankResult = await deliver(queueItem); await this.#processCrankResult(crankResult, queueItem); @@ -177,6 +197,19 @@ export class KernelQueue { await wakeUpPromise; } } + + if (delivered) { + // After the crank has been committed, not inside it. The audit has to + // run after the flush, because a buffered item's references were + // counted when it was enqueued and so read as a leak mid-flush — but + // the flush is also what answers an external caller of + // `enqueueMessage`. Auditing while the delivery savepoint still existed + // put those two together: a violation would roll back the state the + // caller had already been answered from. Out here a violation still + // kills the run loop, which is what an audit failure means, without + // pretending to undo a crank that has landed. + this.#kernelStore.assertRefCountsIfAuditing(); + } } } @@ -366,9 +399,6 @@ export class KernelQueue { // would discard the state that answer was computed from. this.#flushCrankBuffer(); } - // After the flush: a buffered item's references were counted when it was - // enqueued, so audited mid-flush every one of them reads as a leak. - this.#kernelStore.assertRefCountsIfAuditing(); } /** From 775636bcaa618ae94186d13c06e9b8d35761f515 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:39:31 +0200 Subject: [PATCH 30/40] test(kernel-store): pin the crank commit point against a foreign savepoint `KernelQueue.#runLoop` calls releasing its `crank` savepoint "this crank's one commit point". `releaseAllSavepoints` releases `t0`, which is the outermost savepoint only when the crank opened the first one, and two production paths open savepoints through `KernelStore.createSavepoint` -- invisible to the ordinal numbering, uncoordinated with the crank, one of them held across an await. Real SQLite through the real driver, one test per interleaving. The assertions record what SQLite actually does rather than what the kernel ought to: a savepoint that is not the outermost one does not commit on release, and `ROLLBACK TO` cancels every savepoint taken after its target. Those are the semantics the kernel-side serialization exists to respect, and they do not change with it. Co-Authored-By: Dimitris Marlagkoutsos Co-Authored-By: Claude Opus 5 (1M context) --- .../nodejs.savepoint-interleaving.test.ts | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts diff --git a/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts b/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts new file mode 100644 index 0000000000..47087733de --- /dev/null +++ b/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; + +import { makeSQLKernelDatabase } from './nodejs.ts'; + +/** + * Why a crank and a savepoint of someone else's must never overlap. + * + * `KernelQueue.#runLoop` calls releasing its `crank` savepoint "this crank's one + * commit point". `releaseAllSavepoints` releases `t0`, which is the outermost + * savepoint only if the crank opened the first one, and `KernelStore`'s own + * `createSavepoint` bypasses `ctx.savepoints` and so is invisible to the ordinal + * numbering. Two production paths use it: `RemoteHandle.handleRemoteMessage` and + * `RemoteManager.handleIncarnationChange`. + * + * Real SQLite through the real driver, one test per interleaving. Each records + * what SQLite actually does, which is the reason the kernel now serializes the + * two: `createSavepoint` refuses inside a crank, `startCrank` refuses while a + * caller holds the store outside one, and callers take their turn through + * `beginOutOfCrank`. See `crank.out-of-crank.test.ts` for the enforcement. + */ +describe('a savepoint the crank does not know about', () => { + it('outside the crank, leaves the crank release with nothing to commit', async () => { + const kdb = await makeSQLKernelDatabase({ dbFilename: ':memory:' }); + const kv = kdb.kernelKVStore; + + // RemoteHandle.handleRemoteMessage, parked on its await. + kdb.createSavepoint('receive_r1_7'); + + // The run loop wakes: startCrank, then the two crank savepoints. + kdb.createSavepoint('t0'); + kdb.createSavepoint('t1'); + kv.set('crankWrite', 'durable'); + + // endCrank -> releaseAllSavepoints -> releaseSavepoint('t0'). `t0` is not the + // outermost savepoint, so this releases into the remote's, not to a commit. + kdb.releaseSavepoint('t0'); + + // The remote message then fails, so RemoteHandle rolls its savepoint back, + // and takes the whole committed-looking crank with it. + kdb.rollbackSavepoint('receive_r1_7'); + + expect(kv.get('crankWrite')).toBeUndefined(); + kdb.close(); + }); + + it('inside the crank, is destroyed by the delivery rollback behind its owner', async () => { + const kdb = await makeSQLKernelDatabase({ dbFilename: ':memory:' }); + const kv = kdb.kernelKVStore; + + // startCrank + kdb.createSavepoint('t0'); + kdb.createSavepoint('t1'); + + // A remote message arrives during `await deliver(queueItem)`. + kdb.createSavepoint('receive_r1_7'); + kv.set('remoteSeq.r1.highestReceivedSeq', '7'); + + // The delivery aborts: rollbackCrank('delivery') issues ROLLBACK TO t1, and + // SQLite cancels every savepoint started after t1 -- including the remote's. + kdb.rollbackSavepoint('t1'); + + // The remote handler, still inside its own try, reaches its release and + // finds the savepoint gone along with everything it wrote. + expect(() => kdb.releaseSavepoint('receive_r1_7')).toThrow( + 'No such savepoint: receive_r1_7', + ); + expect(kv.get('remoteSeq.r1.highestReceivedSeq')).toBeUndefined(); + kdb.close(); + }); + + it('inside a crank that succeeds, is committed under its owner', async () => { + const kdb = await makeSQLKernelDatabase({ dbFilename: ':memory:' }); + const kv = kdb.kernelKVStore; + + // A remote message arrives and gets as far as its await, so the seq row it + // writes "at the end, within the transaction" is not written yet. + kdb.createSavepoint('t0'); + kdb.createSavepoint('t1'); + kdb.createSavepoint('receive_r1_7'); + kv.set('remoteHalfDone', 'yes'); + + // endCrank releases t0, and with it everything stacked above. + kdb.releaseSavepoint('t0'); + + // The remote handler resumes to find its savepoint gone -- and its + // half-finished work durable regardless, so it reports a failure for an + // effect that has landed and the peer retries it. + expect(() => kdb.releaseSavepoint('receive_r1_7')).toThrow( + 'No such savepoint: receive_r1_7', + ); + expect(kv.get('remoteHalfDone')).toBe('yes'); + kdb.close(); + }); +}); From 5e7a314cb92238fb30fa06c09fa962907ce89902 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Sep 2026 18:39:49 +0200 Subject: [PATCH 31/40] fix(ocap-kernel): keep a crank and an out-of-crank savepoint from overlapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Savepoints taken through `KernelStore.createSavepoint` bypass `ctx.savepoints` and so are invisible to `createCrankSavepoint`'s ordinal naming. One held open across a crank left `releaseAllSavepoints` releasing `t0` into it rather than to a commit, so the holder's later rollback discarded the whole crank. One opened inside a crank was cancelled by a delivery rollback it had nothing to do with, or committed by a release beneath it — in both cases after the peer had been told the message was durably received. Both directions are now refused. `createSavepoint` throws inside a crank, `startCrank` throws while a caller holds the store, and the two production callers take their turn through `beginOutOfCrank`/`endOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before each crank. The run loop re-checks that gate rather than awaiting it once: a caller registers synchronously, so one arriving in a microtask queued ahead of the loop's resumption would otherwise meet `startCrank`'s refusal and kill the kernel over ordinary concurrent remote traffic. `handleRemoteMessage` decodes an incoming `redeemURL` before opening its savepoint rather than awaiting inside it. `redeemLocalOcapURL` only parses and decrypts, so the message stays atomic, and the window is now synchronous — which is the rule the whole arrangement rests on, since awaiting while holding the store would park the run loop for the duration. Trade-off: an inbound remote message or a peer handshake arriving mid-crank now waits for that crank to end, so its latency is bounded by the slowest crank rather than independent of it. Routing inbound messages through the run queue, as SwingSet's comms vat does, would remove that coupling and is the better long-term shape. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/remotes/kernel/RemoteHandle.test.ts | 52 +++++++ .../src/remotes/kernel/RemoteHandle.ts | 137 ++++++++++++------ .../src/remotes/kernel/RemoteManager.test.ts | 15 ++ .../src/remotes/kernel/RemoteManager.ts | 59 ++++---- .../src/rpc/kernel-control/get-status.test.ts | 3 + packages/ocap-kernel/src/store/index.test.ts | 3 + packages/ocap-kernel/src/store/index.ts | 9 +- .../store/methods/crank.out-of-crank.test.ts | 125 ++++++++++++++++ .../ocap-kernel/src/store/methods/crank.ts | 72 +++++++++ 9 files changed, 408 insertions(+), 67 deletions(-) create mode 100644 packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index e1e219958c..e807b17e9b 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -238,6 +238,58 @@ describe('RemoteHandle', () => { ); // Still attempted: abandoning the rollback is not a way to pass this. expect(rollbackSavepoint).toHaveBeenCalledWith('receive_r0_1'); + // And the turn is given back even on the way out. A leaked waiter leaves + // the gate unresolved, and the run loop parks on it for good. + expect(mockKernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => mockKernelStore.startCrank()).not.toThrow(); + }); + + it('gives the run loop its turn back after handling a message', async () => { + const remote = makeRemote(); + const delivery = JSON.stringify({ + seq: 1, + method: 'deliver', + params: ['bringOutYourDead'], + }); + + await remote.handleRemoteMessage(delivery); + + expect(mockKernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => mockKernelStore.startCrank()).not.toThrow(); + }); + + // The savepoint window has to stay synchronous: an await inside it parks the + // run loop for the duration and lets a crank interleave with the savepoint. + it('decodes an incoming redeemURL before opening its savepoint', async () => { + const replyKRef = mockKernelStore.initKernelObject('kernel'); + // Wrapped rather than spied: the store is hardened. + const createSavepoint = vi.fn(mockKernelStore.createSavepoint); + mockKernelStore = { ...mockKernelStore, createSavepoint } as KernelStore; + const remote = makeRemote(); + mockKernelStore.initEndpoint(remote.remoteId); + + let finishDecoding!: (kref: string) => void; + vi.spyOn(mockRemoteComms, 'redeemLocalOcapURL').mockReturnValue( + new Promise((resolve) => { + finishDecoding = resolve; + }), + ); + + const handled = remote.handleRemoteMessage( + JSON.stringify({ + seq: 1, + method: 'redeemURL', + params: ['as if it was a URL', 'replyKey'], + }), + ); + await Promise.resolve(); + + expect(createSavepoint).not.toHaveBeenCalled(); + + finishDecoding(replyKRef); + await handled; + + expect(createSavepoint).toHaveBeenCalledWith('receive_r0_1'); }); // A dead run loop will never deliver the message, and `handleRemoteMessage` diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts index ddb6d0d64a..fdcdef2c8f 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts @@ -83,6 +83,11 @@ type DeferredRedeemURLRequest = | { type: 'redeemURL'; replyKey: string; ref: ERef } | { type: 'redeemURL'; replyKey: string; error: string }; +/** What a redeemURL's URL decoded to, before anything is written down. */ +type RedeemURLResolution = + | { replyKey: string; kref: KRef } + | { replyKey: string; error: string }; + type DeferredRedeemURLReply = | { type: 'redeemURLReply'; replyKey: string; ref: KRef } | { type: 'redeemURLReply'; replyKey: string; error: string }; @@ -869,29 +874,61 @@ export class RemoteHandle implements EndpointHandle { } /** - * Prepare to handle an incoming redeemURL message. Validates and translates - * but does not send the reply. Returns data needed to complete after commit. + * Decode an incoming redeemURL message's URL. Awaits a decrypt and writes + * nothing, so it runs before the savepoint opens: a savepoint held across an + * await interleaves with the run loop's crank. * * @param url - The ocap URL attempting to be redeemed. * @param replyKey - A sender-provided tag to send with the reply. - * @returns Data needed to complete the operation after commit. + * @returns The kref the URL names, or the reason it names none. */ - async #handleRedeemURLRequest( + async #resolveRedeemURLRequest( url: string, replyKey: string, - ): Promise { + ): Promise { assert.typeof(replyKey, 'string'); - let kref: KRef; try { - kref = await this.#remoteComms.redeemLocalOcapURL(url); + return { + replyKey, + kref: await this.#remoteComms.redeemLocalOcapURL(url), + }; } catch (error) { + // Only the message crosses the wire, so the stack and cause chain stop + // here. This catch is unqualified — a bad URL and a broken decode path + // look the same to the peer — so it is logged locally to tell them apart. + this.#logger.error( + `${this.#peerId.slice(0, 8)}:: redeeming URL for ${replyKey} failed`, + error, + ); return { - type: 'redeemURL', replyKey, - error: `${(error as Error).message}`, + error: error instanceof Error ? error.message : String(error), }; } - const ref = this.#kernelStore.translateRefKtoE(this.remoteId, kref, true); + } + + /** + * Enter a resolved redeemURL request in the c-list. Synchronous, and so safe + * inside the savepoint. Returns data needed to complete after commit. + * + * @param resolution - What {@link #resolveRedeemURLRequest} decoded. + * @returns Data needed to complete the operation after commit. + */ + #recordRedeemURLRequest( + resolution: RedeemURLResolution | undefined, + ): DeferredRedeemURLRequest { + if (!resolution) { + throw Error('redeemURL reached the store without being resolved'); + } + const { replyKey } = resolution; + if ('error' in resolution) { + return { type: 'redeemURL', replyKey, error: resolution.error }; + } + const ref = this.#kernelStore.translateRefKtoE( + this.remoteId, + resolution.kref, + true, + ); return { type: 'redeemURL', replyKey, ref }; } @@ -1001,48 +1038,66 @@ export class RemoteHandle implements EndpointHandle { return null; } + // Decoded before the savepoint opens, because it awaits. It writes nothing, + // so the atomicity below is unaffected. + const redeemURLResolution = + method === 'redeemURL' + ? await this.#resolveRedeemURLRequest(...params) + : undefined; + // Wrap message processing in a transaction for atomicity: Either both (1) // message processing and (2) seq update succeed together, or neither // happens. This ensures crash-safe exactly-once delivery. + // + // The savepoint is the outermost one on the connection and so its own + // commit point, which it can only be while no crank is open — hence the + // turn taken here, and the rule that everything between the two is + // synchronous. const savepointName = `receive_${this.remoteId}_${seq}`; - this.#kernelStore.createSavepoint(savepointName); + await this.#kernelStore.beginOutOfCrank(); let deferredCompletion: DeferredCompletion | undefined; try { - switch (method) { - case 'deliver': - this.#handleRemoteDeliver(params); - break; - case 'redeemURL': - deferredCompletion = await this.#handleRedeemURLRequest(...params); - break; - case 'redeemURLReply': - deferredCompletion = this.#handleRedeemURLReply(...params); - break; - default: - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - throw Error(`unknown remote message type ${method}`); - } + this.#kernelStore.createSavepoint(savepointName); + try { + switch (method) { + case 'deliver': + this.#handleRemoteDeliver(params); + break; + case 'redeemURL': + deferredCompletion = + this.#recordRedeemURLRequest(redeemURLResolution); + break; + case 'redeemURLReply': + deferredCompletion = this.#handleRedeemURLReply(...params); + break; + default: + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw Error(`unknown remote message type ${method}`); + } - // Persist sequence tracking at the end, within the transaction - this.#kernelStore.setRemoteHighestReceivedSeq(this.remoteId, seq); + // Persist sequence tracking at the end, within the transaction + this.#kernelStore.setRemoteHighestReceivedSeq(this.remoteId, seq); - // Commit the transaction - this.#kernelStore.releaseSavepoint(savepointName); - } catch (error) { - // Rollback on any error - in-memory state unchanged since we didn't update it yet - try { - this.#kernelStore.rollbackSavepoint(savepointName); - } catch (rollbackError) { - // A failed RELEASE above discards the whole savepoint stack, so this - // rollback would report a savepoint already gone over the real error. - this.#logger.error( - `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, - rollbackError, - ); + // Commit the transaction + this.#kernelStore.releaseSavepoint(savepointName); + } catch (error) { + // Rollback on any error - in-memory state unchanged since we didn't update it yet + try { + this.#kernelStore.rollbackSavepoint(savepointName); + } catch (rollbackError) { + // A failed RELEASE above discards the whole savepoint stack, so this + // rollback would report a savepoint already gone over the real error. + this.#logger.error( + `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, + rollbackError, + ); + } + throw error; } - throw error; + } finally { + this.#kernelStore.endOutOfCrank(); } // All in-memory state changes happen after commit diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index b6b9dd08a9..97724a6d6d 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -948,6 +948,21 @@ describe('RemoteManager', () => { // finalize must not run if the persisted phase failed: in-memory // mutations would otherwise drift from the rolled-back kv view. expect(finalizeSpy).not.toHaveBeenCalled(); + // And the turn is given back on the way out. A leaked waiter leaves the + // gate unresolved, and the run loop parks on it for good. + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => kernelStore.startCrank()).not.toThrow(); + }); + + it('gives the run loop its turn back after advancing the incarnation', async () => { + const peerId = 'peer-that-restarted-cleanly'; + remoteManager.establishRemote(peerId); + kernelStore.setPeerIncarnation(peerId, 'incarnation-A'); + + await getOnIncarnationChange()(peerId, 'incarnation-B'); + + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => kernelStore.startCrank()).not.toThrow(); }); it('reports the release failure rather than a missing savepoint', async () => { diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts index 2ac1aa5aca..9b3f1fabb2 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts @@ -238,36 +238,45 @@ export class RemoteManager { ? Array.from(this.#kernelStore.getPromisesByDecider(remote.remoteId)) : []; + // The savepoint is the outermost one on the connection and so its own + // commit point, which it can only be while no crank is open. Everything + // between here and `endOutOfCrank` is synchronous, for the reason + // `beginOutOfCrank` gives. const savepoint = `peerIncarnation_${peerId}`; - this.#kernelStore.createSavepoint(savepoint); + await this.#kernelStore.beginOutOfCrank(); try { - if (isRestart) { - this.#logger?.log( - `Peer ${peerId.slice(0, 8)} restarted (incarnation ${stored.slice(0, 8)} → ${observedIncarnation.slice(0, 8)})`, - ); - if (remote) { - remote.persistPeerRestart(); - } else { - // No live RemoteHandle for the peer but a persisted incarnation - // exists — usually a transient race during kernel boot before - // initRemoteComms has finished restoring remotes. The persisted - // bookkeeping the missing handle would have cleaned up may leak. - // Surfacing as a warning so operators can correlate. - this.#logger?.warn( - `Peer ${peerId.slice(0, 8)} restart detected but no live RemoteHandle; advancing persisted incarnation without c-list cleanup`, + this.#kernelStore.createSavepoint(savepoint); + try { + if (isRestart) { + this.#logger?.log( + `Peer ${peerId.slice(0, 8)} restarted (incarnation ${stored.slice(0, 8)} → ${observedIncarnation.slice(0, 8)})`, ); + if (remote) { + remote.persistPeerRestart(); + } else { + // No live RemoteHandle for the peer but a persisted incarnation + // exists — usually a transient race during kernel boot before + // initRemoteComms has finished restoring remotes. The persisted + // bookkeeping the missing handle would have cleaned up may leak. + // Surfacing as a warning so operators can correlate. + this.#logger?.warn( + `Peer ${peerId.slice(0, 8)} restart detected but no live RemoteHandle; advancing persisted incarnation without c-list cleanup`, + ); + } } + this.#kernelStore.setPeerIncarnation(peerId, observedIncarnation); + this.#kernelStore.releaseSavepoint(savepoint); + } catch (error) { + try { + this.#kernelStore.rollbackSavepoint(savepoint); + } catch (rollbackError) { + // Same reasoning as `RemoteHandle.handleRemoteMessage`. + this.#logger?.error(`Rollback of ${savepoint} failed`, rollbackError); + } + throw error; } - this.#kernelStore.setPeerIncarnation(peerId, observedIncarnation); - this.#kernelStore.releaseSavepoint(savepoint); - } catch (error) { - try { - this.#kernelStore.rollbackSavepoint(savepoint); - } catch (rollbackError) { - // Same reasoning as `RemoteHandle.handleRemoteMessage`. - this.#logger?.error(`Rollback of ${savepoint} failed`, rollbackError); - } - throw error; + } finally { + this.#kernelStore.endOutOfCrank(); } // Post-commit fan-out: in-memory state changes and run-queue diff --git a/packages/ocap-kernel/src/rpc/kernel-control/get-status.test.ts b/packages/ocap-kernel/src/rpc/kernel-control/get-status.test.ts index 6440b1401b..2724b1de47 100644 --- a/packages/ocap-kernel/src/rpc/kernel-control/get-status.test.ts +++ b/packages/ocap-kernel/src/rpc/kernel-control/get-status.test.ts @@ -59,6 +59,9 @@ describe('getStatusHandler', () => { const store = { startCrank: vi.fn(), endCrank: vi.fn(), + beginOutOfCrank: vi.fn().mockResolvedValue(undefined), + endOutOfCrank: vi.fn(), + outOfCrankWorkPending: vi.fn().mockReturnValue(undefined), createCrankSavepoint: vi.fn(), rollbackCrank: vi.fn(), collectGarbage: vi.fn(), diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index ab7d8d040d..b19fce5d5c 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -49,6 +49,7 @@ describe('kernel store', () => { 'allocateErefForKref', 'assertRefCountsIfAuditing', 'auditRefCounts', + 'beginOutOfCrank', 'bufferCrankOutput', 'cleanupOrphanMessages', 'cleanupTerminatedVat', @@ -77,6 +78,7 @@ describe('kernel store', () => { 'dequeueRun', 'detectWake', 'endCrank', + 'endOutOfCrank', 'enqueuePromiseMessage', 'enqueueRun', 'erefToKref', @@ -152,6 +154,7 @@ describe('kernel store', () => { 'markVatAsTerminated', 'nextReapAction', 'nextTerminatedVatCleanup', + 'outOfCrankWorkPending', 'pinObject', 'provideIncarnationId', 'recomputeRefCounts', diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 0960bbbe43..7186fca951 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -68,7 +68,7 @@ * kernelService.${serviceName} = ${koid} // kref of kernel service object ${serviceName} */ -import { Fail } from '@endo/errors'; +import { Fail, q } from '@endo/errors'; import type { KernelDatabase, KVStore, VatStore } from '@metamask/kernel-store'; import { Logger } from '@metamask/logger'; @@ -300,9 +300,16 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { /** * Create a savepoint for atomic operations on persistent storage. * + * These are invisible to `createCrankSavepoint`'s ordinal naming, so one + * opened inside a crank would be rolled back by a delivery that has nothing + * to do with it — after its owner had already reported success to a peer. + * Callers take their turn through `beginOutOfCrank`. + * * @param name - The savepoint name. */ function createSavepoint(name: string): void { + !context.inCrank || + Fail`createSavepoint ${q(name)} inside a crank; use beginOutOfCrank`; kdb.createSavepoint(name); } diff --git a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts new file mode 100644 index 0000000000..567d9d7d57 --- /dev/null +++ b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts @@ -0,0 +1,125 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import { makeKernelStore } from '../index.ts'; + +/** + * A savepoint taken through `KernelStore.createSavepoint` is invisible to + * `createCrankSavepoint`'s ordinal naming, so overlapping one with a crank + * leaves the crank's release committing into someone else's transaction, or the + * caller's writes discarded by a delivery rollback it had nothing to do with. + * See `kernel-store`'s `nodejs.savepoint-interleaving.test.ts` for what SQLite + * does in each case. + * + * The two are kept apart instead: callers take their turn through + * `beginOutOfCrank`, and both directions of the overlap are refused. + */ +describe('store work outside a crank', () => { + let kernelStore: ReturnType; + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + }); + + it('refuses a savepoint taken inside a crank', () => { + kernelStore.startCrank(); + + expect(() => kernelStore.createSavepoint('receive_r1_7')).toThrow( + 'createSavepoint "receive_r1_7" inside a crank', + ); + }); + + it('refuses a crank started while a caller holds the store', async () => { + await kernelStore.beginOutOfCrank(); + + expect(() => kernelStore.startCrank()).toThrow( + 'startCrank while 1 caller(s) hold the store outside a crank', + ); + + kernelStore.endOutOfCrank(); + }); + + it('lets a caller through once the crank it arrived during has ended', async () => { + kernelStore.startCrank(); + + let held = false; + const turn = (async () => { + await kernelStore.beginOutOfCrank(); + held = true; + })(); + + await Promise.resolve(); + expect(held).toBe(false); + + kernelStore.endCrank(); + await turn; + + expect(held).toBe(true); + kernelStore.endOutOfCrank(); + }); + + it('holds the next crank off while a caller is waiting', async () => { + kernelStore.startCrank(); + const turn = kernelStore.beginOutOfCrank(); + + // What the run loop consults between `endCrank` and the next `startCrank`. + expect(kernelStore.outOfCrankWorkPending()).toBeDefined(); + + kernelStore.endCrank(); + await turn; + kernelStore.endOutOfCrank(); + + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + }); + + it('nothing to wait for while no caller is holding', () => { + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + }); + + it('lets a caller that arrives as the gate clears take its turn too', async () => { + await kernelStore.beginOutOfCrank(); + + // The run loop's protocol: re-check the gate until nothing is waiting, then + // start the crank with no await in between. + let crankStarted = false; + const runLoopTurn = (async () => { + let pending = kernelStore.outOfCrankWorkPending(); + while (pending) { + await pending; + pending = kernelStore.outOfCrankWorkPending(); + } + kernelStore.startCrank(); + crankStarted = true; + })(); + + // The first caller releases, and a second registers in the microtask that + // resolution queues — ahead of the run loop's own continuation. Checking + // the gate once would have the run loop resume into `startCrank` with this + // caller already holding, which is a refusal it cannot survive. + kernelStore.endOutOfCrank(); + await kernelStore.beginOutOfCrank(); + expect(crankStarted).toBe(false); + + kernelStore.endOutOfCrank(); + await runLoopTurn; + + expect(crankStarted).toBe(true); + }); + + it('refuses an unmatched release, which would park the run loop for good', () => { + expect(() => kernelStore.endOutOfCrank()).toThrow( + 'endOutOfCrank without beginOutOfCrank', + ); + }); + + it('holds the crank off until the last of several callers is done', async () => { + await kernelStore.beginOutOfCrank(); + await kernelStore.beginOutOfCrank(); + + kernelStore.endOutOfCrank(); + expect(kernelStore.outOfCrankWorkPending()).toBeDefined(); + + kernelStore.endOutOfCrank(); + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + }); +}); diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index df4b02a665..d30c5d977b 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -13,17 +13,86 @@ import type { CrankBufferItem, StoreContext } from '../types.ts'; */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { + // Callers waiting to take a savepoint of their own, and a gate the run loop + // holds off on starting a crank for while any of them are. The run loop is + // otherwise synchronous from `endCrank` to the next `startCrank`, so it never + // leaves a gap for them on its own. + let outOfCrankWaiters = 0; + let outOfCrankIdle: ReturnType> | undefined; + /** * Start a crank. */ function startCrank(): void { !ctx.inCrank || Fail`startCrank while already in a crank`; + // A savepoint taken outside a crank is the outermost one on the connection, + // and so the transaction's commit point. Opening a crank underneath it + // would put this crank's writes inside someone else's transaction, to be + // committed or discarded by them. + outOfCrankWaiters === 0 || + Fail`startCrank while ${q(outOfCrankWaiters)} caller(s) hold the store outside a crank`; ctx.inCrank = true; const { promise, resolve } = makePromiseKit(); ctx.crankSettled = promise; ctx.resolveCrank = resolve; } + /** + * Take a turn at the store outside any crank, for work that must be its own + * transaction: an inbound remote message, a peer's restart. Resolves only + * once no crank is open, and holds the run loop off until the matching + * {@link endOutOfCrank}. + * + * The caller's work must be synchronous. Awaiting while holding this would + * park the run loop for the duration, and awaiting between the savepoint and + * its release is the interleaving this exists to prevent. + */ + async function beginOutOfCrank(): Promise { + outOfCrankWaiters += 1; + // Created before the await below, so a run loop reaching `startCrank` + // in the meantime sees the gate rather than racing past it. + outOfCrankIdle ??= makePromiseKit(); + try { + while (ctx.inCrank) { + // Awaiting `undefined` would spin this loop as fast as the microtask + // queue allows, wedging the event loop with nothing to show for it. + ctx.crankSettled !== undefined || + Fail`inCrank with no crankSettled to wait on`; + await ctx.crankSettled; + } + } catch (error) { + // The count is incremented above, before any of this can fail, so the + // caller's `finally` has nothing to release yet. Left as it was, a gate + // that never reaches zero parks the run loop for good. + endOutOfCrank(); + throw error; + } + } + + /** + * Give the run loop the store back. Must be called for every + * {@link beginOutOfCrank}, from a `finally`. + */ + function endOutOfCrank(): void { + // An unmatched call would drive the count negative, and a gate that never + // reaches zero parks the run loop for good. + outOfCrankWaiters > 0 || Fail`endOutOfCrank without beginOutOfCrank`; + outOfCrankWaiters -= 1; + if (outOfCrankWaiters === 0) { + outOfCrankIdle?.resolve(); + outOfCrankIdle = undefined; + } + } + + /** + * @returns A promise to await before starting a crank, or undefined if + * nothing is waiting — undefined rather than a resolved promise so that the + * run loop's usual path stays synchronous. + */ + function outOfCrankWorkPending(): Promise | undefined { + return outOfCrankIdle?.promise; + } + /** * Create a savepoint in the crank. * @@ -161,6 +230,9 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { endCrank, releaseAllSavepoints, waitForCrank, + beginOutOfCrank, + endOutOfCrank, + outOfCrankWorkPending, bufferCrankOutput, flushCrankBuffer, isInCrank, From 19689c6def414a8f153191349cd43abf972d8814 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Sep 2026 18:40:02 +0200 Subject: [PATCH 32/40] test(kernel-test): reap until the shared import is released, not three times 57bf77179 replaced a fixed reap count with polling for `reapImporterUntil` and left `reapAndSettle` on three attempts. Three was enough on an idle machine and not under a loaded one: the whole suite in parallel failed `survives until both importers let go` about once in seven runs, on `main` as well as here. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/garbage-collection.test.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 2fc1dc1605..fb20462b62 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -258,10 +258,18 @@ describe('Garbage Collection', () => { * * @param vatId - The vat to reap. * @param rootKRef - That vat's root, to poke with cranks afterwards. + * @param settled - Whether the state under test has arrived yet. */ - async function reapAndSettle(vatId: VatId, rootKRef: KRef): Promise { - kernel.reapVats((id) => id === vatId); - for (let i = 0; i < 3; i++) { + async function reapAndSettle( + vatId: VatId, + rootKRef: KRef, + settled: () => boolean, + ): Promise { + // Reap until the vat's GC is visible rather than a fixed number of times: + // three was enough on an idle machine and not under a loaded one, which + // made this the last flake in the file. + for (let attempt = 0; attempt < 5 && !settled(); attempt += 1) { + kernel.reapVats((id) => id === vatId); await kernel.queueMessage(rootKRef, 'noop', []); await waitUntilQuiescent(500); } @@ -298,7 +306,11 @@ describe('Garbage Collection', () => { await kernel.queueMessage(importerKRef, 'makeWeak', [objectId]); await kernel.queueMessage(importerKRef, 'forgetImport', []); await waitUntilQuiescent(); - await reapAndSettle(importerVatId, importerKRef); + await reapAndSettle(importerVatId, importerKRef, () => + kernelStore + .getImporters(sharedKRef) + .every((vatId) => vatId !== importerVatId), + ); // The exporter must not have been told to drop it: the second importer // legitimately still holds it @@ -331,7 +343,11 @@ describe('Garbage Collection', () => { await kernel.queueMessage(secondImporterKRef, 'makeWeak', [objectId]); await kernel.queueMessage(secondImporterKRef, 'forgetImport', []); await waitUntilQuiescent(); - await reapAndSettle(secondImporterVatId, secondImporterKRef); + await reapAndSettle( + secondImporterVatId, + secondImporterKRef, + () => kernelStore.getImporters(sharedKRef).length === 0, + ); expect(kernelStore.getImporters(sharedKRef)).toStrictEqual([]); // Only the createObject result's stored value still names it From 9b4717f650a3ec3922ddb9d8d3a6141a3d83d91d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Sep 2026 18:40:19 +0200 Subject: [PATCH 33/40] docs: changelog entries for the driver, audit, and savepoint fixes Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 6 +++++- packages/ocap-kernel/CHANGELOG.md | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index a2504faf83..2f50c46905 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -14,7 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The rollback failure is still what gets thrown, even if aborting the transaction fails too - `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO`; the release failure is still what gets thrown ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - The wasm driver no longer considers itself in a transaction after a commit or abort throws, and discards the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - - Otherwise later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work. The nodejs driver reads `db.inTransaction` and was never affected + - Otherwise later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work. The nodejs driver reads `db.inTransaction` and was never affected by the wedged flag +- The nodejs driver discards the transaction a failed `COMMIT` leaves open, as the wasm driver already did ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - `releaseSavepoint` reaches `commitIfNeeded` outside any try of its own, so a throwing `COMMIT` left the transaction open with nothing to end it. Reading `db.inTransaction` prevents a wedged flag but does not end an unowned transaction +- Neither driver commits a transaction whose discarding abort failed ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - The savepoint list was emptied while SQLite still held the transaction, so the next `createSavepoint` skipped `BEGIN`, nested inside it, and committed the abandoned crank on release. Both drivers now retry the abort before beginning, and abort rather than commit while one is outstanding - Both drivers log an abort that fails while recovering from a failed savepoint operation ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index c5d80480b5..ed4ea0ad7c 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -76,6 +76,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Otherwise a cache kept the abandoned crank's value and the next write persisted it, a GC action taken out of the set for an aborted delivery was lost rather than retried, and a later `collectGarbage` killed the run loop on a promise the rollback had deleted - Cleanup after a dead crank no longer replaces the error that killed the run loop: a failing `endCrank` reports it as `cause`, and neither `endCrank` nor `rollbackCrank` retries a savepoint that a failed rollback or release has already discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Keep a crank and a savepoint taken through `KernelStore.createSavepoint` from overlapping ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - Those savepoints bypass `ctx.savepoints` and so are invisible to `createCrankSavepoint`'s ordinal naming. One open across a crank left `releaseAllSavepoints` releasing `t0` into it rather than to a commit, so the caller's later rollback discarded the whole crank; one opened inside a crank was cancelled by a delivery rollback it had nothing to do with, or made durable by a release beneath it, after the peer had already been told the message was committed + - Both directions are now refused, and the two production callers — `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change — take their turn through the new `beginOutOfCrank`/`endOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before starting each crank. Work between the two must be synchronous + - `handleRemoteMessage` now decodes an incoming `redeemURL` before opening its savepoint instead of awaiting inside it; the decode writes nothing, so the message stays atomic + - Consequence: an inbound remote message or a peer's handshake arriving mid-crank waits for that crank to end, so their latency is now bounded by the slowest crank rather than being independent of it. A crank that sends to a remote can be slow. The alternative is to route inbound messages through the run queue as SwingSet's comms vat does, which is a larger change than this one +- The reference count audit runs after the crank is committed rather than inside it, so a violation kills the run loop instead of rolling back a delivery whose caller the crank buffer flush had already answered ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - It still runs after the flush, which is what kept buffered items from reading as leaks - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) From 7579669760c840c874316a32d4905bb9d6b2145c Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 8 Sep 2026 10:40:26 +0200 Subject: [PATCH 34/40] fix(kernel-store): refuse every write while a transaction is abandoned, not just the ones that begin or commit Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 3 + .../nodejs.transaction-survival.test.ts | 82 ++++++--- packages/kernel-store/src/sqlite/nodejs.ts | 30 +++- .../sqlite/wasm.transaction-survival.test.ts | 158 ++++++++++++++++++ packages/kernel-store/src/sqlite/wasm.ts | 30 +++- 5 files changed, 277 insertions(+), 26 deletions(-) create mode 100644 packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 2f50c46905..9aaf4489c4 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `releaseSavepoint` reaches `commitIfNeeded` outside any try of its own, so a throwing `COMMIT` left the transaction open with nothing to end it. Reading `db.inTransaction` prevents a wedged flag but does not end an unowned transaction - Neither driver commits a transaction whose discarding abort failed ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - The savepoint list was emptied while SQLite still held the transaction, so the next `createSavepoint` skipped `BEGIN`, nested inside it, and committed the abandoned crank on release. Both drivers now retry the abort before beginning, and abort rather than commit while one is outstanding +- Every write refuses while a transaction the drivers could not abort is outstanding, not just the ones that pass through `beginIfNeeded` or `commitIfNeeded` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - `kernelKVStore.set`, `kernelKVStore.delete`, `clear`, `deleteVatStore`, the savepoint operations, and the nodejs driver's vatstore update joined the abandoned transaction and reported success. Teardown such as `reset` looked like it had landed, and the writes went with the transaction on close + - `executeQuery` stays exempt: its callers are debug surfaces, which would not expect a query to roll anything back - Both drivers log an abort that fails while recovering from a failed savepoint operation ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts index ba9a442597..673fb0a87e 100644 --- a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { SQL_QUERIES } from './common.ts'; import { makeSQLKernelDatabase } from './nodejs.ts'; +import type { KernelDatabase } from '../types.ts'; /** * Two invariants the crank layer relies on: @@ -123,25 +125,67 @@ describe('the nodejs driver after a failure it tolerates', () => { ]); }); - it('refuses to write once the transaction cannot be discarded at all', async () => { - const kdb = await makeSQLKernelDatabase({}); - mockDb.inTransaction = true; - mockDb._spStack = ['t0', 't1']; - issued = []; - - failOnce.add('ROLLBACK TO SAVEPOINT t1'); - failOnce.add('ROLLBACK TRANSACTION'); - expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); - - // The retry fails too, so there is no honest way to carry on: a savepoint - // taken now would be released into the abandoned transaction, committing - // the crank the abort was trying to discard. - failOnce.add('ROLLBACK TRANSACTION'); - expect(() => kdb.createSavepoint('teardown')).toThrow( - 'refusing further writes', - ); - expect(issued).not.toContain('SAVEPOINT teardown'); - }); + // Teardown after the run loop dies reaches the store by many doors, most of + // which never touch `beginIfNeeded`. + it.each([ + { + what: 'a savepoint', + write: (kdb: KernelDatabase) => kdb.createSavepoint('teardown'), + sql: 'SAVEPOINT teardown', + }, + { + what: 'a savepoint rollback', + write: (kdb: KernelDatabase) => kdb.rollbackSavepoint('t0'), + sql: 'ROLLBACK TO SAVEPOINT t0', + }, + { + what: 'a savepoint release', + write: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + sql: 'RELEASE SAVEPOINT t0', + }, + { + what: 'a kv write', + write: (kdb: KernelDatabase) => kdb.kernelKVStore.set('k', 'v'), + sql: SQL_QUERIES.SET, + }, + { + what: 'a kv delete', + write: (kdb: KernelDatabase) => kdb.kernelKVStore.delete('k'), + sql: SQL_QUERIES.DELETE, + }, + { + what: 'a clear', + write: (kdb: KernelDatabase) => kdb.clear(), + sql: SQL_QUERIES.CLEAR, + }, + { + what: 'a vatstore delete', + write: (kdb: KernelDatabase) => kdb.deleteVatStore('v1'), + sql: SQL_QUERIES.DELETE_VS_ALL, + }, + { + what: 'a vatstore update', + write: (kdb: KernelDatabase) => + kdb.makeVatStore('v1').updateKVData([['k', 'v']], []), + sql: SQL_QUERIES.SET_VS, + }, + ])( + 'refuses $what once the transaction cannot be discarded at all', + async ({ write, sql }) => { + const kdb = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['t0', 't1']; + issued = []; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => write(kdb)).toThrow('refusing further writes'); + expect(issued).not.toContain(sql); + }, + ); it('discards the transaction when the commit fails', async () => { const kdb = await makeSQLKernelDatabase({}); diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index ec17e1c117..07d9212575 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -41,9 +41,16 @@ async function initDB(dbFilename: string, logger?: Logger): Promise { * Makes a persistent {@link KVStore} on top of a SQLite database. * * @param db - The (open) database to use. + * @param options - Options for the store. + * @param options.assertWritable - Throws if this connection can no longer + * persist anything, which every write here must ask first: one issued into a + * transaction nothing can end reports success and is lost with it. * @returns A key/value store using the given database. */ -function makeKVStore(db: Database): KVStore { +function makeKVStore( + db: Database, + { assertWritable }: { assertWritable: () => void }, +): KVStore { const sqlKVInit = db.prepare(SQL_QUERIES.CREATE_TABLE); sqlKVInit.run(); @@ -93,6 +100,7 @@ function makeKVStore(db: Database): KVStore { * @param value - The value to assign to it. */ function kvSet(key: string, value: string): void { + assertWritable(); sqlKVSet.run(key, value); } @@ -104,6 +112,7 @@ function makeKVStore(db: Database): KVStore { * @param key - The key to remove. */ function kvDelete(key: string): void { + assertWritable(); sqlKVDelete.run(key); } @@ -135,7 +144,7 @@ export async function makeSQLKernelDatabase({ }): Promise { const db = await initDB(dbFilename ?? DEFAULT_DB_FILENAME, logger); - const kvStore = makeKVStore(db); + const kvStore = makeKVStore(db, { assertWritable: assertNotAbandoned }); const sqlKVInitVS = db.prepare(SQL_QUERIES.CREATE_TABLE_VS); sqlKVInitVS.run(); @@ -256,6 +265,17 @@ export async function makeSQLKernelDatabase({ sqlKVClearVS.run(); } + const kvClearInTransaction = db.transaction(kvClear); + + /** + * Delete everything from the database, refusing rather than handing the + * deletes to a transaction that will never commit. + */ + function clear(): void { + assertNotAbandoned(); + kvClearInTransaction(); + } + /** * Execute an arbitrary query and return the results. * @@ -300,6 +320,7 @@ export async function makeSQLKernelDatabase({ * @param deletes - A set of keys that have been deleted. */ function updateKVData(sets: [string, string][], deletes: string[]): void { + assertNotAbandoned(); db.transaction(() => { for (const [key, value] of sets) { sqlVatstoreSet.run(vatID, key, value); @@ -322,6 +343,7 @@ export async function makeSQLKernelDatabase({ * @param vatId - The vat whose store is to be deleted. */ function deleteVatStore(vatId: string): void { + assertNotAbandoned(); sqlVatstoreDeleteAll.run(vatId); } @@ -347,6 +369,7 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function rollbackSavepoint(name: string): void { + assertNotAbandoned(); assertSafeIdentifier(name); const idx = db._spStack.lastIndexOf(name); if (idx < 0) { @@ -376,6 +399,7 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function releaseSavepoint(name: string): void { + assertNotAbandoned(); assertSafeIdentifier(name); const idx = db._spStack.lastIndexOf(name); if (idx < 0) { @@ -399,7 +423,7 @@ export async function makeSQLKernelDatabase({ return { kernelKVStore: kvStore, executeQuery: kvExecuteQuery, - clear: db.transaction(kvClear), + clear, makeVatStore, deleteVatStore, createSavepoint, diff --git a/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts new file mode 100644 index 0000000000..7c545e1104 --- /dev/null +++ b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts @@ -0,0 +1,158 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +import { SQL_QUERIES } from './common.ts'; +import { makeSQLKernelDatabase } from './wasm.ts'; +import type { KernelDatabase } from '../types.ts'; + +/** + * The nodejs sibling of this file states the invariants both drivers owe the + * crank layer. This one exists because the two drivers reach them differently: + * where nodejs asks SQLite whether a transaction is open, wasm caches the + * answer in `_inTx`, and its vatstore writes go through `safeMutate` rather + * than a driver-level transaction helper. + */ + +/** Every statement and exec call, in order. */ +let issued: string[] = []; +/** SQL that throws when next run. */ +let failOnce: Set = new Set(); + +/** + * Run one statement against the mock. A statement that throws changes nothing, + * as in SQLite; the driver keeps its own view of the transaction in `_inTx`, + * which is the thing under test and so must not be modelled here. + * + * @param text - The SQL being run. + */ +function runSql(text: string): void { + issued.push(text); + if (failOnce.delete(text)) { + throw new Error(`SQLITE_IOERR: ${text}`); + } +} + +const makeStatement = (text: string): Record => ({ + bind: vi.fn(), + step: () => { + runSql(text); + return false; + }, + reset: vi.fn(), + get: vi.fn(), + getString: vi.fn(), + getColumnName: vi.fn(), + columnCount: 0, +}); + +const mockDb = { + prepare: vi.fn((text: string) => makeStatement(text)), + exec: vi.fn(runSql), + _inTx: false, + _spStack: [] as string[], + close: vi.fn(), +}; + +const DbMock = vi.fn(function () { + return mockDb; +}); + +vi.mock('@sqlite.org/sqlite-wasm', () => ({ + default: vi.fn(async () => ({ oo1: { OpfsDb: DbMock, DB: DbMock } })), +})); +vi.mock('./env.ts', () => ({ getDBFolder: vi.fn(() => 'test-folder') })); + +describe('the wasm driver after a failure it tolerates', () => { + beforeEach(() => { + issued = []; + failOnce = new Set(); + mockDb._inTx = false; + mockDb._spStack = []; + }); + + // Teardown after the run loop dies reaches the store by many doors, most of + // which never touch `beginIfNeeded`. + it.each([ + { + what: 'a savepoint', + write: (kdb: KernelDatabase) => kdb.createSavepoint('teardown'), + sql: 'SAVEPOINT teardown', + }, + { + what: 'a savepoint rollback', + write: (kdb: KernelDatabase) => kdb.rollbackSavepoint('t0'), + sql: 'ROLLBACK TO SAVEPOINT t0', + }, + { + what: 'a savepoint release', + write: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + sql: 'RELEASE SAVEPOINT t0', + }, + { + what: 'a kv write', + write: (kdb: KernelDatabase) => kdb.kernelKVStore.set('k', 'v'), + sql: SQL_QUERIES.SET, + }, + { + what: 'a kv delete', + write: (kdb: KernelDatabase) => kdb.kernelKVStore.delete('k'), + sql: SQL_QUERIES.DELETE, + }, + { + what: 'a clear', + write: (kdb: KernelDatabase) => kdb.clear(), + sql: SQL_QUERIES.CLEAR, + }, + { + what: 'a vatstore delete', + write: (kdb: KernelDatabase) => kdb.deleteVatStore('v1'), + sql: SQL_QUERIES.DELETE_VS_ALL, + }, + { + what: 'a vatstore update', + write: (kdb: KernelDatabase) => + kdb.makeVatStore('v1').updateKVData([['k', 'v']], []), + sql: SQL_QUERIES.SET_VS, + }, + ])( + 'refuses $what once the transaction cannot be discarded at all', + async ({ write, sql }) => { + const kdb = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['t0', 't1']; + issued = []; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => write(kdb)).toThrow('refusing further writes'); + expect(issued).not.toContain(sql); + }, + ); + + it('discards the transaction and lets the next savepoint begin its own', async () => { + const kdb = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['t0', 't1']; + issued = []; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + issued = []; + kdb.createSavepoint('teardown'); + kdb.releaseSavepoint('teardown'); + + // The abort is retried and succeeds, so what the COMMIT makes durable is + // `teardown`'s own transaction and not the crank that was thrown away. + expect(issued).toStrictEqual([ + 'ROLLBACK TRANSACTION', + 'BEGIN TRANSACTION', + 'SAVEPOINT teardown', + 'RELEASE SAVEPOINT teardown', + 'COMMIT TRANSACTION', + ]); + }); +}); diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index a9a5db1d29..67a3ac3368 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -51,10 +51,20 @@ export async function initDB( * Makes a {@link KVStore} on top of a SQLite database * * @param db - The (open) database to use. - * @param logger - A logger object for recording activity. + * @param options - Options for the store. + * @param options.assertWritable - Throws if this connection can no longer + * persist anything, which every write here must ask first: one issued into a + * transaction nothing can end reports success and is lost with it. + * @param options.logger - A logger object for recording activity. * @returns A key/value store using the given database. */ -function makeKVStore(db: Database, logger?: Logger): KVStore { +function makeKVStore( + db: Database, + { + assertWritable, + logger, + }: { assertWritable: () => void; logger?: Logger | undefined }, +): KVStore { db.exec(SQL_QUERIES.CREATE_TABLE); const sqlKVGet = db.prepare(SQL_QUERIES.GET); @@ -116,6 +126,7 @@ function makeKVStore(db: Database, logger?: Logger): KVStore { * @param value - The value to assign to it. */ function kvSet(key: string, value: string): void { + assertWritable(); logger?.debug(`kv set '${key}' to '${value}'`); sqlKVSet.bind([key, value]); sqlKVSet.step(); @@ -130,6 +141,7 @@ function makeKVStore(db: Database, logger?: Logger): KVStore { * @param key - The key to remove. */ function kvDelete(key: string): void { + assertWritable(); logger?.debug(`kv delete '${key}'`); sqlKVDelete.bind([key]); sqlKVDelete.step(); @@ -165,7 +177,10 @@ export async function makeSQLKernelDatabase({ const db = await initDB(dbFilename ?? DEFAULT_DB_FILENAME, logger); logger?.debug('Initializing kernel store'); - const kvStore = makeKVStore(db, logger?.subLogger({ tags: ['kv'] })); + const kvStore = makeKVStore(db, { + assertWritable: assertNotAbandoned, + logger: logger?.subLogger({ tags: ['kv'] }), + }); db.exec(SQL_QUERIES.CREATE_TABLE_VS); @@ -318,6 +333,7 @@ export async function makeSQLKernelDatabase({ * Delete everything from the database. */ function kvClear(): void { + assertNotAbandoned(); logger?.debug('clearing all kernel state'); sqlKVClear.step(); sqlKVClear.reset(); @@ -326,7 +342,10 @@ export async function makeSQLKernelDatabase({ } /** - * Execute a SQL query. + * Execute a SQL query. Unlike the other write paths this one does not consult + * `assertNotAbandoned`, whose side effect is a rollback: the debug surfaces + * that call it would not expect a query to end a transaction. `step` will run + * DML given it, where the nodejs driver's `all` refuses anything but a SELECT. * * @param sql - The SQL query to execute. * @returns An array of results. @@ -413,6 +432,7 @@ export async function makeSQLKernelDatabase({ * @param vatId - The vat whose store is to be deleted. */ function deleteVatStore(vatId: string): void { + assertNotAbandoned(); sqlVatstoreDeleteAll.bind([vatId]); sqlVatstoreDeleteAll.step(); sqlVatstoreDeleteAll.reset(); @@ -440,6 +460,7 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function rollbackSavepoint(name: string): void { + assertNotAbandoned(); assertSafeIdentifier(name); const idx = db._spStack.lastIndexOf(name); if (idx < 0) { @@ -469,6 +490,7 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function releaseSavepoint(name: string): void { + assertNotAbandoned(); assertSafeIdentifier(name); const idx = db._spStack.lastIndexOf(name); if (idx < 0) { From 76947581af0cc51d21dea1324d6ffc6f2c69a9b5 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 8 Sep 2026 10:40:36 +0200 Subject: [PATCH 35/40] fix(ocap-kernel): let `stop` release the workers when the store refuses its timestamp Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 2 ++ packages/ocap-kernel/src/Kernel.test.ts | 21 +++++++++++++++++++++ packages/ocap-kernel/src/Kernel.ts | 10 +++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index ed4ea0ad7c..f62b8098f1 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -61,6 +61,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `Kernel.stop` records the last active time on a best-effort basis, logging rather than throwing when the store refuses the write ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) + - A store that can no longer persist anything refuses every write, and this one sits ahead of stopping remote comms, terminating the vat workers, and closing the database. The timestamp is meaningless once the state it dates is unreachable; the worker processes are not - A message delivered to a kernel-owned kref with no registered service now rejects the caller with `ENDPOINT_UNREACHABLE` instead of throwing, which escaped the crank and killed the run loop — turning one unreachable reference into a dead kernel ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007)) - Reachable without any kernel bug: an anonymous kernel object hosts something that cannot outlive the process, such as an accepted socket connection, so a vat holding one across a restart or a message to one still queued from the previous incarnation lands here. That surviving reference is exactly what stops the init sweep deleting the object, so its `kernel` owner survives with it - Matches what `KernelRouter` already does for a delivery whose endpoint has vanished. A message sent with no result promise has nobody to report to, so it is logged instead diff --git a/packages/ocap-kernel/src/Kernel.test.ts b/packages/ocap-kernel/src/Kernel.test.ts index d85cb02234..6cf8892062 100644 --- a/packages/ocap-kernel/src/Kernel.test.ts +++ b/packages/ocap-kernel/src/Kernel.test.ts @@ -787,6 +787,27 @@ describe('Kernel', () => { expect(timestamp).toBeGreaterThanOrEqual(before); expect(timestamp).toBeLessThanOrEqual(after); }); + + it('releases the vat workers even when the store refuses the timestamp', async () => { + const workerTerminateAllMock = vi + .spyOn(mockPlatformServices, 'terminateAll') + .mockResolvedValue(undefined); + const kernel = await Kernel.make( + mockPlatformServices, + mockKernelDatabase, + ); + // What a driver holding a transaction it could not abort does to every + // write, teardown's included. + vi.spyOn(mockKernelDatabase.kernelKVStore, 'set').mockImplementation( + () => { + throw new Error('refusing further writes on this connection'); + }, + ); + + await kernel.stop(); + + expect(workerTerminateAllMock).toHaveBeenCalledOnce(); + }); }); describe('restartVat()', () => { diff --git a/packages/ocap-kernel/src/Kernel.ts b/packages/ocap-kernel/src/Kernel.ts index c041d060be..d44467b88f 100644 --- a/packages/ocap-kernel/src/Kernel.ts +++ b/packages/ocap-kernel/src/Kernel.ts @@ -862,7 +862,15 @@ export class Kernel { */ async stop(): Promise { await this.#kernelQueue.waitForCrank(); - this.#kernelStore.recordLastActiveTime(); + try { + this.#kernelStore.recordLastActiveTime(); + } catch (error) { + // A store that can no longer persist anything refuses this write, and + // everything below it releases something: remote comms, the vat workers, + // the database handle. A timestamp is not worth leaking those over, and + // is meaningless anyway once the state it dates is unreachable. + this.#logger.error('could not record last active time', error); + } await this.#platformServices.stopRemoteComms(); this.#remoteManager.cleanup(); await this.#platformServices.terminateAll(); From 99eeb9c1a0602566447c377c744d9d43a5138860 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 11 Sep 2026 13:42:21 +0200 Subject: [PATCH 36/40] chore: point this PR's changelog links at the repo's new org The repo moved to Consensys-Incorporated. Only the links this branch adds are rewritten; entries for PRs that really did live at MetaMask/ocap-kernel keep naming it. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 2 +- packages/kernel-node-runtime/CHANGELOG.md | 2 +- packages/kernel-store/CHANGELOG.md | 12 ++++++------ packages/ocap-kernel/CHANGELOG.md | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index bd09c572be..d13c38154c 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index 643a7042d1..fbc3f4dbf9 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics reach the log rather than nowhere ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 9aaf4489c4..0016c482ca 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,17 +12,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO`; the release failure is still what gets thrown ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) -- The wasm driver no longer considers itself in a transaction after a commit or abort throws, and discards the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO`; the release failure is still what gets thrown ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) +- The wasm driver no longer considers itself in a transaction after a commit or abort throws, and discards the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Otherwise later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work. The nodejs driver reads `db.inTransaction` and was never affected by the wedged flag -- The nodejs driver discards the transaction a failed `COMMIT` leaves open, as the wasm driver already did ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The nodejs driver discards the transaction a failed `COMMIT` leaves open, as the wasm driver already did ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - `releaseSavepoint` reaches `commitIfNeeded` outside any try of its own, so a throwing `COMMIT` left the transaction open with nothing to end it. Reading `db.inTransaction` prevents a wedged flag but does not end an unowned transaction -- Neither driver commits a transaction whose discarding abort failed ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Neither driver commits a transaction whose discarding abort failed ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - The savepoint list was emptied while SQLite still held the transaction, so the next `createSavepoint` skipped `BEGIN`, nested inside it, and committed the abandoned crank on release. Both drivers now retry the abort before beginning, and abort rather than commit while one is outstanding -- Every write refuses while a transaction the drivers could not abort is outstanding, not just the ones that pass through `beginIfNeeded` or `commitIfNeeded` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Every write refuses while a transaction the drivers could not abort is outstanding, not just the ones that pass through `beginIfNeeded` or `commitIfNeeded` ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - `kernelKVStore.set`, `kernelKVStore.delete`, `clear`, `deleteVatStore`, the savepoint operations, and the nodejs driver's vatstore update joined the abandoned transaction and reported success. Teardown such as `reset` looked like it had landed, and the writes went with the transaction on close - `executeQuery` stays exempt: its callers are debug surfaces, which would not expect a query to roll anything back -- Both drivers log an abort that fails while recovering from a failed savepoint operation ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Both drivers log an abort that fails while recovering from a failed savepoint operation ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index f62b8098f1..41054cda63 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -61,7 +61,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `Kernel.stop` records the last active time on a best-effort basis, logging rather than throwing when the store refuses the write ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `Kernel.stop` records the last active time on a best-effort basis, logging rather than throwing when the store refuses the write ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - A store that can no longer persist anything refuses every write, and this one sits ahead of stopping remote comms, terminating the vat workers, and closing the database. The timestamp is meaningless once the state it dates is unreachable; the worker processes are not - A message delivered to a kernel-owned kref with no registered service now rejects the caller with `ENDPOINT_UNREACHABLE` instead of throwing, which escaped the crank and killed the run loop — turning one unreachable reference into a dead kernel ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007)) - Reachable without any kernel bug: an anonymous kernel object hosts something that cannot outlive the process, such as an accepted socket connection, so a vat holding one across a restart or a message to one still queued from the previous incarnation lands here. That surviving reference is exactly what stops the init sweep deleting the object, so its `kernel` owner survives with it @@ -72,18 +72,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction, so terminating the vat and collecting garbage after an aborted delivery are committed with the crank instead of autocommitting a statement at a time ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Keep a crank's store work inside one transaction, so terminating the vat and collecting garbage after an aborted delivery are committed with the crank instead of autocommitting a statement at a time ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately -- `rollbackCrank` now also reverts the state a database rollback cannot reach, whether or not the rollback itself succeeded: cached stored values are re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- `rollbackCrank` now also reverts the state a database rollback cannot reach, whether or not the rollback itself succeeded: cached stored values are re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Otherwise a cache kept the abandoned crank's value and the next write persisted it, a GC action taken out of the set for an aborted delivery was lost rather than retried, and a later `collectGarbage` killed the run loop on a promise the rollback had deleted -- Cleanup after a dead crank no longer replaces the error that killed the run loop: a failing `endCrank` reports it as `cause`, and neither `endCrank` nor `rollbackCrank` retries a savepoint that a failed rollback or release has already discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) -- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) -- Keep a crank and a savepoint taken through `KernelStore.createSavepoint` from overlapping ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- Cleanup after a dead crank no longer replaces the error that killed the run loop: a failing `endCrank` reports it as `cause`, and neither `endCrank` nor `rollbackCrank` retries a savepoint that a failed rollback or release has already discarded ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) +- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) +- Keep a crank and a savepoint taken through `KernelStore.createSavepoint` from overlapping ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Those savepoints bypass `ctx.savepoints` and so are invisible to `createCrankSavepoint`'s ordinal naming. One open across a crank left `releaseAllSavepoints` releasing `t0` into it rather than to a commit, so the caller's later rollback discarded the whole crank; one opened inside a crank was cancelled by a delivery rollback it had nothing to do with, or made durable by a release beneath it, after the peer had already been told the message was committed - Both directions are now refused, and the two production callers — `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change — take their turn through the new `beginOutOfCrank`/`endOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before starting each crank. Work between the two must be synchronous - `handleRemoteMessage` now decodes an incoming `redeemURL` before opening its savepoint instead of awaiting inside it; the decode writes nothing, so the message stays atomic - Consequence: an inbound remote message or a peer's handshake arriving mid-crank waits for that crank to end, so their latency is now bounded by the slowest crank rather than being independent of it. A crank that sends to a remote can be slow. The alternative is to route inbound messages through the run queue as SwingSet's comms vat does, which is a larger change than this one -- The reference count audit runs after the crank is committed rather than inside it, so a violation kills the run loop instead of rolling back a delivery whose caller the crank buffer flush had already answered ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) +- The reference count audit runs after the crank is committed rather than inside it, so a violation kills the run loop instead of rolling back a delivery whose caller the crank buffer flush had already answered ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - It still runs after the flush, which is what kept buffered items from reading as leaks - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts From 33fd2e158b4a7fac0d826be4056098113345e0ab Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 14 Sep 2026 16:35:49 +0200 Subject: [PATCH 37/40] fix(kernel-store): ask SQLite whether a transaction is open, rather than caching it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite ends a transaction itself after SQLITE_FULL, SQLITE_IOERR and SQLITE_BUSY. The wasm driver's `ROLLBACK` was refused as a result, it read that refusal as a transaction it could not end, and latched `txAbandoned` — refusing every write for the life of the worker, on a healthy database. That driver is the browser extension's kernel store. It now reads `sqlite3_get_autocommit` the way the nodejs driver reads `db.inTransaction`, which leaves the two with the same algorithm. Also drops better-sqlite3's `verbose`, which #1021 switched on by accident when it began passing a logger from the production runtimes. The wasm driver's savepoint semantics are now tested against the real wasm build; the mocked files keep the cases that need injected I/O failures, with their mocks tracking the transaction the way SQLite does. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 10 +- .../nodejs.transaction-survival.test.ts | 56 ++- packages/kernel-store/src/sqlite/nodejs.ts | 12 +- packages/kernel-store/src/sqlite/wasm.test.ts | 328 +----------------- .../sqlite/wasm.transaction-survival.test.ts | 149 +++++++- .../src/sqlite/wasm.transactions.test.ts | 89 +++++ packages/kernel-store/src/sqlite/wasm.ts | 81 +++-- .../methods/crank.cross-crank-gc.test.ts | 85 +++++ .../src/store/methods/crank.test.ts | 48 ++- 9 files changed, 456 insertions(+), 402 deletions(-) create mode 100644 packages/kernel-store/src/sqlite/wasm.transactions.test.ts create mode 100644 packages/ocap-kernel/src/store/methods/crank.cross-crank-gc.test.ts diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 0016c482ca..ba26b8ff90 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -13,10 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too - `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO`; the release failure is still what gets thrown ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) -- The wasm driver no longer considers itself in a transaction after a commit or abort throws, and discards the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - - Otherwise later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work. The nodejs driver reads `db.inTransaction` and was never affected by the wedged flag -- The nodejs driver discards the transaction a failed `COMMIT` leaves open, as the wasm driver already did ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - - `releaseSavepoint` reaches `commitIfNeeded` outside any try of its own, so a throwing `COMMIT` left the transaction open with nothing to end it. Reading `db.inTransaction` prevents a wedged flag but does not end an unowned transaction +- The wasm driver asks SQLite whether a transaction is open instead of caching the answer, so an error SQLite recovers from no longer refuses every write for the life of the worker ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - SQLite ends the transaction itself after `SQLITE_FULL`, `SQLITE_IOERR` and `SQLITE_BUSY`. The driver's `ROLLBACK` was refused as a result, it read that refusal as a transaction it could not end, and latched — on a healthy database, in the browser extension's kernel store. The nodejs driver already read `db.inTransaction`; both now recover once the transaction is gone +- Both drivers discard the transaction a failed `COMMIT` leaves open ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - `releaseSavepoint` reaches `commitIfNeeded` outside any try of its own, so a throwing `COMMIT` left the transaction open with nothing to end it. Later writes autocommitted — including the next savepoint, created bare, where `RELEASE` commits and no rollback can undo the work +- The nodejs driver no longer logs every SQL statement it executes ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - Handing it a logger for the error paths above also enabled better-sqlite3's `verbose` option, which fires on every statement with values inlined, at `info`. Kernel store rows carry vat state, c-list entries and capability data - Neither driver commits a transaction whose discarding abort failed ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - The savepoint list was emptied while SQLite still held the transaction, so the next `createSavepoint` skipped `BEGIN`, nested inside it, and committed the abandoned crank on release. Both drivers now retry the abort before beginning, and abort rather than commit while one is outstanding - Every write refuses while a transaction the drivers could not abort is outstanding, not just the ones that pass through `beginIfNeeded` or `commitIfNeeded` ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) diff --git a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts index 673fb0a87e..27cdd35a01 100644 --- a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts @@ -1,3 +1,4 @@ +import type { Logger } from '@metamask/logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import { SQL_QUERIES } from './common.ts'; @@ -7,15 +8,13 @@ import type { KernelDatabase } from '../types.ts'; /** * Two invariants the crank layer relies on: * - * - a failed `ROLLBACK TO` discards the whole transaction, so truncating - * `ctx.savepoints` to zero still matches the database; + * - a failed `ROLLBACK TO` discards the whole transaction; * - `commitIfNeeded` leaves no transaction behind. * - * Both drivers catch and log an abort that fails while discarding a transaction, - * so "the whole transaction is discarded" holds only when that abort succeeds. - * This driver has no `_inTx` flag, reading `db.inTransaction` from SQLite - * instead, which prevents a wedged flag but does not by itself end an ownerless - * transaction. + * Both hold only while the abort doing the discarding succeeds. When it does + * not, the driver logs it and refuses every later write rather than let one + * join a transaction nothing will commit; `ctx.savepoints` is truncated to + * zero either way, so it no longer matches the database. */ /** Every statement and exec call, in order. */ @@ -202,4 +201,47 @@ describe('the nodejs driver after a failure it tolerates', () => { expect(issued).toContain('ROLLBACK TRANSACTION'); expect(mockDb.inTransaction).toBe(false); }); + + // SQLite ending the transaction is the only way out of an abort that keeps + // failing. Refusing writes past that point refuses them forever. + it('writes again once the transaction it could not end is gone', async () => { + const kdb = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['t0', 't1']; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + mockDb.inTransaction = false; + + issued = []; + kdb.kernelKVStore.set('k', 'v'); + expect(issued).toStrictEqual([SQL_QUERIES.SET]); + }); + + it('reports the abort that failed while discarding the transaction', async () => { + const logger = { + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + subLogger: vi.fn(() => logger), + } as unknown as Logger; + const kdb = await makeSQLKernelDatabase({ logger }); + mockDb.inTransaction = true; + mockDb._spStack = ['t0']; + + failOnce.add('COMMIT TRANSACTION'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.releaseSavepoint('t0')).toThrow( + 'SQLITE_IOERR: COMMIT TRANSACTION', + ); + + expect(logger.error).toHaveBeenCalledWith( + 'failed to discard transaction after commit', + expect.objectContaining({ + message: 'SQLITE_IOERR: ROLLBACK TRANSACTION', + }), + ); + }); }); diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index 07d9212575..23f4b417e1 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -28,11 +28,9 @@ export type Database = SqliteDatabase & { async function initDB(dbFilename: string, logger?: Logger): Promise { const dbPath = await getDBFilename(dbFilename); logger?.debug('dbPath:', dbPath); - const db = new Sqlite(dbPath, { - verbose: (logger ? logger.info.bind(logger) : undefined) as - | ((...args: unknown[]) => void) - | undefined, - }) as Database; + // No `verbose`: it fires on every statement with values inlined, and kernel + // store rows carry vat state and c-list entries. + const db = new Sqlite(dbPath) as Database; db._spStack = []; return db; } @@ -159,9 +157,7 @@ export async function makeSQLKernelDatabase({ const sqlCommitTransaction = db.prepare(SQL_QUERIES.COMMIT_TRANSACTION); const sqlAbortTransaction = db.prepare(SQL_QUERIES.ABORT_TRANSACTION); - // Set when an abort meant to discard a transaction fails. Reading - // `db.inTransaction` keeps this driver from wedging a flag the way a cached - // one can, but it cannot end a transaction nothing owns: the writes of the + // Set when an abort meant to discard a transaction fails. The writes of the // crank we gave up on are still in it, and a savepoint taken inside it would // be released into it. let txAbandoned = false; diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 921c3f6430..b4a20f99fa 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -26,11 +26,13 @@ const mockStatement = { columnCount: 2, }; +let txOpen = false; + const mockDb = { exec: vi.fn(), prepare: vi.fn(() => mockStatement), - _inTx: false, + pointer: 1, _spStack: [] as string[], close: vi.fn(), @@ -47,6 +49,7 @@ vi.mock('@sqlite.org/sqlite-wasm', () => ({ OpfsDb: OpfsDbMock, DB: DBMock, }, + capi: { sqlite3_get_autocommit: () => (txOpen ? 0 : 1) }, })), })); @@ -81,6 +84,7 @@ describe('makeSQLKernelDatabase', () => { return mockDb; }), }, + capi: { sqlite3_get_autocommit: () => (txOpen ? 0 : 1) }, }) as unknown as Sqlite3Static, ); const logger = { @@ -368,322 +372,6 @@ describe('makeSQLKernelDatabase', () => { }); }); - describe('savepoint functionality', () => { - beforeEach(() => { - mockDb.exec.mockClear(); - mockDb._inTx = false; - mockDb._spStack = []; - }); - - it('creates a savepoint using sanitized name', async () => { - const db = await makeSQLKernelDatabase({}); - db.createSavepoint('valid_name'); - - expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT valid_name'); - }); - - it('rejects invalid savepoint names', async () => { - const db = await makeSQLKernelDatabase({}); - expect(() => db.createSavepoint('invalid-name')).toThrowError( - 'Invalid identifier', - ); - expect(() => db.createSavepoint('123numeric')).toThrowError( - 'Invalid identifier', - ); - expect(() => db.createSavepoint('spaces not allowed')).toThrowError( - 'Invalid identifier', - ); - expect(() => db.createSavepoint("point'; DROP TABLE kv--")).toThrowError( - 'Invalid identifier', - ); - expect(mockDb.exec).not.toHaveBeenCalledWith( - expect.stringContaining('DROP TABLE'), - ); - }); - - it('rolls back to a savepoint', async () => { - const db = await makeSQLKernelDatabase({}); - db.createSavepoint('test_point'); - db.rollbackSavepoint('test_point'); - expect(mockDb.exec).toHaveBeenCalledWith( - 'ROLLBACK TO SAVEPOINT test_point', - ); - }); - - it('releases a savepoint', async () => { - const db = await makeSQLKernelDatabase({}); - db.createSavepoint('test_point'); - db.releaseSavepoint('test_point'); - expect(mockDb.exec).toHaveBeenCalledWith('RELEASE SAVEPOINT test_point'); - }); - - it('createSavepoint begins transaction if needed', async () => { - const db = await makeSQLKernelDatabase({}); - db.createSavepoint('test_point'); - expect(mockDb._inTx).toBe(true); - expect(mockDb._spStack).toContain('test_point'); - expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT test_point'); - }); - - it('rollbackSavepoint validates savepoint exists', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['existing_point']; - expect(() => db.rollbackSavepoint('nonexistent_point')).toThrowError( - 'No such savepoint: nonexistent_point', - ); - }); - - it('rollbackSavepoint removes all points after target', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1', 'point2', 'point3']; - db.rollbackSavepoint('point2'); - expect(mockDb._spStack).toStrictEqual(['point1']); - expect(mockDb.exec).toHaveBeenCalledWith('ROLLBACK TO SAVEPOINT point2'); - }); - - it('rollbackSavepoint closes transaction if no savepoints remain', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - db.rollbackSavepoint('point1'); - expect(mockDb._spStack).toStrictEqual([]); - expect(mockDb._inTx).toBe(false); - }); - - // Otherwise every later write on this connection joins a transaction nothing - // will ever commit, reports success, and vanishes on close. - it('rollbackSavepoint discards the transaction when the rollback fails', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - - expect(() => db.rollbackSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._spStack).toStrictEqual([]); - expect(mockDb._inTx).toBe(false); - }); - - // The rollback failure is the diagnosis; a failed abort on top of it only - // repeats that the same connection is broken. - it('rollbackSavepoint reports the rollback failure even if the abort fails too', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - mockStatement.step.mockImplementationOnce(() => { - throw new Error('cannot rollback'); - }); - - expect(() => db.rollbackSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._spStack).toStrictEqual([]); - }); - - it('releaseSavepoint validates savepoint exists', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['existing_point']; - expect(() => db.releaseSavepoint('nonexistent_point')).toThrowError( - 'No such savepoint: nonexistent_point', - ); - }); - - it('releaseSavepoint removes all points after target', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1', 'point2', 'point3']; - db.releaseSavepoint('point2'); - expect(mockDb._spStack).toStrictEqual(['point1']); - expect(mockDb.exec).toHaveBeenCalledWith('RELEASE SAVEPOINT point2'); - }); - - it('releaseSavepoint commits transaction if no savepoints remain', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - db.releaseSavepoint('point1'); - expect(mockDb._spStack).toStrictEqual([]); - expect(mockDb._inTx).toBe(false); - }); - - it('releaseSavepoint discards the transaction when the release fails', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - - expect(() => db.releaseSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._spStack).toStrictEqual([]); - expect(mockDb._inTx).toBe(false); - }); - - it('releaseSavepoint reports the release failure even if the abort fails too', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - mockStatement.step.mockImplementationOnce(() => { - throw new Error('cannot rollback'); - }); - - expect(() => db.releaseSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._inTx).toBe(false); - }); - - it('stops believing it is in a transaction when the abort fails too', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - mockStatement.step.mockImplementationOnce(() => { - throw new Error('cannot rollback'); - }); - - expect(() => db.rollbackSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._inTx).toBe(false); - }); - - it('retries the abort before beginning the next savepoint', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockDb.exec.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - mockStatement.step.mockImplementationOnce(() => { - throw new Error('cannot rollback'); - }); - expect(() => db.rollbackSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - mockDb.exec.mockClear(); - mockStatement.step.mockClear(); - db.createSavepoint('next'); - - // The abort that failed above may have left SQLite holding a transaction - // `_inTx` no longer accounts for, and BEGIN inside one throws. So the - // abort is retried first: two prepared statements, that and BEGIN. The - // SAVEPOINT itself goes through `exec`. - expect(mockStatement.step).toHaveBeenCalledTimes(2); - expect(mockDb._inTx).toBe(true); - expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); - }); - - it('stops believing it is in a transaction when the commit fails', async () => { - const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - // The RELEASE goes through `exec`; COMMIT is the first prepared statement - // this path steps. - mockStatement.step.mockImplementationOnce(() => { - throw new Error('disk I/O error'); - }); - - expect(() => db.releaseSavepoint('point1')).toThrowError( - 'disk I/O error', - ); - - expect(mockDb._inTx).toBe(false); - - mockDb.exec.mockClear(); - mockStatement.step.mockClear(); - db.createSavepoint('next'); - expect(mockStatement.step).toHaveBeenCalledOnce(); - expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); - }); - - it('releaseSavepoint discards the transaction when the commit fails', async () => { - const db = await makeSQLKernelDatabase({}); - mockStatement.step.mockClear(); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - // The RELEASE goes through `exec`; COMMIT is the first prepared statement - // this path steps. - mockStatement.step.mockImplementationOnce(() => { - throw new Error('database is locked'); - }); - - expect(() => db.releaseSavepoint('point1')).toThrowError( - 'database is locked', - ); - - // Every statement shares one mock, so the ABORT is only visible as a - // second step. - expect(mockStatement.step).toHaveBeenCalledTimes(2); - expect(mockDb._inTx).toBe(false); - }); - - it('releaseSavepoint reports the commit failure even if the abort fails too', async () => { - const logger = { - debug: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - subLogger: vi.fn(() => logger), - } as unknown as Logger; - const db = await makeSQLKernelDatabase({ logger }); - mockStatement.step.mockClear(); - mockDb._inTx = true; - mockDb._spStack = ['point1']; - mockStatement.step - .mockImplementationOnce(() => { - throw new Error('database is locked'); - }) - .mockImplementationOnce(() => { - throw new Error('cannot rollback'); - }); - - expect(() => db.releaseSavepoint('point1')).toThrowError( - 'database is locked', - ); - - expect(logger.error).toHaveBeenCalledWith( - 'failed to discard transaction after commit', - expect.objectContaining({ message: 'cannot rollback' }), - ); - expect(mockDb._inTx).toBe(false); - }); - - it('supports nested savepoints', async () => { - const db = await makeSQLKernelDatabase({}); - db.createSavepoint('outer'); - db.createSavepoint('inner'); - expect(mockDb._spStack).toStrictEqual(['outer', 'inner']); - db.rollbackSavepoint('inner'); - expect(mockDb._spStack).toStrictEqual(['outer']); - expect(mockDb._inTx).toBe(true); - db.releaseSavepoint('outer'); - expect(mockDb._spStack).toStrictEqual([]); - expect(mockDb._inTx).toBe(false); - }); - }); - it('deleteVatStore removes all data for a given vat', async () => { Object.values(mockStatement).forEach((mock) => { if (typeof mock === 'function' && mock.mockReset) { @@ -758,14 +446,12 @@ describe('transaction management', () => { } }); mockDb.exec.mockReset(); - mockDb._inTx = false; + txOpen = false; mockDb._spStack = []; }); it('safeMutate rollbacks transaction on error', async () => { const db = await makeSQLKernelDatabase({}); - mockDb._inTx = false; - mockDb._spStack = []; mockStatement.step.mockImplementationOnce(() => { throw new Error('Database error'); }); @@ -778,7 +464,7 @@ describe('transaction management', () => { it('safeMutate does not commit if already in transaction', async () => { const db = await makeSQLKernelDatabase({}); - mockDb._inTx = true; + txOpen = true; mockDb._spStack = []; const vatStore = db.makeVatStore('test-vat'); vatStore.updateKVData([['key', 'value']], []); diff --git a/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts index 7c545e1104..6aadfcd2bd 100644 --- a/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts @@ -1,3 +1,4 @@ +import type { Logger } from '@metamask/logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import { SQL_QUERIES } from './common.ts'; @@ -6,21 +7,22 @@ import type { KernelDatabase } from '../types.ts'; /** * The nodejs sibling of this file states the invariants both drivers owe the - * crank layer. This one exists because the two drivers reach them differently: - * where nodejs asks SQLite whether a transaction is open, wasm caches the - * answer in `_inTx`, and its vatstore writes go through `safeMutate` rather - * than a driver-level transaction helper. + * crank layer. This one exists because the two drivers reach them over + * different plumbing: wasm steps prepared statements rather than running them, + * and its vatstore writes go through `safeMutate` rather than a driver-level + * transaction helper. */ /** Every statement and exec call, in order. */ let issued: string[] = []; /** SQL that throws when next run. */ let failOnce: Set = new Set(); +/** What SQLite would report through `sqlite3_get_autocommit`. */ +let txOpen = false; /** * Run one statement against the mock. A statement that throws changes nothing, - * as in SQLite; the driver keeps its own view of the transaction in `_inTx`, - * which is the thing under test and so must not be modelled here. + * as in SQLite, so an injected failure leaves the transaction as it was. * * @param text - The SQL being run. */ @@ -29,6 +31,14 @@ function runSql(text: string): void { if (failOnce.delete(text)) { throw new Error(`SQLITE_IOERR: ${text}`); } + if (text === 'BEGIN TRANSACTION') { + txOpen = true; + } else if (text === 'COMMIT TRANSACTION' || text === 'ROLLBACK TRANSACTION') { + if (!txOpen) { + throw new Error('cannot rollback - no transaction is active'); + } + txOpen = false; + } } const makeStatement = (text: string): Record => ({ @@ -47,7 +57,7 @@ const makeStatement = (text: string): Record => ({ const mockDb = { prepare: vi.fn((text: string) => makeStatement(text)), exec: vi.fn(runSql), - _inTx: false, + pointer: 1, _spStack: [] as string[], close: vi.fn(), }; @@ -57,7 +67,10 @@ const DbMock = vi.fn(function () { }); vi.mock('@sqlite.org/sqlite-wasm', () => ({ - default: vi.fn(async () => ({ oo1: { OpfsDb: DbMock, DB: DbMock } })), + default: vi.fn(async () => ({ + oo1: { OpfsDb: DbMock, DB: DbMock }, + capi: { sqlite3_get_autocommit: () => (txOpen ? 0 : 1) }, + })), })); vi.mock('./env.ts', () => ({ getDBFolder: vi.fn(() => 'test-folder') })); @@ -65,7 +78,7 @@ describe('the wasm driver after a failure it tolerates', () => { beforeEach(() => { issued = []; failOnce = new Set(); - mockDb._inTx = false; + txOpen = false; mockDb._spStack = []; }); @@ -117,7 +130,7 @@ describe('the wasm driver after a failure it tolerates', () => { 'refuses $what once the transaction cannot be discarded at all', async ({ write, sql }) => { const kdb = await makeSQLKernelDatabase({}); - mockDb._inTx = true; + txOpen = true; mockDb._spStack = ['t0', 't1']; issued = []; @@ -133,7 +146,7 @@ describe('the wasm driver after a failure it tolerates', () => { it('discards the transaction and lets the next savepoint begin its own', async () => { const kdb = await makeSQLKernelDatabase({}); - mockDb._inTx = true; + txOpen = true; mockDb._spStack = ['t0', 't1']; issued = []; @@ -155,4 +168,118 @@ describe('the wasm driver after a failure it tolerates', () => { 'COMMIT TRANSACTION', ]); }); + + // SQLite having ended the transaction leaves the driver's savepoints gone + // and its ROLLBACK refused, which it must not read as a transaction it could + // not end: that refuses every write from here on. + it('keeps writing after SQLite rolls the transaction back itself', async () => { + const kdb = await makeSQLKernelDatabase({}); + txOpen = true; + mockDb._spStack = ['t0', 't1']; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + txOpen = false; + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + issued = []; + kdb.kernelKVStore.set('k', 'v'); + expect(issued).toStrictEqual([SQL_QUERIES.SET]); + }); + + // SQLite ending the transaction is also the only way out of an abort that + // keeps failing. Refusing writes past that point refuses them forever. + it('writes again once the transaction it could not end is gone', async () => { + const kdb = await makeSQLKernelDatabase({}); + txOpen = true; + mockDb._spStack = ['t0', 't1']; + + failOnce.add('ROLLBACK TO SAVEPOINT t1'); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t1')).toThrow('SQLITE_IOERR'); + + txOpen = false; + + issued = []; + kdb.kernelKVStore.set('k', 'v'); + expect(issued).toStrictEqual([SQL_QUERIES.SET]); + }); + + // A savepoint left on the stack keeps the transaction open with nothing to + // commit or abort it, so every later write joins it and vanishes on close. + it.each([ + { + what: 'a savepoint rollback', + act: (kdb: KernelDatabase) => kdb.rollbackSavepoint('t0'), + failing: 'ROLLBACK TO SAVEPOINT t0', + }, + { + what: 'a savepoint release', + act: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + failing: 'RELEASE SAVEPOINT t0', + }, + { + what: 'a commit', + act: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + failing: 'COMMIT TRANSACTION', + }, + ])('discards the transaction when $what fails', async ({ act, failing }) => { + const kdb = await makeSQLKernelDatabase({}); + txOpen = true; + mockDb._spStack = ['t0']; + issued = []; + + failOnce.add(failing); + expect(() => act(kdb)).toThrow(`SQLITE_IOERR: ${failing}`); + + expect(issued.at(-1)).toBe('ROLLBACK TRANSACTION'); + expect(txOpen).toBe(false); + expect(mockDb._spStack).toStrictEqual([]); + }); + + // The first failure is the diagnosis; a failed abort on top of it only + // repeats that the same connection is broken. + it.each([ + { + what: 'a savepoint rollback', + act: (kdb: KernelDatabase) => kdb.rollbackSavepoint('t0'), + failing: 'ROLLBACK TO SAVEPOINT t0', + after: 'rollback', + }, + { + what: 'a savepoint release', + act: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + failing: 'RELEASE SAVEPOINT t0', + after: 'release', + }, + { + what: 'a commit', + act: (kdb: KernelDatabase) => kdb.releaseSavepoint('t0'), + failing: 'COMMIT TRANSACTION', + after: 'commit', + }, + ])( + 'reports the failure in $what rather than the abort that followed it', + async ({ act, failing, after }) => { + const logger = { + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + subLogger: vi.fn(() => logger), + } as unknown as Logger; + const kdb = await makeSQLKernelDatabase({ logger }); + txOpen = true; + mockDb._spStack = ['t0']; + + failOnce.add(failing); + failOnce.add('ROLLBACK TRANSACTION'); + expect(() => act(kdb)).toThrow(`SQLITE_IOERR: ${failing}`); + + expect(logger.error).toHaveBeenCalledWith( + `failed to discard transaction after ${after}`, + expect.objectContaining({ + message: 'SQLITE_IOERR: ROLLBACK TRANSACTION', + }), + ); + }, + ); }); diff --git a/packages/kernel-store/src/sqlite/wasm.transactions.test.ts b/packages/kernel-store/src/sqlite/wasm.transactions.test.ts new file mode 100644 index 0000000000..c04dbe7752 --- /dev/null +++ b/packages/kernel-store/src/sqlite/wasm.transactions.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; + +import { makeSQLKernelDatabase } from './wasm.ts'; +import type { KernelDatabase } from '../types.ts'; + +/** + * The wasm driver against the real SQLite build. Its siblings mock the database + * to inject I/O failures; this file exists for the savepoint and transaction + * semantics only SQLite itself can state. + */ + +const makeDb = async (): Promise => + makeSQLKernelDatabase({ dbFilename: ':memory:' }); + +describe('the wasm driver on real SQLite', () => { + it('undoes the writes a rolled-back savepoint covers', async () => { + const kdb = await makeDb(); + kdb.kernelKVStore.set('kept', 'before'); + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('undone', 'during'); + kdb.kernelKVStore.set('kept', 'during'); + kdb.rollbackSavepoint('t0'); + + expect(kdb.kernelKVStore.get('undone')).toBeUndefined(); + expect(kdb.kernelKVStore.get('kept')).toBe('before'); + }); + + it('keeps the writes a released savepoint covers', async () => { + const kdb = await makeDb(); + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('key', 'value'); + kdb.releaseSavepoint('t0'); + + expect(kdb.kernelKVStore.get('key')).toBe('value'); + }); + + it('rolls the inner savepoint back without disturbing the outer one', async () => { + const kdb = await makeDb(); + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('outer', 'yes'); + kdb.createSavepoint('t1'); + kdb.kernelKVStore.set('inner', 'yes'); + kdb.rollbackSavepoint('t1'); + kdb.releaseSavepoint('t0'); + + expect(kdb.kernelKVStore.get('outer')).toBe('yes'); + expect(kdb.kernelKVStore.get('inner')).toBeUndefined(); + }); + + // The first savepoint has to open a transaction of its own, or releasing it + // autocommits. See https://github.com/Agoric/agoric-sdk/issues/8423. + it('opens a transaction for the outermost savepoint', async () => { + const kdb = await makeDb(); + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('key', 'value'); + + // SQLite refuses this unless a transaction is open. + expect(() => kdb.executeQuery('ROLLBACK TRANSACTION')).not.toThrow(); + expect(kdb.kernelKVStore.get('key')).toBeUndefined(); + }); + + // What SQLITE_FULL leaves behind: the transaction and every savepoint in it + // are gone, so the crank's rollback fails. Taking that for a transaction the + // driver cannot end refuses every later write on a healthy database. + it('keeps writing after SQLite ends the transaction itself', async () => { + const kdb = await makeDb(); + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('lost', 'value'); + kdb.executeQuery('ROLLBACK TRANSACTION'); + + expect(() => kdb.rollbackSavepoint('t0')).toThrow('no such savepoint: t0'); + + kdb.kernelKVStore.set('after', 'value'); + expect(kdb.kernelKVStore.get('after')).toBe('value'); + }); + + it('takes the next crank in a transaction of its own', async () => { + const kdb = await makeDb(); + kdb.createSavepoint('t0'); + kdb.executeQuery('ROLLBACK TRANSACTION'); + expect(() => kdb.rollbackSavepoint('t0')).toThrow('no such savepoint: t0'); + + kdb.createSavepoint('t0'); + kdb.kernelKVStore.set('key', 'value'); + kdb.releaseSavepoint('t0'); + + expect(kdb.kernelKVStore.get('key')).toBe('value'); + }); +}); diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 67a3ac3368..afc8651dff 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -1,5 +1,8 @@ import { Logger } from '@metamask/logger'; -import type { Database as SqliteDatabase } from '@sqlite.org/sqlite-wasm'; +import type { + Database as SqliteDatabase, + Sqlite3Static, +} from '@sqlite.org/sqlite-wasm'; import sqlite3InitModule from '@sqlite.org/sqlite-wasm'; import { @@ -11,11 +14,22 @@ import { getDBFolder } from './env.ts'; import type { KVStore, VatStore, KernelDatabase } from '../types.ts'; export type Database = SqliteDatabase & { - _inTx: boolean; + /** + * Whether SQLite holds a transaction on this connection, as SQLite reports + * it. A cached answer goes stale: SQLite rolls a transaction back on its own + * after SQLITE_FULL, SQLITE_IOERR or SQLITE_BUSY, and a driver that trusted + * its own bookkeeping would issue a ROLLBACK there, read the resulting + * "cannot rollback - no transaction is active" as a transaction it could not + * end, and refuse every later write on a healthy database. + */ + readonly inTransaction: boolean; // stack of active savepoint names _spStack: string[]; }; +/** `sqlite3_get_autocommit` is bound by the wasm build but absent from its types. */ +type AutocommitCapi = { sqlite3_get_autocommit: (pDb: number) => number }; + /** * Ensure that SQLite is initialized. * @@ -40,8 +54,15 @@ export async function initDB( db = new sqlite3.oo1.DB(`:memory:`, 'cw'); } + const { sqlite3_get_autocommit: getAutocommit } = + sqlite3.capi as Sqlite3Static['capi'] & AutocommitCapi; + const dbWithTx = db as Database; - dbWithTx._inTx = false; + Object.defineProperty(dbWithTx, 'inTransaction', { + configurable: true, + get: () => + dbWithTx.pointer !== undefined && getAutocommit(dbWithTx.pointer) === 0, + }); dbWithTx._spStack = []; return dbWithTx; @@ -194,12 +215,9 @@ export async function makeSQLKernelDatabase({ const sqlCommitTransaction = db.prepare(SQL_QUERIES.COMMIT_TRANSACTION); const sqlAbortTransaction = db.prepare(SQL_QUERIES.ABORT_TRANSACTION); - // Set when an abort meant to discard a transaction fails, which leaves SQLite - // holding one `_inTx` no longer accounts for: nothing will try to end it - // again, and a savepoint taken inside it would be released into it. The - // nodejs driver reads `db.inTransaction` instead of caching it and so cannot - // wedge the flag, but it has the same unowned-transaction gap; both drivers - // close it the same way. + // Set when an abort meant to discard a transaction fails. The writes of the + // crank we gave up on are still in it, and a savepoint taken inside it would + // be released into it. let txAbandoned = false; /** @@ -209,12 +227,11 @@ export async function makeSQLKernelDatabase({ */ function beginIfNeeded(): boolean { assertNotAbandoned(); - if (db._inTx) { + if (db.inTransaction) { return false; } sqlBeginTransaction.step(); sqlBeginTransaction.reset(); - db._inTx = true; return true; } @@ -224,10 +241,6 @@ export async function makeSQLKernelDatabase({ * discarding, and a COMMIT makes those writes durable outright. Retried once * first, since the failure may have been transient. * - * Checked before `_inTx` rather than after: `abortTransaction` clears the flag - * on its way to failing, so the abandoned transaction is exactly the one - * `_inTx` no longer admits to. - * * @throws If the transaction is still there afterwards, because reporting * that this connection can no longer persist anything is the only honest * answer left — returning normally would tell the caller its write landed. @@ -249,11 +262,9 @@ export async function makeSQLKernelDatabase({ */ function commitIfNeeded(): void { assertNotAbandoned(); - if (!db._inTx || db._spStack.length > 0) { + if (!db.inTransaction || db._spStack.length > 0) { return; } - // Cleared first, for the reason `rollbackIfNeeded` gives. - db._inTx = false; try { sqlCommitTransaction.step(); sqlCommitTransaction.reset(); @@ -266,15 +277,13 @@ export async function makeSQLKernelDatabase({ } /** - * Abort the transaction, whatever `_inTx` currently says — the callers that - * most need this have already cleared it. + * Rollback a transaction */ - function abortTransaction(): void { - // Cleared before the abort, which can throw: left true, `beginIfNeeded` is - // a no-op forever after and writes autocommit one statement at a time (see - // `createSavepoint`). - db._inTx = false; - db._spStack.length = 0; + function rollbackIfNeeded(): void { + if (!db.inTransaction) { + txAbandoned = false; + return; + } try { sqlAbortTransaction.step(); sqlAbortTransaction.reset(); @@ -282,16 +291,10 @@ export async function makeSQLKernelDatabase({ txAbandoned = true; throw error; } - txAbandoned = false; - } - - /** - * Rollback a transaction - */ - function rollbackIfNeeded(): void { - if (db._inTx) { - abortTransaction(); - } + db._spStack.length = 0; + // Normally false now. If SQLite still reports a transaction, it is still + // not ours to commit. + txAbandoned = db.inTransaction; } /** @@ -302,8 +305,9 @@ export async function makeSQLKernelDatabase({ * ...". */ function discardTransaction(after: string): void { + db._spStack.length = 0; try { - abortTransaction(); + rollbackIfNeeded(); } catch (error) { logger?.error(`failed to discard transaction after ${after}`, error); } @@ -345,7 +349,8 @@ export async function makeSQLKernelDatabase({ * Execute a SQL query. Unlike the other write paths this one does not consult * `assertNotAbandoned`, whose side effect is a rollback: the debug surfaces * that call it would not expect a query to end a transaction. `step` will run - * DML given it, where the nodejs driver's `all` refuses anything but a SELECT. + * DML given it, where the nodejs driver's `all` refuses a statement that + * returns no rows. * * @param sql - The SQL query to execute. * @returns An array of results. diff --git a/packages/ocap-kernel/src/store/methods/crank.cross-crank-gc.test.ts b/packages/ocap-kernel/src/store/methods/crank.cross-crank-gc.test.ts new file mode 100644 index 0000000000..e503d7981a --- /dev/null +++ b/packages/ocap-kernel/src/store/methods/crank.cross-crank-gc.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import { makeKernelStore } from '../index.ts'; + +/** + * `revertStateBeneathRollback` reverts `maybeFreeKrefs`, which nothing else + * rolls back. The set is not per-crank: only `collectGarbage` empties it, so a + * candidate added while no crank was open is still owed a collection and has to + * survive an unrelated crank's rollback. + * + * `RemoteManager.#handlePeerIncarnation` is one such producer. It runs from a + * network callback with no crank open, under its own `peerIncarnation_` + * savepoint, and `persistPeerRestart` -> `forgetEndpointImports` adds every + * export the restarting peer abandoned. It calls no `collectGarbage` of its own, + * so those krefs wait for the next crank's harvest -- and their kv state is + * already committed by the time it comes. A rollback that discarded them would + * leave the objects orphaned, undeleted, and invisible even to the reference + * count audit, which sees an orphan with no holders and a count of zero as + * consistent. + */ +describe('a GC candidate produced outside a crank', () => { + let kernelStore: ReturnType; + + /** + * Abandon a remote's export the way a peer restart does. + * + * @returns The kref of the now-ownerless object. + */ + function orphanARemoteExport(): string { + const kref = kernelStore.initKernelObject('r1'); + kernelStore.addCListEntry('r1', kref, 'o+1'); + // RemoteManager.#handlePeerIncarnation, inside its own savepoint, no crank. + kernelStore.forgetEndpointImports('r1'); + return kref; + } + + /** + * Run one crank, optionally rolling its delivery back. + * + * @param options - How the crank ends. + * @param options.rollback - Whether the delivery aborts. + */ + function runCrank({ rollback = false }: { rollback?: boolean } = {}): void { + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + if (rollback) { + kernelStore.rollbackCrank('delivery'); + } + kernelStore.collectGarbage(); + kernelStore.endCrank(); + } + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + }); + + it('is collected by the next crank that succeeds', () => { + const kref = orphanARemoteExport(); + + runCrank(); + + expect(kernelStore.kernelRefExists(kref)).toBe(false); + }); + + it('is collected by the next crank that rolls back', () => { + const kref = orphanARemoteExport(); + + runCrank({ rollback: true }); + + expect(kernelStore.kernelRefExists(kref)).toBe(false); + }); + + it('survives a rollback of a crank that never touched it', () => { + const kref = orphanARemoteExport(); + + runCrank({ rollback: true }); + for (let crank = 0; crank < 5; crank += 1) { + runCrank(); + } + + expect(kernelStore.kernelRefExists(kref)).toBe(false); + }); +}); diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 7df88392c1..5c0561a8cf 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -2,7 +2,18 @@ import type { KernelDatabase } from '@metamask/kernel-store'; import { expect, describe, it, vi, beforeEach } from 'vitest'; import { getCrankMethods } from './crank.ts'; -import type { StoreContext } from '../types.ts'; +import type { KRef } from '../../types.ts'; +import type { Savepoint, StoreContext } from '../types.ts'; + +/** + * Build savepoint records holding no collection candidates, for tests that only + * care which savepoints are listed. + * + * @param names - The savepoint names, in order. + * @returns The savepoint records. + */ +const savepoints = (...names: string[]): Savepoint[] => + names.map((name) => ({ name, maybeFreeKrefs: new Set() })); describe('crank methods', () => { let context: StoreContext; @@ -10,6 +21,12 @@ describe('crank methods', () => { let crankMethods: ReturnType; let mockCrankBuffer: unknown[]; + /** + * @returns The names of the currently listed savepoints, in order. + */ + const savepointNames = (): string[] => + context.savepoints.map(({ name }) => name); + beforeEach(() => { mockCrankBuffer = []; context = { @@ -53,7 +70,7 @@ describe('crank methods', () => { context.inCrank = true; crankMethods.createCrankSavepoint('test'); - expect(context.savepoints).toStrictEqual(['test']); + expect(savepointNames()).toStrictEqual(['test']); expect(kdb.createSavepoint).toHaveBeenCalledWith('t0'); }); @@ -62,7 +79,7 @@ describe('crank methods', () => { crankMethods.createCrankSavepoint('first'); crankMethods.createCrankSavepoint('second'); - expect(context.savepoints).toStrictEqual(['first', 'second']); + expect(savepointNames()).toStrictEqual(['first', 'second']); expect(kdb.createSavepoint).toHaveBeenCalledWith('t0'); expect(kdb.createSavepoint).toHaveBeenCalledWith('t1'); }); @@ -94,7 +111,7 @@ describe('crank methods', () => { describe('rollbackCrank', () => { it('forgets the savepoint even if the database rollback fails', () => { context.inCrank = true; - context.savepoints = ['start']; + context.savepoints = savepoints('start'); vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { throw new Error('database is gone'); }); @@ -112,17 +129,17 @@ describe('crank methods', () => { it('should rollback to specified savepoint', () => { context.inCrank = true; - context.savepoints = ['first', 'second', 'third']; + context.savepoints = savepoints('first', 'second', 'third'); crankMethods.rollbackCrank('second'); expect(kdb.rollbackSavepoint).toHaveBeenCalledWith('t1'); - expect(context.savepoints).toStrictEqual(['first']); + expect(savepointNames()).toStrictEqual(['first']); }); it('should throw when savepoint does not exist', () => { context.inCrank = true; - context.savepoints = ['first', 'second']; + context.savepoints = savepoints('first', 'second'); expect(() => crankMethods.rollbackCrank('nonexistent')).toThrow( 'no such savepoint as ""nonexistent""', @@ -143,12 +160,12 @@ describe('crank methods', () => { crankMethods.rollbackCrank('b'); crankMethods.createCrankSavepoint('b2'); expect(kdb.createSavepoint).toHaveBeenLastCalledWith('t1'); - expect(context.savepoints).toStrictEqual(['a', 'b2']); + expect(savepointNames()).toStrictEqual(['a', 'b2']); }); it('clears the crank buffer', () => { context.inCrank = true; - context.savepoints = ['start']; + context.savepoints = savepoints('start'); mockCrankBuffer.push({ type: 'send' }, { type: 'notify' }); crankMethods.rollbackCrank('start'); @@ -175,9 +192,14 @@ describe('crank methods', () => { it('reverts the caches the database cannot reach even when the rollback fails', () => { context.inCrank = true; + // Predates the savepoint, so it survives: only `collectGarbage` empties + // this set, and a candidate owed a collection before this crank began is + // still owed one after it is abandoned. context.maybeFreeKrefs.add('kp1'); crankMethods.createCrankSavepoint('crank'); crankMethods.createCrankSavepoint('delivery'); + // Added by the crank being rolled back, so it goes. + context.maybeFreeKrefs.add('kp2'); vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { throw new Error('disk I/O error'); }); @@ -189,7 +211,7 @@ describe('crank methods', () => { expect(context.refreshCachedValues).toHaveBeenCalled(); expect(context.refreshRunQueue).toHaveBeenCalled(); expect(context.runQueueLengthCache).toBe(-1); - expect([...context.maybeFreeKrefs]).toStrictEqual([]); + expect([...context.maybeFreeKrefs]).toStrictEqual(['kp1']); }); it('keeps the rollback failure as the cause when reverting also fails', () => { @@ -223,7 +245,7 @@ describe('crank methods', () => { it('should release savepoints if they exist', () => { context.inCrank = true; - context.savepoints = ['test']; + context.savepoints = savepoints('test'); crankMethods.endCrank(); expect(kdb.releaseSavepoint).toHaveBeenCalledWith('t0'); expect(context.savepoints).toStrictEqual([]); @@ -251,7 +273,7 @@ describe('crank methods', () => { it('settles the crank even if releasing savepoints fails', async () => { crankMethods.startCrank(); - context.savepoints = ['test']; + context.savepoints = savepoints('test'); const waiter = crankMethods.waitForCrank(); vi.mocked(kdb.releaseSavepoint).mockImplementationOnce(() => { throw new Error('database is gone'); @@ -278,7 +300,7 @@ describe('crank methods', () => { describe('releaseAllSavepoints', () => { it('should release all savepoints', () => { context.inCrank = true; - context.savepoints = ['test']; + context.savepoints = savepoints('test'); crankMethods.releaseAllSavepoints(); expect(kdb.releaseSavepoint).toHaveBeenCalledWith('t0'); expect(context.savepoints).toStrictEqual([]); From 62cb7fa698fd8b89eae4b26e3b7ee9ecb94d27d5 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 14 Sep 2026 16:36:16 +0200 Subject: [PATCH 38/40] fix(ocap-kernel): restore the GC candidates a rollback took, and close the gate's own gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes to the crank layer, all of which #1021 either introduced or left open: `rollbackCrank` restores `maybeFreeKrefs` to the savepoint's snapshot rather than emptying it. That set is not per-crank — only `collectGarbage` empties it — so a candidate produced outside a crank, as a peer restart abandoning a remote's exports does, was dropped by an unrelated crank's rollback and the objects leaked with nothing left to notice them. The reference count audit cannot see it either: an orphan with no holders and a count of zero looks consistent. Guard adopted from #1039. `RemoteManager` snapshots the restarting peer's promises inside its turn at the store rather than before waiting for one. The wait spans a whole crank, so a promise that crank made the peer decider of was never rejected and the sending vat waited on it forever. This one was #1021's own regression. `beginOutOfCrank`/`endOutOfCrank` are replaced by `withStoreOutOfCrank`. A turn that was never given back left the run loop waiting on a promise nothing resolves — no failure, no log, no timeout — and the callback's type now says the held section has to be synchronous. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 9 +- .../src/KernelQueue.audit-ordering.test.ts | 4 +- packages/ocap-kernel/src/KernelQueue.ts | 10 +- .../src/remotes/kernel/RemoteHandle.ts | 23 +++-- .../src/remotes/kernel/RemoteManager.test.ts | 32 +++++++ .../src/remotes/kernel/RemoteManager.ts | 34 +++---- packages/ocap-kernel/src/store/index.test.ts | 3 +- .../store/methods/crank.out-of-crank.test.ts | 94 ++++++++++++------- .../ocap-kernel/src/store/methods/crank.ts | 85 +++++++++++++---- packages/ocap-kernel/src/store/types.ts | 13 ++- packages/ocap-kernel/test/savepoint-stack.ts | 18 +++- 11 files changed, 228 insertions(+), 97 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 41054cda63..65ecfd4182 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -58,6 +58,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** `getPinnedObjects` now names each pinned object once, however many pins it holds; `getPinCount` gives the number of pins ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - **BREAKING:** `incrementRefCount` now throws on a kref the kernel has already deleted, rather than writing a resurrected row ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - A missing object row read as `(0, 0)` and was written back as a live-looking object with no owner; a missing promise row read as `NaN`, which no decrement can bring to zero, so the promise could never be collected. `decrementRefCount` still tolerates a missing object row, since releasing a reference to something already gone is ordinary teardown +- **BREAKING:** `KernelStore`'s `beginOutOfCrank` and `endOutOfCrank` are replaced by `withStoreOutOfCrank(work)`, which takes the turn and gives it back however `work` ends ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - A turn that was never given back left the run loop waiting on a promise nothing resolves: no failure, no log, no timeout. `work` must be synchronous, which its type now says ### Fixed @@ -73,11 +75,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly - Keep a crank's store work inside one transaction, so terminating the vat and collecting garbage after an aborted delivery are committed with the crank instead of autocommitting a statement at a time ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately -- `rollbackCrank` now also reverts the state a database rollback cannot reach, whether or not the rollback itself succeeded: cached stored values are re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - Buffered vat outputs are flushed after that work rather than before it, so a delivery rollback can no longer discard the state an answer was computed from. A release or commit that fails while ending the crank still aborts it after the answer has gone out. Termination still settles the dying vat's own promises immediately +- `rollbackCrank` now also reverts the state a database rollback cannot reach, whether or not the rollback itself succeeded: cached stored values are re-read, and the garbage-collection candidate set goes back to what the savepoint held ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Otherwise a cache kept the abandoned crank's value and the next write persisted it, a GC action taken out of the set for an aborted delivery was lost rather than retried, and a later `collectGarbage` killed the run loop on a promise the rollback had deleted + - Restored rather than emptied, because only `collectGarbage` empties that set: a candidate produced outside a crank — a peer restart abandoning a remote's exports is the real path — is still owed a collection, and clearing it leaked those objects with nothing left to notice, the reference count audit included - Cleanup after a dead crank no longer replaces the error that killed the run loop: a failing `endCrank` reports it as `cause`, and neither `endCrank` nor `rollbackCrank` retries a savepoint that a failed rollback or release has already discarded ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) +- A peer restart now rejects the promises the restarting peer was made decider of during the crank it waited out, not only those it held beforehand ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - Waiting for the store spans a whole crank, and a promise created in that crank was left with no decider and nothing to settle it, so the vat that sent the message waited forever - Keep a crank and a savepoint taken through `KernelStore.createSavepoint` from overlapping ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Those savepoints bypass `ctx.savepoints` and so are invisible to `createCrankSavepoint`'s ordinal naming. One open across a crank left `releaseAllSavepoints` releasing `t0` into it rather than to a commit, so the caller's later rollback discarded the whole crank; one opened inside a crank was cancelled by a delivery rollback it had nothing to do with, or made durable by a release beneath it, after the peer had already been told the message was committed - Both directions are now refused, and the two production callers — `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change — take their turn through the new `beginOutOfCrank`/`endOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before starting each crank. Work between the two must be synchronous diff --git a/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts b/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts index 278cdc49e7..3cd87fbf20 100644 --- a/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.audit-ordering.test.ts @@ -11,9 +11,7 @@ vi.mock('./garbage-collection/garbage-collection.ts', () => ({ /** * `#processCrankResult` flushes the crank buffer last of the crank's own work, - * "after the fallible work above, not before it", because the flush settles the - * promise `enqueueMessage` gave an external caller and a later rollback would - * discard the state that answer was computed from. + * for the reason the comment there gives. * * `assertRefCountsIfAuditing` has to run after that flush too, because a * buffered item's references were counted at enqueue time and so read as a leak diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 24839b7273..8154ae015e 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -387,7 +387,9 @@ export class KernelQueue { // retries. This is probably where we should implement the vat restart logic. } // This kills the worker, so its writes must outlive the rollback above: a - // store that still believed the vat was alive would relaunch it. + // store that still believed the vat was alive would relaunch it. They are + // still inside the crank's transaction, so a release or commit that fails + // in `endCrank` discards them along with the rest of it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); @@ -395,8 +397,10 @@ export class KernelQueue { this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { // After the fallible work above, not before it: the flush settles the - // promise `enqueueMessage` gave an external caller, so a later rollback - // would discard the state that answer was computed from. + // promise `enqueueMessage` gave an external caller, so a delivery + // rollback would otherwise discard the state that answer was computed + // from. It closes that window only — a release or commit that fails in + // `endCrank` still aborts the crank after the answer has gone out. this.#flushCrankBuffer(); } } diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts index fdcdef2c8f..95d616ee94 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts @@ -1054,11 +1054,11 @@ export class RemoteHandle implements EndpointHandle { // turn taken here, and the rule that everything between the two is // synchronous. const savepointName = `receive_${this.remoteId}_${seq}`; - await this.#kernelStore.beginOutOfCrank(); - let deferredCompletion: DeferredCompletion | undefined; - - try { + const deferredCompletion = await this.#kernelStore.withStoreOutOfCrank< + DeferredCompletion | undefined + >(() => { + let completion: DeferredCompletion | undefined; this.#kernelStore.createSavepoint(savepointName); try { switch (method) { @@ -1066,11 +1066,10 @@ export class RemoteHandle implements EndpointHandle { this.#handleRemoteDeliver(params); break; case 'redeemURL': - deferredCompletion = - this.#recordRedeemURLRequest(redeemURLResolution); + completion = this.#recordRedeemURLRequest(redeemURLResolution); break; case 'redeemURLReply': - deferredCompletion = this.#handleRedeemURLReply(...params); + completion = this.#handleRedeemURLReply(...params); break; default: // eslint-disable-next-line @typescript-eslint/restrict-template-expressions @@ -1087,8 +1086,9 @@ export class RemoteHandle implements EndpointHandle { try { this.#kernelStore.rollbackSavepoint(savepointName); } catch (rollbackError) { - // A failed RELEASE above discards the whole savepoint stack, so this - // rollback would report a savepoint already gone over the real error. + // Only a failed RELEASE discards the savepoint stack, making this + // rollback report a savepoint already gone; a throw from the delivery + // or the seq write leaves a real rollback failure here. this.#logger.error( `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, rollbackError, @@ -1096,9 +1096,8 @@ export class RemoteHandle implements EndpointHandle { } throw error; } - } finally { - this.#kernelStore.endOutOfCrank(); - } + return completion; + }); // All in-memory state changes happen after commit diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index 97724a6d6d..22dcfb0f28 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -898,6 +898,38 @@ describe('RemoteManager', () => { ]); }); + // The turn waits out whatever crank is open, which is long enough for that + // crank to make the remote decider of a promise. Nothing else would ever + // reject it, so the vat that sent the message waits forever. + it('rejects a promise the remote became decider of while it waited for the crank', async () => { + const peerId = 'peer-with-late-promise'; + const remote = remoteManager.establishRemote(peerId); + const { remoteId } = remote; + kernelStore.setPeerIncarnation(peerId, 'incarnation-A'); + const resolvePromisesSpy = vi.spyOn(mockKernelQueue, 'resolvePromises'); + + kernelStore.startCrank(); + const handled = getOnIncarnationChange()(peerId, 'incarnation-B'); + await Promise.resolve(); + + const [kpid] = kernelStore.initKernelPromise(); + kernelStore.setPromiseDecider(kpid, remoteId); + kernelStore.addCListEntry(remoteId, kpid, 'rp+1'); + kernelStore.endCrank(); + + await handled; + + expect(resolvePromisesSpy).toHaveBeenCalledWith(remoteId, [ + [ + kpid, + true, + expect.objectContaining({ + body: expect.stringContaining('[KERNEL:PEER_RESTARTED]'), + }), + ], + ]); + }); + it('persists the incarnation and reports restart even when no remote handle exists', async () => { const peerId = 'unknown-peer'; const resolvePromisesSpy = vi.spyOn(mockKernelQueue, 'resolvePromises'); diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts index 9b3f1fabb2..ca46e53449 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts @@ -5,7 +5,7 @@ import { RemoteHandle } from './RemoteHandle.ts'; import type { KernelQueue } from '../../KernelQueue.ts'; import { makeKernelError } from '../../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../../store/index.ts'; -import type { PlatformServices, RemoteId } from '../../types.ts'; +import type { KRef, PlatformServices, RemoteId } from '../../types.ts'; import type { RemoteIdentity, RemoteComms, @@ -230,21 +230,22 @@ export class RemoteManager { const isRestart = stored !== undefined; const remote = isRestart ? this.#remotesByPeer.get(peerId) : undefined; - // Snapshot the decider list BEFORE any kv mutation so the c-list lookup - // can still find the promises through the entries forgetEndpointImports - // is about to tear down. We materialize into an array because the - // generator iterates over kv state that we'll mutate. - const promisesToReject = remote - ? Array.from(this.#kernelStore.getPromisesByDecider(remote.remoteId)) - : []; - // The savepoint is the outermost one on the connection and so its own - // commit point, which it can only be while no crank is open. Everything - // between here and `endOutOfCrank` is synchronous, for the reason - // `beginOutOfCrank` gives. + // commit point, which it can only be while no crank is open. const savepoint = `peerIncarnation_${peerId}`; - await this.#kernelStore.beginOutOfCrank(); - try { + const promisesToReject = await this.#kernelStore.withStoreOutOfCrank< + KRef[] + >(() => { + // Before any kv mutation, so the c-list lookup can still find the + // promises through the entries `forgetEndpointImports` is about to tear + // down. Inside the turn, because waiting for it spans a whole crank, and + // a promise this peer was made decider of during that crank would never + // be rejected — hanging the vat that sent it. Materialized because the + // generator iterates the kv state we are about to mutate. + const deciding = remote + ? Array.from(this.#kernelStore.getPromisesByDecider(remote.remoteId)) + : []; + this.#kernelStore.createSavepoint(savepoint); try { if (isRestart) { @@ -275,9 +276,8 @@ export class RemoteManager { } throw error; } - } finally { - this.#kernelStore.endOutOfCrank(); - } + return deciding; + }); // Post-commit fan-out: in-memory state changes and run-queue // mutations are not reversible by a savepoint, so they wait until the diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index b19fce5d5c..4bebf0f540 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -49,7 +49,6 @@ describe('kernel store', () => { 'allocateErefForKref', 'assertRefCountsIfAuditing', 'auditRefCounts', - 'beginOutOfCrank', 'bufferCrankOutput', 'cleanupOrphanMessages', 'cleanupTerminatedVat', @@ -78,7 +77,6 @@ describe('kernel store', () => { 'dequeueRun', 'detectWake', 'endCrank', - 'endOutOfCrank', 'enqueuePromiseMessage', 'enqueueRun', 'erefToKref', @@ -201,6 +199,7 @@ describe('kernel store', () => { 'undoOcapURLRetention', 'unpinObject', 'waitForCrank', + 'withStoreOutOfCrank', ]); }); }); diff --git a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts index 567d9d7d57..bfe9bb6d48 100644 --- a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts @@ -12,7 +12,9 @@ import { makeKernelStore } from '../index.ts'; * does in each case. * * The two are kept apart instead: callers take their turn through - * `beginOutOfCrank`, and both directions of the overlap are refused. + * `withStoreOutOfCrank`, and both directions of the overlap are refused. The + * turn is taken and given back by that one call because a turn never given back + * parks the run loop with nothing to show for it. */ describe('store work outside a crank', () => { let kernelStore: ReturnType; @@ -30,23 +32,20 @@ describe('store work outside a crank', () => { }); it('refuses a crank started while a caller holds the store', async () => { - await kernelStore.beginOutOfCrank(); - - expect(() => kernelStore.startCrank()).toThrow( - 'startCrank while 1 caller(s) hold the store outside a crank', - ); - - kernelStore.endOutOfCrank(); + await kernelStore.withStoreOutOfCrank(() => { + expect(() => kernelStore.startCrank()).toThrow( + 'startCrank while 1 caller(s) hold the store outside a crank', + ); + }); }); it('lets a caller through once the crank it arrived during has ended', async () => { kernelStore.startCrank(); let held = false; - const turn = (async () => { - await kernelStore.beginOutOfCrank(); + const turn = kernelStore.withStoreOutOfCrank(() => { held = true; - })(); + }); await Promise.resolve(); expect(held).toBe(false); @@ -55,19 +54,17 @@ describe('store work outside a crank', () => { await turn; expect(held).toBe(true); - kernelStore.endOutOfCrank(); }); it('holds the next crank off while a caller is waiting', async () => { kernelStore.startCrank(); - const turn = kernelStore.beginOutOfCrank(); + const turn = kernelStore.withStoreOutOfCrank(() => undefined); // What the run loop consults between `endCrank` and the next `startCrank`. expect(kernelStore.outOfCrankWorkPending()).toBeDefined(); kernelStore.endCrank(); await turn; - kernelStore.endOutOfCrank(); expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); }); @@ -76,12 +73,37 @@ describe('store work outside a crank', () => { expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); }); - it('lets a caller that arrives as the gate clears take its turn too', async () => { - await kernelStore.beginOutOfCrank(); + it('gives the store back when the work throws', async () => { + await expect( + kernelStore.withStoreOutOfCrank(() => { + throw new Error('the delivery failed'); + }), + ).rejects.toThrow('the delivery failed'); + + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => kernelStore.startCrank()).not.toThrow(); + }); + + it('returns what the work returned', async () => { + expect(await kernelStore.withStoreOutOfCrank(() => 'committed')).toBe( + 'committed', + ); + }); - // The run loop's protocol: re-check the gate until nothing is waiting, then - // start the crank with no await in between. + // The run loop's protocol: re-check the gate until nothing is waiting, then + // start the crank with no await in between. Checking once would have it + // resume into `startCrank` with a caller already holding, which is a refusal + // it cannot survive. + it('lets a caller that arrives as the gate clears take its turn too', async () => { let crankStarted = false; + let second: Promise | undefined; + + const first = kernelStore.withStoreOutOfCrank(() => { + // Registers before the first caller's turn is given back, so the gate + // never reaches zero between the two. + second = kernelStore.withStoreOutOfCrank(() => undefined); + }); + const runLoopTurn = (async () => { let pending = kernelStore.outOfCrankWorkPending(); while (pending) { @@ -92,34 +114,34 @@ describe('store work outside a crank', () => { crankStarted = true; })(); - // The first caller releases, and a second registers in the microtask that - // resolution queues — ahead of the run loop's own continuation. Checking - // the gate once would have the run loop resume into `startCrank` with this - // caller already holding, which is a refusal it cannot survive. - kernelStore.endOutOfCrank(); - await kernelStore.beginOutOfCrank(); + await first; expect(crankStarted).toBe(false); - kernelStore.endOutOfCrank(); + await second; await runLoopTurn; expect(crankStarted).toBe(true); }); - it('refuses an unmatched release, which would park the run loop for good', () => { - expect(() => kernelStore.endOutOfCrank()).toThrow( - 'endOutOfCrank without beginOutOfCrank', - ); - }); - it('holds the crank off until the last of several callers is done', async () => { - await kernelStore.beginOutOfCrank(); - await kernelStore.beginOutOfCrank(); + kernelStore.startCrank(); + const order: string[] = []; + const first = kernelStore.withStoreOutOfCrank(() => { + order.push('first'); + }); + const second = kernelStore.withStoreOutOfCrank(() => { + order.push('second'); + // The first caller has had its turn and given it back. The gate is still + // closed, because this one has not. + order.push( + kernelStore.outOfCrankWorkPending() === undefined ? 'open' : 'closed', + ); + }); - kernelStore.endOutOfCrank(); - expect(kernelStore.outOfCrankWorkPending()).toBeDefined(); + kernelStore.endCrank(); + await Promise.all([first, second]); - kernelStore.endOutOfCrank(); + expect(order).toStrictEqual(['first', 'second', 'closed']); expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); }); }); diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index d30c5d977b..e76a641fc9 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -2,7 +2,7 @@ import { Fail, q } from '@endo/errors'; import { makePromiseKit } from '@endo/promise-kit'; import type { KernelDatabase } from '@metamask/kernel-store'; -import type { CrankBufferItem, StoreContext } from '../types.ts'; +import type { CrankBufferItem, Savepoint, StoreContext } from '../types.ts'; /** * Get the crank methods. @@ -43,14 +43,14 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { * once no crank is open, and holds the run loop off until the matching * {@link endOutOfCrank}. * - * The caller's work must be synchronous. Awaiting while holding this would - * park the run loop for the duration, and awaiting between the savepoint and - * its release is the interleaving this exists to prevent. + * Private to this module; callers take their turn through + * {@link withStoreOutOfCrank}. */ async function beginOutOfCrank(): Promise { outOfCrankWaiters += 1; - // Created before the await below, so a run loop reaching `startCrank` - // in the meantime sees the gate rather than racing past it. + // Created before the await below, so a run loop that consults + // `outOfCrankWorkPending` in the meantime has something to wait on. The + // count it increments above is what `startCrank` itself refuses on. outOfCrankIdle ??= makePromiseKit(); try { while (ctx.inCrank) { @@ -84,6 +84,31 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { } } + /** + * Hold the store outside any crank for the duration of `work`, and give it + * back however `work` ends. Paired here rather than by callers because a turn + * never given back leaves the run loop waiting on a promise nothing resolves: + * no failure, no log, no timeout. + * + * `work` is typed to return a value rather than a promise because it has to + * be synchronous: awaiting here parks the run loop for the duration, and + * awaiting between a savepoint and its release is the interleaving the turn + * exists to prevent. + * + * @param work - The synchronous work to do while holding the store. + * @returns What `work` returned. + */ + async function withStoreOutOfCrank( + work: () => Result, + ): Promise { + await beginOutOfCrank(); + try { + return work(); + } finally { + endOutOfCrank(); + } + } + /** * @returns A promise to await before starting a crank, or undefined if * nothing is waiting — undefined rather than a resolved promise so that the @@ -102,7 +127,12 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { ctx.inCrank || Fail`createCrankSavepoint outside of crank`; const ordinal = ctx.savepoints.length; kdb.createSavepoint(`t${ordinal}`); - ctx.savepoints.push(name); + // Copied, not referenced: `maybeFreeKrefs` is mutated in place from here on, + // and this is the "before" a rollback restores. + ctx.savepoints.push({ + name, + maybeFreeKrefs: new Set(ctx.maybeFreeKrefs), + }); } /** @@ -114,16 +144,23 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { ctx.inCrank || Fail`rollbackCrank outside of crank`; ctx.crankBuffer.length = 0; // Discard buffered outputs for (const ordinal of ctx.savepoints.keys()) { - if (ctx.savepoints[ordinal] === savepoint) { + const restored = ctx.savepoints[ordinal]; + if (restored?.name === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); ctx.savepoints.length = ordinal; } catch (error) { ctx.savepoints.length = 0; - revertStateBeneathRollback(error); + // Before the rethrow, and not only on the path below. A failed + // rollback discards the whole transaction, so the database has moved + // back at least as far as a successful rollback would have taken it + // and these caches are at least as stale. Rethrowing ahead of this + // would leave the dying crank holding the GC action it consumed and + // the freed krefs it was about to collect. + revertStateBeneathRollback(restored, error); throw error; } - revertStateBeneathRollback(); + revertStateBeneathRollback(restored); return; } } @@ -134,16 +171,33 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { * Revert what a database rollback cannot reach: the in-memory caches built * over the abandoned crank's writes. * - * @param rollbackError - The error the rollback threw, if it threw. + * @param restored - The savepoint being rolled back to, whose snapshot of + * `maybeFreeKrefs` is the "before" this restores. + * @param rollbackError - The error the rollback threw, if it threw. Kept as + * the `cause` should reverting fail too, since it is the root cause an + * operator needs. */ - function revertStateBeneathRollback(rollbackError?: unknown): void { + function revertStateBeneathRollback( + restored: Savepoint, + rollbackError?: unknown, + ): void { try { ctx.refreshRunQueue(); ctx.runQueueLengthCache = -1; ctx.refreshCachedValues(); - // Clearing all of them is correct only while a rollback discards the whole - // delivery, which is all any caller asks for. + // Nothing rolls back RAM. Krefs this crank added are collection + // candidates only because of decrements that were just undone; left in + // place, `collectGarbage` throws on a later crank for any promise this one + // created, killing the run loop over work that no longer exists. + // Restored to the savepoint's snapshot rather than cleared, because the + // set is not per-crank: only `collectGarbage` empties it, so a candidate + // added while the run loop was idle — `terminateVat` unpinning a root is + // the real path — is still owed a collection and must survive an + // unrelated crank's rollback. ctx.maybeFreeKrefs.clear(); + for (const kref of restored.maybeFreeKrefs) { + ctx.maybeFreeKrefs.add(kref); + } } catch (revertError) { if (rollbackError === undefined) { throw revertError; @@ -230,8 +284,7 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { endCrank, releaseAllSavepoints, waitForCrank, - beginOutOfCrank, - endOutOfCrank, + withStoreOutOfCrank, outOfCrankWorkPending, bufferCrankOutput, flushCrankBuffer, diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index b9886c174f..10bacd658b 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -23,7 +23,7 @@ export type StoreContext = { inCrank: boolean; crankSettled?: Promise; resolveCrank?: (() => void) | undefined; - savepoints: string[]; + savepoints: Savepoint[]; crankBuffer: CrankBufferItem[]; // Buffer for sends and notifications during crank subclusters: StoredValue; // Holds Subcluster[] nextSubclusterId: StoredValue; // Holds string @@ -32,6 +32,17 @@ export type StoreContext = { logger?: Logger | undefined; }; +/** + * A database savepoint, paired with the RAM state a database rollback cannot + * reach. `maybeFreeKrefs` is the collection-candidate set as it stood when the + * savepoint was taken, so a rollback can put back exactly what the abandoned + * work added and no more. + */ +export type Savepoint = { + name: string; + maybeFreeKrefs: Set; +}; + export type StoredValue = { get(): string | undefined; set(newValue: string): void; diff --git a/packages/ocap-kernel/test/savepoint-stack.ts b/packages/ocap-kernel/test/savepoint-stack.ts index c4b7e587fc..68c43975fa 100644 --- a/packages/ocap-kernel/test/savepoint-stack.ts +++ b/packages/ocap-kernel/test/savepoint-stack.ts @@ -14,8 +14,8 @@ export type FailingReleaseStore = { /** * Wrap a kernel store so that `releaseSavepoint` fails the way a full disk does, - * modelling the drivers' bookkeeping as of #1012 and verified against both: a - * failed RELEASE clears the savepoint stack, and rolling back a name that is no + * modelling the drivers' bookkeeping as of #1021 and verified against both: a + * failed RELEASE clears the savepoint stack, and touching a name that is no * longer on it throws `No such savepoint`. What is under test is therefore the * caller's error handling, not an expected outcome baked into the mock. * @@ -30,18 +30,26 @@ export function withFailingSavepointRelease( ): FailingReleaseStore { const savepoints: string[] = []; const releaseFailure = new Error('database or disk is full'); - const rollbackSavepoint = vi.fn((name: string) => { + const assertOnStack = (name: string): void => { if (!savepoints.includes(name)) { throw new Error(`No such savepoint: ${name}`); } - }); + }; + const rollbackSavepoint = vi.fn(assertOnStack); return { kernelStore: { ...kernelStore, createSavepoint: (name: string) => { + // The refusal `KernelStore.createSavepoint` makes, so a caller that + // took its savepoint inside a crank fails here as it would in + // production rather than passing against a more permissive model. + if (kernelStore.isInCrank()) { + throw new Error(`createSavepoint "${name}" inside a crank`); + } savepoints.push(name); }, - releaseSavepoint: () => { + releaseSavepoint: (name: string) => { + assertOnStack(name); savepoints.length = 0; throw releaseFailure; }, From 6f9e29db7b54a38330a0b272eb60dcafdeace6f2 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 14 Sep 2026 19:00:05 +0200 Subject: [PATCH 39/40] fix(ocap-kernel): refuse out-of-crank work that is not synchronous MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `withStoreOutOfCrank` gives the turn back from its `finally` the moment `work()` returns. An async callback returns a promise there, so the store went back to the run loop with the callback still between its savepoint and the release — the interleaving the turn exists to prevent. `() => Result` did not forbid it, though the JSDoc and changelog said the type did. The type now refuses a callback whose return type is thenable, and a thenable that reaches the call through inference is thrown on instead of being handed the turn back mid-flight. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 2 +- .../store/methods/crank.out-of-crank.test.ts | 16 ++++++++++ .../ocap-kernel/src/store/methods/crank.ts | 30 +++++++++++++++---- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 65ecfd4182..138e014ebd 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -59,7 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** `incrementRefCount` now throws on a kref the kernel has already deleted, rather than writing a resurrected row ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - A missing object row read as `(0, 0)` and was written back as a live-looking object with no owner; a missing promise row read as `NaN`, which no decrement can bring to zero, so the promise could never be collected. `decrementRefCount` still tolerates a missing object row, since releasing a reference to something already gone is ordinary teardown - **BREAKING:** `KernelStore`'s `beginOutOfCrank` and `endOutOfCrank` are replaced by `withStoreOutOfCrank(work)`, which takes the turn and gives it back however `work` ends ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - - A turn that was never given back left the run loop waiting on a promise nothing resolves: no failure, no log, no timeout. `work` must be synchronous, which its type now says + - A turn that was never given back left the run loop waiting on a promise nothing resolves: no failure, no log, no timeout. `work` must be synchronous: the type refuses a callback that returns a promise, and one that reaches it through inference anyway is thrown on rather than given the turn back mid-flight ### Fixed diff --git a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts index bfe9bb6d48..5130b57dda 100644 --- a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts @@ -90,6 +90,22 @@ describe('store work outside a crank', () => { ); }); + // The turn is given back the moment the work returns, so work that awaits + // resumes with the run loop free to start a crank — and the savepoint both + // callers take inside it would nest in that crank rather than being the + // commit point. The type refuses this; the check is for what inference lets + // through. + it('refuses work that is not synchronous', async () => { + await expect( + kernelStore.withStoreOutOfCrank( + (async () => undefined) as unknown as () => undefined, + ), + ).rejects.toThrow('work that is not synchronous'); + + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + expect(() => kernelStore.startCrank()).not.toThrow(); + }); + // The run loop's protocol: re-check the gate until nothing is waiting, then // start the crank with no await in between. Checking once would have it // resume into `startCrank` with a caller already holding, which is a refusal diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index e76a641fc9..db38a24ef4 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -4,6 +4,16 @@ import type { KernelDatabase } from '@metamask/kernel-store'; import type { CrankBufferItem, Savepoint, StoreContext } from '../types.ts'; +/** + * @param value - Anything. + * @returns Whether it is a thenable, which is what `await` acts on. + */ +function isPromiseLike(value: unknown): boolean { + return ( + typeof (value as PromiseLike | undefined)?.then === 'function' + ); +} + /** * Get the crank methods. * @@ -90,20 +100,28 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { * never given back leaves the run loop waiting on a promise nothing resolves: * no failure, no log, no timeout. * - * `work` is typed to return a value rather than a promise because it has to - * be synchronous: awaiting here parks the run loop for the duration, and - * awaiting between a savepoint and its release is the interleaving the turn - * exists to prevent. + * `work` must be synchronous. The turn is given back the moment it returns, + * so work that awaits resumes with the run loop free to start a crank — and + * both callers take a savepoint inside it, which would then nest inside that + * crank rather than being the commit point it has to be. The type refuses the + * plain cases and the check below catches the rest; a caller that needs to + * await does it with what `work` hands back. * * @param work - The synchronous work to do while holding the store. * @returns What `work` returned. */ async function withStoreOutOfCrank( - work: () => Result, + work: () => Result & + (Result extends PromiseLike ? never : unknown), ): Promise { await beginOutOfCrank(); try { - return work(); + const result = work(); + // Thrown from inside the `try`, so the `finally` still gives the turn + // back rather than parking the run loop on top of the mistake. + !isPromiseLike(result) || + Fail`withStoreOutOfCrank given work that is not synchronous`; + return result; } finally { endOutOfCrank(); } From 0304ddc6a4cbfbfe09dbfe83900e274d4a1b0b95 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 15 Sep 2026 13:05:47 +0200 Subject: [PATCH 40/40] fix(ocap-kernel,kernel-store): keep a recorded vat death and an orphaned transaction out of the rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the crank's transaction discipline could still leave the store disagreeing with what had already happened: - `vatPowers.exitVat` terminates without aborting, so `#crankRollbackAttempted` stayed unset and a throw from `collectGarbage` or the crank flush sent the run loop's catch into `rollbackCrank('delivery')` — undoing `deleteVat` and `markVatAsTerminated` for a worker already killed, whose callers had already been rejected, and then committing that. The flag is now `#deliveryRollbackAllowed` and a recorded death withholds it. - `withStoreOutOfCrank`'s synchrony type was distributive, so a union return (one promise-returning branch of several) and a forwarded generic parameter both got through; and the runtime check threw without settling the thenable it refused, making an abandoned rejection end the process. `Synchronous` judges the union whole, and the thenable gets a handler before the throw. - `createSavepoint` was the one savepoint path with no discard guard: a `SAVEPOINT` that failed after this call's own `BEGIN` left a transaction with nothing on the savepoint stack to reach it and `txAbandoned` unset, so every later write joined it and an unrelated release committed the lot. It now validates the name ahead of the `BEGIN` and discards a transaction it began. Each of the three is pinned by a test verified against the mutation. Also: `createSavepoint`'s refusal and two changelog entries named `beginOutOfCrank`, which is module-private; the BREAKING entry announced a migration off two methods that never shipped, and is now an `### Added` entry for `withStoreOutOfCrank`/`outOfCrankWorkPending`. Co-Authored-By: Claude Opus 5 (1M context) --- .../nodejs.savepoint-interleaving.test.ts | 2 +- .../nodejs.transaction-survival.test.ts | 37 ++++++++++++++++++ packages/kernel-store/src/sqlite/nodejs.ts | 18 +++++++-- .../sqlite/wasm.transaction-survival.test.ts | 37 ++++++++++++++++++ packages/kernel-store/src/sqlite/wasm.ts | 18 +++++++-- packages/ocap-kernel/CHANGELOG.md | 6 +-- packages/ocap-kernel/src/KernelQueue.test.ts | 29 ++++++++++++++ packages/ocap-kernel/src/KernelQueue.ts | 36 +++++++++++------ packages/ocap-kernel/src/store/index.ts | 4 +- .../store/methods/crank.out-of-crank.test.ts | 26 +++++++++++++ .../src/store/methods/crank.test.ts | 2 +- .../ocap-kernel/src/store/methods/crank.ts | 39 ++++++++++++++----- 12 files changed, 220 insertions(+), 34 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts b/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts index 47087733de..de77d17054 100644 --- a/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.savepoint-interleaving.test.ts @@ -16,7 +16,7 @@ import { makeSQLKernelDatabase } from './nodejs.ts'; * what SQLite actually does, which is the reason the kernel now serializes the * two: `createSavepoint` refuses inside a crank, `startCrank` refuses while a * caller holds the store outside one, and callers take their turn through - * `beginOutOfCrank`. See `crank.out-of-crank.test.ts` for the enforcement. + * `withStoreOutOfCrank`. See `crank.out-of-crank.test.ts` for the enforcement. */ describe('a savepoint the crank does not know about', () => { it('outside the crank, leaves the crank release with nothing to commit', async () => { diff --git a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts index 27cdd35a01..125451653d 100644 --- a/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.transaction-survival.test.ts @@ -124,6 +124,43 @@ describe('the nodejs driver after a failure it tolerates', () => { ]); }); + it('discards a transaction it began when the savepoint then fails', async () => { + const kdb = await makeSQLKernelDatabase({}); + issued = []; + + // Out of any transaction, so this call issues the BEGIN itself, and then + // the SAVEPOINT fails. Nothing would be on the savepoint stack to reach + // that transaction through and `txAbandoned` would be unset, so the orphan + // would take every later write and be committed by an unrelated release. + failOnce.add('SAVEPOINT t0'); + expect(() => kdb.createSavepoint('t0')).toThrow('SQLITE_IOERR'); + + expect(issued).toStrictEqual([ + 'BEGIN TRANSACTION', + 'SAVEPOINT t0', + 'ROLLBACK TRANSACTION', + ]); + expect(mockDb.inTransaction).toBe(false); + expect(mockDb._spStack).toStrictEqual([]); + }); + + it('leaves a transaction it found open for its owner to discard', async () => { + const kdb = await makeSQLKernelDatabase({}); + // A crank in progress: the transaction and `t0` are the run loop's. + mockDb.inTransaction = true; + mockDb._spStack = ['t0']; + issued = []; + + failOnce.add('SAVEPOINT t1'); + expect(() => kdb.createSavepoint('t1')).toThrow('SQLITE_IOERR'); + + // Discarding here would abort the crank behind its back; it still has `t0` + // to reach its own rollback through. + expect(issued).toStrictEqual(['SAVEPOINT t1']); + expect(mockDb.inTransaction).toBe(true); + expect(mockDb._spStack).toStrictEqual(['t0']); + }); + // Teardown after the run loop dies reaches the store by many doors, most of // which never touch `beginIfNeeded`. it.each([ diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index 23f4b417e1..90fd9a104b 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -349,13 +349,25 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function createSavepoint(name: string): void { + // Ahead of the BEGIN, so a name this refuses leaves no transaction behind. + assertSafeIdentifier(name); // We must be in a transaction when creating the savepoint or releasing it // later will cause an autocommit. // See https://github.com/Agoric/agoric-sdk/issues/8423 - beginIfNeeded(); - assertSafeIdentifier(name); + const startedTransaction = beginIfNeeded(); const query = SQL_QUERIES.CREATE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // Nothing was pushed, so no savepoint path would reach this transaction + // and `txAbandoned` is unset: left open, it would take every later write + // and be committed by some unrelated release. One we merely found open + // is the caller's to discard, not ours. + if (startedTransaction) { + discardTransaction('savepoint creation'); + } + throw error; + } db._spStack.push(name); } diff --git a/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts index 6aadfcd2bd..c10adbc597 100644 --- a/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.transaction-survival.test.ts @@ -82,6 +82,43 @@ describe('the wasm driver after a failure it tolerates', () => { mockDb._spStack = []; }); + it('discards a transaction it began when the savepoint then fails', async () => { + const kdb = await makeSQLKernelDatabase({}); + issued = []; + + // Out of any transaction, so this call issues the BEGIN itself, and then + // the SAVEPOINT fails. Nothing would be on the savepoint stack to reach + // that transaction through and `txAbandoned` would be unset, so the orphan + // would take every later write and be committed by an unrelated release. + failOnce.add('SAVEPOINT t0'); + expect(() => kdb.createSavepoint('t0')).toThrow('SQLITE_IOERR'); + + expect(issued).toStrictEqual([ + 'BEGIN TRANSACTION', + 'SAVEPOINT t0', + 'ROLLBACK TRANSACTION', + ]); + expect(txOpen).toBe(false); + expect(mockDb._spStack).toStrictEqual([]); + }); + + it('leaves a transaction it found open for its owner to discard', async () => { + const kdb = await makeSQLKernelDatabase({}); + // A crank in progress: the transaction and `t0` are the run loop's. + txOpen = true; + mockDb._spStack = ['t0']; + issued = []; + + failOnce.add('SAVEPOINT t1'); + expect(() => kdb.createSavepoint('t1')).toThrow('SQLITE_IOERR'); + + // Discarding here would abort the crank behind its back; it still has `t0` + // to reach its own rollback through. + expect(issued).toStrictEqual(['SAVEPOINT t1']); + expect(txOpen).toBe(true); + expect(mockDb._spStack).toStrictEqual(['t0']); + }); + // Teardown after the run loop dies reaches the store by many doors, most of // which never touch `beginIfNeeded`. it.each([ diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index afc8651dff..a18c8f7adf 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -449,13 +449,25 @@ export async function makeSQLKernelDatabase({ * @param name - The name of the savepoint. */ function createSavepoint(name: string): void { + // Ahead of the BEGIN, so a name this refuses leaves no transaction behind. + assertSafeIdentifier(name); // We must be in a transaction when creating the savepoint or releasing it // later will cause an autocommit. // See https://github.com/Agoric/agoric-sdk/issues/8423 - beginIfNeeded(); - assertSafeIdentifier(name); + const startedTransaction = beginIfNeeded(); const query = SQL_QUERIES.CREATE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // Nothing was pushed, so no savepoint path would reach this transaction + // and `txAbandoned` is unset: left open, it would take every later write + // and be committed by some unrelated release. One we merely found open + // is the caller's to discard, not ours. + if (startedTransaction) { + discardTransaction('savepoint creation'); + } + throw error; + } db._spStack.push(name); } diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 138e014ebd..7216f159f0 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -44,6 +44,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `getOcapURLObjects`, `getOcapURLIssuanceCount`, `retainForOcapURL`, `undoOcapURLRetention` and `releaseOcapURLRetentions` to the kernel store, and `VatManager.releaseVatRootPin` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - `undoOcapURLRetention` unwinds one issuance whose URL was never minted; `releaseOcapURLRetentions` drops a target's whole retention, for disavowing every URL naming it at once - Add `getPinCount` to the kernel store, which reports how many pins are held on an object ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Add `withStoreOutOfCrank` and `outOfCrankWorkPending` to the kernel store, for work that has to hold the store outside any crank ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) + - `work` must be synchronous: the turn is given back the moment it returns. A caller that needs to await does it with what `work` hands back ### Changed @@ -58,8 +60,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** `getPinnedObjects` now names each pinned object once, however many pins it holds; `getPinCount` gives the number of pins ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - **BREAKING:** `incrementRefCount` now throws on a kref the kernel has already deleted, rather than writing a resurrected row ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - A missing object row read as `(0, 0)` and was written back as a live-looking object with no owner; a missing promise row read as `NaN`, which no decrement can bring to zero, so the promise could never be collected. `decrementRefCount` still tolerates a missing object row, since releasing a reference to something already gone is ordinary teardown -- **BREAKING:** `KernelStore`'s `beginOutOfCrank` and `endOutOfCrank` are replaced by `withStoreOutOfCrank(work)`, which takes the turn and gives it back however `work` ends ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - - A turn that was never given back left the run loop waiting on a promise nothing resolves: no failure, no log, no timeout. `work` must be synchronous: the type refuses a callback that returns a promise, and one that reaches it through inference anyway is thrown on rather than given the turn back mid-flight ### Fixed @@ -85,7 +85,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Waiting for the store spans a whole crank, and a promise created in that crank was left with no decider and nothing to settle it, so the vat that sent the message waited forever - Keep a crank and a savepoint taken through `KernelStore.createSavepoint` from overlapping ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) - Those savepoints bypass `ctx.savepoints` and so are invisible to `createCrankSavepoint`'s ordinal naming. One open across a crank left `releaseAllSavepoints` releasing `t0` into it rather than to a commit, so the caller's later rollback discarded the whole crank; one opened inside a crank was cancelled by a delivery rollback it had nothing to do with, or made durable by a release beneath it, after the peer had already been told the message was committed - - Both directions are now refused, and the two production callers — `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change — take their turn through the new `beginOutOfCrank`/`endOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before starting each crank. Work between the two must be synchronous + - Both directions are now refused, and the two production callers — `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change — take their turn through `withStoreOutOfCrank`, which the run loop consults via `outOfCrankWorkPending` before starting each crank - `handleRemoteMessage` now decodes an incoming `redeemURL` before opening its savepoint instead of awaiting inside it; the decode writes nothing, so the message stays atomic - Consequence: an inbound remote message or a peer's handshake arriving mid-crank waits for that crank to end, so their latency is now bounded by the slowest crank rather than being independent of it. A crank that sends to a remote can be slow. The alternative is to route inbound messages through the run queue as SwingSet's comms vat does, which is a larger change than this one - The reference count audit runs after the crank is committed rather than inside it, so a violation kills the run loop instead of rolling back a delivery whose caller the crank buffer flush had already answered ([#1021](https://github.com/Consensys-Incorporated/ocap-kernel/pull/1021)) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index f66e79b05a..523f6bf255 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -518,6 +518,35 @@ describe('KernelQueue', () => { expect(kernelStore.rollbackCrank).toHaveBeenCalledOnce(); }); + it('does not roll back a recorded vat death when the crank then throws', async () => { + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce({ + type: 'send', + target: 'ko123', + message: { result: 'kp99' } as KernelMessage, + }); + // `vatPowers.exitVat`: terminate without abort, so nothing has rolled + // back yet and the delivery itself succeeded. + const info = { body: '"exit"', slots: [] }; + const deliver = vi + .fn() + .mockResolvedValue({ terminate: { vatId: 'v1', info } }); + ( + kernelStore.collectGarbage as unknown as MockInstance + ).mockImplementation(() => { + throw new Error(STOP_RUN_LOOP); + }); + + await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); + + // A rollback here would undo the death of a worker that is already gone, + // and `endCrank` would commit a store that relaunches it on next start. + expect(terminateVat).toHaveBeenCalledWith('v1', info); + expect(kernelStore.rollbackCrank).not.toHaveBeenCalled(); + }); + it('reports both failures when the rollback also fails', async () => { ( kernelStore.runQueueLength as unknown as MockInstance diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 8154ae015e..1aafef5b43 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -55,11 +55,15 @@ export class KernelQueue { #wakeUpTheRunQueue: (() => void) | null; /** - * Whether this crank's savepoint has already been handed to `rollbackCrank`. - * Attempted, not necessarily succeeded: it is forgotten either way, so asking - * twice could only report "no such savepoint" over the real error. + * Whether the run loop's catch may still roll this crank's delivery back. + * + * False once the savepoint has been handed to `rollbackCrank` — attempted, + * not necessarily succeeded, since it is forgotten either way and asking + * twice could only report "no such savepoint" over the real error — and + * false once a vat's death has been recorded, which must outlive whatever + * throws after it. */ - #crankRollbackAttempted: boolean = false; + #deliveryRollbackAllowed: boolean = true; /** * The run loop's state, as one value so that a failure recorded for a loop @@ -144,7 +148,7 @@ export class KernelQueue { } this.#kernelStore.startCrank(); - this.#crankRollbackAttempted = false; + this.#deliveryRollbackAllowed = true; try { // Two savepoints, because rolling back the outermost one discards the // enclosing transaction (see `rollbackSavepoint`) and an aborted crank @@ -176,7 +180,7 @@ export class KernelQueue { } catch (error) { // An aborted crank already asked; asking again could only throw "no // such savepoint" over the real error. - if (!this.#crankRollbackAttempted) { + if (this.#deliveryRollbackAllowed) { try { this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { @@ -362,8 +366,9 @@ export class KernelQueue { try { this.#kernelStore.rollbackCrank('delivery'); } finally { - // Set even when the rollback threw: the savepoint is gone either way. - this.#crankRollbackAttempted = true; + // Cleared even when the rollback threw: the savepoint is gone either + // way. + this.#deliveryRollbackAllowed = false; } // Discard kernel subscriptions that were queued for invocation this.#resolvedWithKernelSubscription = []; @@ -386,13 +391,20 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // This kills the worker, so its writes must outlive the rollback above: a - // store that still believed the vat was alive would relaunch it. They are - // still inside the crank's transaction, so a release or commit that fails - // in `endCrank` discards them along with the rest of it. + // This kills the worker, so its writes must outlive any rollback: a store + // that still believed the vat was alive would relaunch it, having already + // rejected the promises it was deciding and answered its callers. The + // abort path above has rolled back already, so they land on top of it; + // `vatPowers.exitVat` terminates *without* aborting, and there the work + // below can still throw into the run loop's catch, so the flag is what + // keeps that catch off them. Withholding it costs nothing, the delivery + // having succeeded on that path. They are still inside the crank's + // transaction either way, so a release or commit that fails in `endCrank` + // discards them along with the rest of it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); + this.#deliveryRollbackAllowed = false; } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 7186fca951..53fc110bf5 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -303,13 +303,13 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { * These are invisible to `createCrankSavepoint`'s ordinal naming, so one * opened inside a crank would be rolled back by a delivery that has nothing * to do with it — after its owner had already reported success to a peer. - * Callers take their turn through `beginOutOfCrank`. + * Callers take their turn through `withStoreOutOfCrank`. * * @param name - The savepoint name. */ function createSavepoint(name: string): void { !context.inCrank || - Fail`createSavepoint ${q(name)} inside a crank; use beginOutOfCrank`; + Fail`createSavepoint ${q(name)} inside a crank; use withStoreOutOfCrank`; kdb.createSavepoint(name); } diff --git a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts index 5130b57dda..9858cf6887 100644 --- a/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.out-of-crank.test.ts @@ -106,6 +106,32 @@ describe('store work outside a crank', () => { expect(() => kernelStore.startCrank()).not.toThrow(); }); + it('settles a thenable it refuses rather than leaving it unhandled', async () => { + let handleRejection: ((reason: unknown) => void) | undefined; + const thenable = { + then: ( + _onFulfilled: (value: unknown) => void, + onRejected: (reason: unknown) => void, + ) => { + handleRejection = onRejected; + }, + }; + + await expect( + kernelStore.withStoreOutOfCrank( + (() => thenable) as unknown as () => undefined, + ), + ).rejects.toThrow('work that is not synchronous'); + + // Nothing awaits the refused thenable, and an unhandled rejection ends the + // process — so the guard has to take the rejection channel before throwing. + expect(handleRejection).toBeDefined(); + expect(() => + handleRejection?.(new Error('rejected after being refused')), + ).not.toThrow(); + expect(kernelStore.outOfCrankWorkPending()).toBeUndefined(); + }); + // The run loop's protocol: re-check the gate until nothing is waiting, then // start the crank with no await in between. Checking once would have it // resume into `startCrank` with a caller already holding, which is a refusal diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 5c0561a8cf..e0a73fe804 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -286,7 +286,7 @@ describe('crank methods', () => { it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); - context.savepoints = ['test']; + context.savepoints = savepoints('test'); vi.mocked(kdb.releaseSavepoint).mockImplementationOnce(() => { throw new Error('database is gone'); }); diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index db38a24ef4..f987b3ba9e 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -8,12 +8,27 @@ import type { CrankBufferItem, Savepoint, StoreContext } from '../types.ts'; * @param value - Anything. * @returns Whether it is a thenable, which is what `await` acts on. */ -function isPromiseLike(value: unknown): boolean { +function isPromiseLike(value: unknown): value is PromiseLike { return ( typeof (value as PromiseLike | undefined)?.then === 'function' ); } +/** + * `never` for any `Result` that has a thenable member, so that the callback + * cannot be written. + * + * The tuple stops the conditional distributing, which is what makes it judge a + * union whole: distributed, `number | Promise` resolves to + * `unknown | never`, which is `unknown`, and lets one promise-returning branch + * of several through. + */ +type Synchronous = [Extract>] extends [ + never, +] + ? unknown + : never; + /** * Get the crank methods. * @@ -103,24 +118,30 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { * `work` must be synchronous. The turn is given back the moment it returns, * so work that awaits resumes with the run loop free to start a crank — and * both callers take a savepoint inside it, which would then nest inside that - * crank rather than being the commit point it has to be. The type refuses the - * plain cases and the check below catches the rest; a caller that needs to - * await does it with what `work` hands back. + * crank rather than being the commit point it has to be. A caller that needs + * to await does it with what `work` hands back. + * + * {@link Synchronous} cannot refuse an explicit + * `withStoreOutOfCrank`, since a `Promise` is assignable to a + * `void` return, nor an `unknown` or `any` one — hence the check below. * * @param work - The synchronous work to do while holding the store. * @returns What `work` returned. */ async function withStoreOutOfCrank( - work: () => Result & - (Result extends PromiseLike ? never : unknown), + work: () => Result & Synchronous, ): Promise { await beginOutOfCrank(); try { const result = work(); - // Thrown from inside the `try`, so the `finally` still gives the turn - // back rather than parking the run loop on top of the mistake. - !isPromiseLike(result) || + if (isPromiseLike(result)) { + // Nothing awaits this thenable, and an abandoned rejection is an + // unhandled one, which ends the process. + Promise.resolve(result).catch(() => undefined); + // Thrown from inside the `try`, so the `finally` still gives the turn + // back rather than parking the run loop on top of the mistake. Fail`withStoreOutOfCrank given work that is not synchronous`; + } return result; } finally { endOutOfCrank();