diff --git a/client/packages/core/__tests__/src/Reactor.test.ts b/client/packages/core/__tests__/src/Reactor.test.ts index bce2f07486..64613d2811 100644 --- a/client/packages/core/__tests__/src/Reactor.test.ts +++ b/client/packages/core/__tests__/src/Reactor.test.ts @@ -26,9 +26,10 @@ const zenecaAttrsStore = new AttrsStoreClass( async function waitForLoaded(reactor) { await reactor.querySubs.waitForMetaToLoad(); await reactor.kv.waitForMetaToLoad(); - await reactor.kv.waitForKeyToLoad('pendingMutations'); + await reactor._pendingMutationsLoaded; await reactor.querySubs.flush(); await reactor.kv.flush(); + await reactor.mutations.flush(); } test('querySubs round-trips', async () => { @@ -183,8 +184,7 @@ test('rewrite mutations works with multiple transactions', () => { 'tx-steps': steps, }; reactor._updatePendingMutations((prev) => { - prev.set(k, mut); - return prev; + prev[k] = mut; }); } @@ -478,3 +478,192 @@ test('getLocalId always returns the same id', async () => { expect(id).toStrictEqual([...ids][0]); }); + +async function makeLegacyDb(name, stores, fill) { + const db = await new Promise((resolve, reject) => { + const req = indexedDB.open(name, 1); + req.onupgradeneeded = () => { + for (const storeName of stores) { + req.result.createObjectStore(storeName); + } + }; + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + await new Promise((resolve, reject) => { + const tx = db.transaction(stores, 'readwrite'); + fill(tx); + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); + db.close(); +} + +test('pending mutations persist as one row per mutation', async () => { + const appId = uuid(); + const reactor = new Reactor({ appId }); + reactor._initStorage(IndexedDBStorage); + reactor._setAttrs(zenecaAttrs); + await waitForLoaded(reactor); + + reactor.pushTx([instatx.tx.books[uuid()].update({ title: 'one' })]); + reactor.pushTx([instatx.tx.books[uuid()].update({ title: 'two' })]).catch( + () => {}, + ); + const [ev1, ev2] = [...reactor._pendingMutations().keys()]; + await reactor.mutations.flush(); + + const mutationStore = new IndexedDBStorage(appId, 'mutations'); + const keys = await mutationStore.getAllKeys(); + expect(keys).toContain(ev1); + expect(keys).toContain(ev2); + expect(await mutationStore.getItem(ev1)).toMatchObject({ op: 'transact' }); + + // The kv store no longer holds the old blob + const kvStore = new IndexedDBStorage(appId, 'kv'); + expect(await kvStore.getItem('pendingMutations')).toBeNull(); + + // Deleting a mutation removes just its row + reactor._handleMutationError('error', ev2, { message: 'test' }); + await reactor.mutations.flush(); + const keysAfter = await mutationStore.getAllKeys(); + expect(keysAfter).toContain(ev1); + expect(keysAfter).not.toContain(ev2); +}); + +test('pending mutations round-trip across reloads and unconfirmed ones are re-sent', async () => { + const appId = uuid(); + const reactor = new Reactor({ appId }); + reactor._initStorage(IndexedDBStorage); + reactor._setAttrs(zenecaAttrs); + await waitForLoaded(reactor); + + reactor.pushTx([instatx.tx.books[uuid()].update({ title: 'offline' })]); + const [eventId] = [...reactor._pendingMutations().keys()]; + await reactor.mutations.flush(); + + const reactor2 = new Reactor({ appId }); + const sent = []; + reactor2._sendMutation = (evId, _mut) => { + sent.push(evId); + }; + reactor2._initStorage(IndexedDBStorage); + await waitForLoaded(reactor2); + + expect([...reactor2._pendingMutations().keys()]).toEqual([eventId]); + expect(sent).toEqual([eventId]); +}); + +test('upgrades v6 databases, splitting pending mutations into rows', async () => { + const appId = uuid(); + const mutA = { op: 'transact', 'tx-steps': [], created: 1, order: 1 }; + const mutB = { + op: 'transact', + 'tx-steps': [], + created: 2, + order: 2, + 'tx-id': 5, + }; + await makeLegacyDb( + `instant_${appId}_6`, + ['kv', 'querySubs', 'syncSubs'], + (tx) => { + const kv = tx.objectStore('kv'); + kv.put( + [ + ['evA', mutA], + ['evB', mutB], + ], + 'pendingMutations', + ); + kv.put({ id: 'u1' }, 'currentUser'); + kv.put( + { + objects: { + pendingMutations: { createdAt: 1, updatedAt: 1, size: 0 }, + currentUser: { createdAt: 1, updatedAt: 1, size: 0 }, + }, + }, + '__meta', + ); + tx.objectStore('querySubs').put({ some: 'sub' }, 'hash1'); + }, + ); + + const mutationStore = new IndexedDBStorage(appId, 'mutations'); + expect(await mutationStore.getItem('evA')).toEqual(mutA); + expect(await mutationStore.getItem('evB')).toEqual(mutB); + // The migration writes a meta row so boot can enumerate the mutations + expect((await mutationStore.getItem('__meta')).objects).toHaveProperty( + 'evA', + ); + + const kvStore = new IndexedDBStorage(appId, 'kv'); + expect(await kvStore.getItem('pendingMutations')).toBeNull(); + expect(await kvStore.getItem('currentUser')).toEqual({ id: 'u1' }); + expect( + (await kvStore.getItem('__meta')).objects.pendingMutations, + ).toBeUndefined(); + + const querySubStore = new IndexedDBStorage(appId, 'querySubs'); + expect(await querySubStore.getItem('hash1')).toEqual({ some: 'sub' }); +}); + +test('a reactor booting on migrated v6 data adopts and re-sends unconfirmed mutations', async () => { + const appId = uuid(); + const mutA = { op: 'transact', 'tx-steps': [], created: 1, order: 1 }; + const mutB = { + op: 'transact', + 'tx-steps': [], + created: 2, + order: 2, + 'tx-id': 5, + }; + await makeLegacyDb( + `instant_${appId}_6`, + ['kv', 'querySubs', 'syncSubs'], + (tx) => { + tx.objectStore('kv').put( + [ + ['evA', mutA], + ['evB', mutB], + ], + 'pendingMutations', + ); + }, + ); + + const reactor = new Reactor({ appId }); + const sent = []; + reactor._sendMutation = (evId, _mut) => { + sent.push(evId); + }; + reactor._initStorage(IndexedDBStorage); + await waitForLoaded(reactor); + + expect([...reactor._pendingMutations().keys()].sort()).toEqual([ + 'evA', + 'evB', + ]); + // evA was never confirmed by the server, so it gets re-sent; evB has a + // tx-id and does not + expect(sent).toEqual(['evA']); +}); + +test('upgrades v5 databases through to per-mutation rows', async () => { + const appId = uuid(); + const mutOld = { op: 'transact', 'tx-steps': [], created: 1, order: 1 }; + await makeLegacyDb(`instant_${appId}_5`, ['kv'], (tx) => { + const kv = tx.objectStore('kv'); + // v5 JSON.stringified values before storing + kv.put(JSON.stringify([['evOld', mutOld]]), 'pendingMutations'); + kv.put({ id: 'u5' }, 'currentUser'); + }); + + const mutationStore = new IndexedDBStorage(appId, 'mutations'); + expect(await mutationStore.getItem('evOld')).toEqual(mutOld); + + const kvStore = new IndexedDBStorage(appId, 'kv'); + expect(await kvStore.getItem('pendingMutations')).toBeNull(); + expect(await kvStore.getItem('currentUser')).toEqual({ id: 'u5' }); +}); diff --git a/client/packages/core/__tests__/src/utils/PersistedObject.test.ts b/client/packages/core/__tests__/src/utils/PersistedObject.test.ts index f6f3069545..a0dd4982ac 100644 --- a/client/packages/core/__tests__/src/utils/PersistedObject.test.ts +++ b/client/packages/core/__tests__/src/utils/PersistedObject.test.ts @@ -1,5 +1,5 @@ import 'fake-indexeddb/auto'; -import { test, expect, describe } from 'vitest'; +import { test, expect, describe, vi } from 'vitest'; import { PersistedObject } from '../../../src/utils/PersistedObject'; import { IndexedDBStorage } from '../../../src'; import { randomUUID } from 'crypto'; @@ -423,3 +423,53 @@ test('IndexedDBStorage recovers when the database connection closes', async () = await idb.removeItem('key4'); expect(await idb.getItem('key4')).toBe(null); }); + +test('IndexedDBStorage explicitly commits write transactions', async () => { + const commitSpy = vi.spyOn(IDBTransaction.prototype, 'commit'); + try { + const idb = new IndexedDBStorage(randomUUID(), 'kv'); + + await idb.setItem('key1', 'value1'); + await idb.multiSet([ + ['key2', 'value2'], + ['key3', 'value3'], + ]); + await idb.removeItem('key3'); + + expect(commitSpy).toHaveBeenCalledTimes(3); + expect(await idb.getItem('key1')).toBe('value1'); + expect(await idb.getItem('key2')).toBe('value2'); + expect(await idb.getItem('key3')).toBe(null); + } finally { + commitSpy.mockRestore(); + } +}); + +test('waitForAllKeysToLoad loads every key listed in meta', async () => { + const appId = randomUUID(); + const opts = { + merge: (_k, storage, memory) => memory || storage, + serialize: (_k, x) => x, + parse: (_k, x) => x, + objectSize: (_v) => 0, + logger: devNullLogger, + saveThrottleMs: 0, + gc: null, + }; + const PO = new PersistedObject({ + persister: new IndexedDBStorage(appId, 'mutations'), + ...opts, + }); + PO.updateInPlace((prev) => { + prev.a = 'one'; + prev.b = 'two'; + }); + await PO.flush(); + + const PO2 = new PersistedObject({ + persister: new IndexedDBStorage(appId, 'mutations'), + ...opts, + }); + const value = await PO2.waitForAllKeysToLoad(); + expect(value).toStrictEqual({ a: 'one', b: 'two' }); +}); diff --git a/client/packages/core/src/IndexedDBStorage.ts b/client/packages/core/src/IndexedDBStorage.ts index 36299ec17c..b27cfc68f3 100644 --- a/client/packages/core/src/IndexedDBStorage.ts +++ b/client/packages/core/src/IndexedDBStorage.ts @@ -14,9 +14,9 @@ import { // using their built-in versioning because they have no ability // to roll back and if multiple tabs are active, then you'll just // be stuck. -const version = 6; +const version = 7; -const storeNames = ['kv', 'querySubs', 'syncSubs'] as const; +const storeNames = ['kv', 'querySubs', 'syncSubs', 'mutations'] as const; // Check that we're not missing a store name in storeNames type MissingStoreNames = Exclude< @@ -177,6 +177,112 @@ async function upgrade5To6(appId: string, v6Db: IDBDatabase): Promise { }); } +async function readAllEntries( + db: IDBDatabase, + storeName: string, +): Promise> { + if (!db.objectStoreNames.contains(storeName)) { + return []; + } + return new Promise((resolve, reject) => { + const tx = db.transaction([storeName], 'readonly'); + const cursorReq = tx.objectStore(storeName).openCursor(); + const data: Array<[string, any]> = []; + cursorReq.onerror = (event) => { + reject(event); + }; + cursorReq.onsuccess = () => { + const cursor = cursorReq.result; + if (cursor) { + data.push([cursor.key as string, cursor.value]); + cursor.continue(); + } else { + resolve(data); + } + }; + }); +} + +function parsePendingMutationsBlob(value: any): Array<[string, any]> { + // Older clients stored the pending mutations map as `[...map.entries()]`, + // and clients before version 6 JSON.stringified it first. + const entries = typeof value === 'string' ? JSON.parse(value) : value; + return Array.isArray(entries) ? entries : []; +} + +// Moves the single `pendingMutations` blob out of the kv store and into +// one row per mutation in the `mutations` store. Bounding each write to a +// single mutation keeps write transactions small, so a frozen tab can't +// park the kv lock mid-save and block sibling tabs. +async function splitPendingMutations(db: IDBDatabase): Promise { + const tx = db.transaction(['kv', 'mutations'], 'readwrite'); + const kvStore = tx.objectStore('kv'); + const mutationStore = tx.objectStore('mutations'); + + const blobReq = kvStore.get('pendingMutations'); + const metaReq = kvStore.get(META_KEY); + blobReq.onsuccess = () => { + const entries = blobReq.result + ? parsePendingMutationsBlob(blobReq.result) + : []; + if (entries.length) { + const meta: Meta = { objects: {} }; + for (const [eventId, mutation] of entries) { + mutationStore.put(mutation, eventId); + meta.objects[eventId] = { + createdAt: Date.now(), + updatedAt: Date.now(), + size: 0, + }; + } + mutationStore.put(meta, META_KEY); + } + kvStore.delete('pendingMutations'); + }; + metaReq.onsuccess = () => { + const kvMeta = metaReq.result; + if (kvMeta?.objects?.pendingMutations) { + delete kvMeta.objects.pendingMutations; + kvStore.put(kvMeta, META_KEY); + } + }; + + return new Promise((resolve, reject) => { + tx.oncomplete = () => resolve(); + tx.onerror = (e) => reject(e); + tx.onabort = (e) => reject(e); + }); +} + +async function upgrade6To7(appId: string, v7Db: IDBDatabase): Promise { + const v6Db = await existingDb(`instant_${appId}_6`); + if (v6Db) { + const stores = ['kv', 'querySubs', 'syncSubs'] as const; + const entriesByStore: Array<[string, Array<[string, any]>]> = []; + for (const storeName of stores) { + entriesByStore.push([storeName, await readAllEntries(v6Db, storeName)]); + } + const tx = v7Db.transaction([...stores], 'readwrite'); + for (const [storeName, entries] of entriesByStore) { + const store = tx.objectStore(storeName); + for (const [key, value] of entries) { + store.put(value, key); + } + } + await new Promise((resolve, reject) => { + tx.oncomplete = (e) => resolve(e); + tx.onerror = (e) => reject(e); + tx.onabort = (e) => reject(e); + }); + } else { + // No version 6 db. If a version 5 db exists, bring its data over + // first; it lands in the old shape with the pending mutations blob + // in kv, which the split below moves to the mutations store. + await upgrade5To6(appId, v7Db); + } + await splitPendingMutations(v7Db); +} + // We create many IndexedDBStorage instances that talk to the same // underlying db, but we only get one `onupgradeneeded` event. This holds // the upgrade promises so that we wait until upgrade finishes before @@ -189,6 +295,7 @@ export default class IndexedDBStorage extends StoreInterface { _appId: string; _prefix: string; _dbPromise: Promise; + _resolveUpgradeDone: (() => void) | null = null; constructor(appId: string, storeName: StoreInterfaceStoreName) { super(appId, storeName); @@ -204,6 +311,8 @@ export default class IndexedDBStorage extends StoreInterface { const request = indexedDB.open(this.dbName, 1); request.onerror = (event) => { + // Unblock any siblings waiting on our upgrade + this._resolveUpgradeDone?.(); reject(event); }; @@ -227,10 +336,10 @@ export default class IndexedDBStorage extends StoreInterface { p.then(() => resolve(db)).catch(() => resolve(db)); } } else { - const p = upgrade5To6(this._appId, db).catch((e) => { - logErrorCb('Error upgrading store from version 5 to 6.')(e); + const p = upgrade6To7(this._appId, db).catch((e) => { + logErrorCb('Error upgrading store to version 7.')(e); }); - upgradePromises.set(this.dbName, p); + p.then(() => this._resolveUpgradeDone?.()); p.then(() => resolve(db)).catch(() => resolve(db)); } }; @@ -250,6 +359,14 @@ export default class IndexedDBStorage extends StoreInterface { db.createObjectStore(storeName); } } + // Register the upgrade barrier now: `onupgradeneeded` runs before any + // other connection's open can succeed, so every sibling instance is + // guaranteed to find this promise and wait for the data migration that + // runs in our `onsuccess`. + const done = new Promise((resolve) => { + this._resolveUpgradeDone = resolve; + }); + upgradePromises.set(this.dbName, done); } // Browsers can close IndexedDB connections unexpectedly (backgrounded tabs, @@ -300,6 +417,7 @@ export default class IndexedDBStorage extends StoreInterface { transaction.oncomplete = () => resolve(); transaction.onerror = () => reject(transaction.error); transaction.onabort = () => reject(transaction.error); + transaction.commit?.(); }); }); } @@ -316,6 +434,7 @@ export default class IndexedDBStorage extends StoreInterface { transaction.oncomplete = () => resolve(); transaction.onerror = () => reject(transaction.error); transaction.onabort = () => reject(transaction.error); + transaction.commit?.(); }); }); } @@ -330,6 +449,7 @@ export default class IndexedDBStorage extends StoreInterface { transaction.oncomplete = () => resolve(); transaction.onerror = () => reject(transaction.error); transaction.onabort = () => reject(transaction.error); + transaction.commit?.(); }); }); } diff --git a/client/packages/core/src/Reactor.js b/client/packages/core/src/Reactor.js index 0a07adcf65..1de36e77c4 100644 --- a/client/packages/core/src/Reactor.js +++ b/client/packages/core/src/Reactor.js @@ -171,22 +171,8 @@ function querySubToStorage(_key, sub) { return jsonSub; } -function kvFromStorage(key, x) { - switch (key) { - case 'pendingMutations': - return new Map(typeof x === 'string' ? JSON.parse(x) : x); - default: - return x; - } -} - -function kvToStorage(key, x) { - switch (key) { - case 'pendingMutations': - return [...x.entries()]; - default: - return x; - } +function identity(_key, x) { + return x; } function onMergeQuerySub(_k, storageSub, inMemorySub) { @@ -227,6 +213,9 @@ export default class Reactor { /** @type {PersistedObject} */ kv; + /** @type {PersistedObject} */ + mutations; + /** @type {SyncTable} */ _syncTable; /** @type {InstantStream} */ @@ -282,6 +271,17 @@ export default class Reactor { _currentUserCached = { isLoading: true, error: undefined, user: undefined }; _beforeUnloadCbs = []; _dataForQueryCache = {}; + // True once the mutations store has loaded (or a local write happened), + // which gates optimistic query results the way the old single + // `pendingMutations` kv key did. + _pendingMutationsReady = false; + /** @type {{version: number, muts: Map} | null} */ + _pendingMutationsCache = null; + /** @type {Promise} */ + _pendingMutationsLoaded = Promise.resolve(); + // eventIds found in storage during load that no in-memory write claimed; + // unconfirmed ones are re-sent once the full set has loaded. + _recoveredMutationIds = new Set(); /** @type {Logger} */ _log; _pendingTxCleanupTimeout; @@ -449,9 +449,7 @@ export default class Reactor { } _onQuerySubLoaded(hash) { - this.kv - .waitForKeyToLoad('pendingMutations') - .then(() => this.notifyOne(hash)); + this._pendingMutationsLoaded.then(() => this.notifyOne(hash)); } _initStorage(Storage) { @@ -475,8 +473,8 @@ export default class Reactor { this.kv = new PersistedObject({ persister: new Storage(this.config.appId, 'kv'), merge: this._onMergeKv, - serialize: kvToStorage, - parse: kvFromStorage, + serialize: identity, + parse: identity, objectSize: () => 0, logger: this._log, saveThrottleMs: 100, @@ -484,16 +482,30 @@ export default class Reactor { // Don't GC the kv store gc: null, }); - this.kv.onKeyLoaded = (k) => { - if (k === 'pendingMutations') { - this.notifyAll(); - } - }; + // One key per pending mutation so each save writes a single small row + // instead of rewriting the whole map. A frozen tab can otherwise park + // the store lock mid-save and block sibling tabs from booting. + this.mutations = new PersistedObject({ + persister: new Storage(this.config.appId, 'mutations'), + merge: this._onMergeMutation, + serialize: identity, + parse: identity, + objectSize: () => 0, + logger: this._log, + saveThrottleMs: 100, + idleCallbackMaxWaitMs: 100, + // Don't GC the mutations store + gc: null, + }); // Trigger immediate load for pendingMutations and currentUser - this.kv.waitForKeyToLoad('pendingMutations'); + this._pendingMutationsLoaded = this.mutations + .waitForAllKeysToLoad() + .catch((e) => this._log.error('Failed to load pending mutations', e)) + .then(() => this._onPendingMutationsLoaded()); this.kv.waitForKeyToLoad(currentUserKey); this._beforeUnloadCbs.push(() => { this.kv.flush(); + this.mutations.flush(); this.querySubs.flush(); }); } @@ -554,27 +566,41 @@ export default class Reactor { this._instantStream.onConnectionStatusChange(status); } - _onMergeKv = (key, storageV, inMemoryV) => { - switch (key) { - case 'pendingMutations': { - const storageEntries = storageV?.entries() ?? []; - const inMemoryEntries = inMemoryV?.entries() ?? []; - const muts = new Map([...storageEntries, ...inMemoryEntries]); - const rewrittenStorageMuts = storageV - ? this._rewriteMutationsSorted(this.attrs, storageV) - : []; - rewrittenStorageMuts.forEach(([k, mut]) => { - if (!inMemoryV?.pendingMutations?.has(k) && !mut['tx-id']) { - this._sendMutation(k, mut); - } - }); - return muts; - } - default: - return inMemoryV || storageV; + _onMergeKv = (_key, storageV, inMemoryV) => { + return inMemoryV || storageV; + }; + + _onMergeMutation = (eventId, storageV, inMemoryV) => { + // A mutation in storage that no in-memory write claimed came from a + // previous session (or another tab). If the server never confirmed it, + // re-send it once the full set has loaded. + if (storageV && !inMemoryV && !storageV['tx-id']) { + this._recoveredMutationIds.add(eventId); } + return inMemoryV || storageV; }; + _onPendingMutationsLoaded() { + this._pendingMutationsReady = true; + // Loading keys doesn't bump the store version, so bump it here to + // invalidate caches keyed on it (dataForQuery, _pendingMutations). + this.mutations.updateInPlace(() => {}); + const recovered = this._recoveredMutationIds; + this._recoveredMutationIds = new Set(); + if (recovered.size) { + const rewritten = this._rewriteMutationsSorted( + this.attrs, + this._pendingMutations(), + ); + rewritten.forEach(([eventId, mut]) => { + if (recovered.has(eventId) && !mut['tx-id']) { + this._sendMutation(eventId, mut); + } + }); + } + this.notifyAll(); + } + _flushEnqueuedRoomData(roomId) { const enqueuedUserPresence = this._presence[roomId]?.result?.user; const enqueuedBroadcasts = this._broadcastQueue[roomId]; @@ -749,9 +775,7 @@ export default class Reactor { // We know we've changed the mutations to fix the attr ids and removed // processed attrs, so we'll persist those changes to prevent optimisticAttrs // from using old attr definitions - this.kv.updateInPlace((prev) => { - prev.pendingMutations = rewrittenMutations; - }); + this._setPendingMutations(rewrittenMutations); } const mutations = sortedMutationEntries(rewrittenMutations.entries()); @@ -828,11 +852,11 @@ export default class Reactor { // update pendingMutation with server-side tx-id this._updatePendingMutations((prev) => { - prev.set(eventId, { - ...prev.get(eventId), + prev[eventId] = { + ...prev[eventId], 'tx-id': txId, confirmed: Date.now(), - }); + }; }); const newAttrs = []; @@ -943,15 +967,44 @@ export default class Reactor { return this._instantStream.createReadStream(opts); } + // Read-only Map view over the mutations store, cached per store version. + // Mutate through _updatePendingMutations, never through this Map. _pendingMutations() { - return this.kv.currentValue.pendingMutations ?? new Map(); + const version = this.mutations.version(); + if ( + !this._pendingMutationsCache || + this._pendingMutationsCache.version !== version + ) { + this._pendingMutationsCache = { + version, + muts: new Map(Object.entries(this.mutations.currentValue)), + }; + } + return this._pendingMutationsCache.muts; } _updatePendingMutations(f) { - this.kv.updateInPlace((prev) => { - const muts = prev.pendingMutations ?? new Map(); - prev.pendingMutations = muts; - f(muts); + this._pendingMutationsReady = true; + this.mutations.updateInPlace((prev) => { + f(prev); + }); + } + + // Replaces the pending mutations with a rewritten map. Only entries whose + // tx-steps actually changed get reassigned, so we only persist those rows. + _setPendingMutations(muts) { + this._updatePendingMutations((prev) => { + for (const eventId of Object.keys(prev)) { + if (!muts.has(eventId)) { + delete prev[eventId]; + } + } + for (const [eventId, mut] of muts) { + const prevMut = prev[eventId]; + if (!prevMut || prevMut['tx-steps'] !== mut['tx-steps']) { + prev[eventId] = mut; + } + } }); } @@ -965,8 +1018,7 @@ export default class Reactor { if (mut && (status !== 'timeout' || !mut['tx-id'])) { this._updatePendingMutations((prev) => { - prev.delete(eventId); - return prev; + delete prev[eventId]; }); this._inFlightMutationEventIds.delete(eventId); const errDetails = { @@ -1407,10 +1459,10 @@ export default class Reactor { return { error: errorMessage }; } if (!this.querySubs) return; - if (!this.kv.currentValue.pendingMutations) return; + if (!this._pendingMutationsReady) return; const querySubVersion = this.querySubs.version(); const querySubs = this.querySubs.currentValue; - const pendingMutationsVersion = this.kv.version(); + const pendingMutationsVersion = this.mutations.version(); const pendingMutations = this._pendingMutations(); const { q, result } = querySubs[hash] || {}; @@ -1504,10 +1556,7 @@ export default class Reactor { } loadedNotifyAll() { - this.kv - .waitForKeyToLoad('pendingMutations') - .then(() => this.notifyAll()) - .catch(() => this.notifyAll()); + this._pendingMutationsLoaded.then(() => this.notifyAll()); } /** Applies transactions locally and sends transact message to server */ @@ -1564,7 +1613,7 @@ export default class Reactor { order, }; this._updatePendingMutations((prev) => { - prev.set(eventId, mutation); + prev[eventId] = mutation; }); const dfd = new Deferred(); @@ -1655,9 +1704,7 @@ export default class Reactor { ); if (rewrittenMutations !== this._pendingMutations()) { // Persist rewritten mutations to avoid stale attr ids in future txs. - this.kv.updateInPlace((prev) => { - prev.pendingMutations = rewrittenMutations; - }); + this._setPendingMutations(rewrittenMutations); } const muts = sortedMutationEntries(rewrittenMutations.entries()); @@ -1682,9 +1729,9 @@ export default class Reactor { } this._updatePendingMutations((prev) => { - for (const [eventId, mut] of Array.from(prev.entries())) { + for (const [eventId, mut] of Object.entries(prev)) { if (mut['tx-id'] && mut['tx-id'] <= minProcessedTxId) { - prev.delete(eventId); + delete prev[eventId]; } } }); @@ -1703,12 +1750,12 @@ export default class Reactor { const now = Date.now(); this._updatePendingMutations((prev) => { - for (const [eventId, mut] of Array.from(prev.entries())) { + for (const [eventId, mut] of Object.entries(prev)) { if ( mut.confirmed && mut.confirmed + this._pendingTxCleanupTimeout < now ) { - prev.delete(eventId); + delete prev[eventId]; } } }); @@ -2258,6 +2305,7 @@ export default class Reactor { // Make sure everything is written to storage before we tell the // other tab to refetch await this.kv.flush(); + await this.mutations.flush(); this._broadcastChannel?.postMessage({ type: 'auth' }); } catch (error) { console.error('Error posting message to broadcast channel', error); @@ -2322,17 +2370,17 @@ export default class Reactor { this._updatePendingMutations((prev) => { // Mark all pending mutations with an error, since we won't be able to // deliver the result - for (const [eventId, _v] of prev.entries()) { + for (const eventId of Object.keys(prev)) { if (this.mutationDeferredStore.get(eventId)) { this._finishTransaction('error', eventId, { message: 'User changed while transaction was in progress.', type: 'user-changed', }); } + delete prev[eventId]; } - - prev.clear(); }); + this.mutations.clearUnloadedKeys(); this._reconnectTimeoutMs = 0; this._transport.close(); diff --git a/client/packages/core/src/utils/PersistedObject.ts b/client/packages/core/src/utils/PersistedObject.ts index 2598e5d660..eef9cb78da 100644 --- a/client/packages/core/src/utils/PersistedObject.ts +++ b/client/packages/core/src/utils/PersistedObject.ts @@ -35,7 +35,11 @@ export type Meta = { objects: Record; }; -export type StoreInterfaceStoreName = 'kv' | 'querySubs' | 'syncSubs'; +export type StoreInterfaceStoreName = + | 'kv' + | 'querySubs' + | 'syncSubs' + | 'mutations'; export abstract class StoreInterface { constructor(appId: string, storeName: StoreInterfaceStoreName) {} @@ -210,6 +214,16 @@ export class PersistedObject { return this.currentValue[k]; } + // Loads every key we know about from meta and resolves once they have + // all been merged into currentValue. Used by stores that need their full + // contents at boot, like pending mutations. + public async waitForAllKeysToLoad(): Promise> { + const meta = await this._getMeta(); + const keys = Object.keys(meta?.objects ?? {}) as K[]; + await Promise.all(keys.map((k) => this.waitForKeyToLoad(k))); + return this.currentValue; + } + // Used for tests public async waitForMetaToLoad() { return this._getMeta(); diff --git a/client/packages/version/src/version.ts b/client/packages/version/src/version.ts index 78c9301904..edaa696c85 100644 --- a/client/packages/version/src/version.ts +++ b/client/packages/version/src/version.ts @@ -2,6 +2,6 @@ // Update the version here and merge your code to main to // publish a new version of all of the packages to npm. -const version = 'v1.0.63'; +const version = 'v1.0.64'; export { version };