From 8fed07edd37d40e1f1c9a71f4ea406c727e9fd3f Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 12:37:46 -0700 Subject: [PATCH 01/18] Add instant-cli backup commands Adds 'instant-cli backup list' and 'instant-cli backup download', built on a new BackupsManager in @instantdb/platform that both surfaces the backup endpoints and documents the archive ordering contract (config.json, then entities/*.jsonl, then files/). Also fixes the dashboard treating a 404 from the storage-files endpoint as 'no storage files': the server signals the empty case with a 200 and the done sentinel, so a 404 can only mean the backup is missing or expired, and now fails the download instead of silently omitting files. --- .../cli/__tests__/backupDownload.test.ts | 192 +++++++++++ client/packages/cli/__tests__/backups.test.ts | 202 +++++++++++ client/packages/cli/package.json | 1 + .../cli/src/commands/backup/download.ts | 212 ++++++++++++ .../packages/cli/src/commands/backup/list.ts | 66 ++++ client/packages/cli/src/index.ts | 60 ++++ client/packages/cli/src/lib/backupDownload.ts | 320 ++++++++++++++++++ client/packages/cli/src/lib/backups.ts | 33 ++ client/packages/cli/src/lib/platformApi.ts | 11 + client/packages/cli/src/lib/webhooks.ts | 11 +- .../platform/__tests__/src/backups.test.ts | 139 ++++++++ client/packages/platform/src/api.ts | 15 + client/packages/platform/src/backups.ts | 248 ++++++++++++++ client/packages/platform/src/index.ts | 8 + client/pnpm-lock.yaml | 18 +- .../components/dash/BackupDownloadDialog.tsx | 17 +- 16 files changed, 1525 insertions(+), 28 deletions(-) create mode 100644 client/packages/cli/__tests__/backupDownload.test.ts create mode 100644 client/packages/cli/__tests__/backups.test.ts create mode 100644 client/packages/cli/src/commands/backup/download.ts create mode 100644 client/packages/cli/src/commands/backup/list.ts create mode 100644 client/packages/cli/src/lib/backupDownload.ts create mode 100644 client/packages/cli/src/lib/backups.ts create mode 100644 client/packages/cli/src/lib/platformApi.ts create mode 100644 client/packages/platform/__tests__/src/backups.test.ts create mode 100644 client/packages/platform/src/backups.ts diff --git a/client/packages/cli/__tests__/backupDownload.test.ts b/client/packages/cli/__tests__/backupDownload.test.ts new file mode 100644 index 0000000000..df09e49bee --- /dev/null +++ b/client/packages/cli/__tests__/backupDownload.test.ts @@ -0,0 +1,192 @@ +import { test, expect, describe, beforeAll, afterAll, afterEach } from 'vitest'; +import { createServer, type Server } from 'node:http'; +import { existsSync } from 'node:fs'; +import { readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import zlib from 'node:zlib'; +import { downloadBackupToFile } from '../src/lib/backupDownload.ts'; + +// Exercises the real pipeline end-to-end against a local HTTP server: zstd +// decompression of entity shards, canonical entry order (config.json, then +// entities/*.jsonl, then files/), and the partial-file rename. + +// Not yet in this @types/node version, same as createZstdDecompress in the +// pipeline itself. +const zstd = (s: string): Buffer => + (zlib as any).zstdCompressSync(Buffer.from(s)); + +const bodies: Record = { + '/config.json': { body: zstd('{"schema":{}}'), encoding: 'zstd' }, + '/entities/todos.jsonl': { + body: zstd('{"entity":{"id":"1"}}\n'), + encoding: 'zstd', + }, + '/entities/$files.jsonl': { + body: zstd('{"entity":{"location-id":"loc-1"}}\n'), + encoding: 'zstd', + }, + '/blobs/loc-1': { body: Buffer.from('blob-one') }, + '/blobs/loc-2': { body: Buffer.from('blob-two') }, +}; + +let server: Server; +let baseUrl: string; + +beforeAll(async () => { + server = createServer((req, res) => { + const found = bodies[req.url ?? '']; + if (!found) { + res.writeHead(404).end(); + return; + } + const headers: Record = {}; + if (found.encoding) headers['content-encoding'] = found.encoding; + res.writeHead(200, headers).end(found.body); + }); + server.listen(0); + await once(server, 'listening'); + const address = server.address(); + if (typeof address === 'string' || address === null) { + throw new Error('Expected a TCP address'); + } + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterAll(() => { + server.close(); +}); + +const backup = { + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 16, + dbSize: 100, + uncompressedSize: 40, + description: null, + expiresAt: new Date('2026-08-08T00:00:00Z'), +}; + +const storageFiles = [ + { locationId: 'loc-1', path: 'a.png', url: () => `${baseUrl}/blobs/loc-1` }, + { locationId: 'loc-2', path: 'b.png', url: () => `${baseUrl}/blobs/loc-2` }, +]; + +const manager = { + listFiles: async (_backupId: string) => [ + { name: 'config.json', size: 10 }, + { name: 'entities/todos.jsonl', size: 10 }, + { name: 'entities/$files.jsonl', size: 10 }, + ], + getFileUrl: async (_backupId: string, name: string) => `${baseUrl}/${name}`, + streamStorageFiles: async function* ( + _backupId: string, + _opts?: { signal?: AbortSignal }, + ) { + for (const f of storageFiles) { + yield { locationId: f.locationId, path: f.path, url: f.url() }; + } + }, +} as any; + +const outPath = join(tmpdir(), `backup-download-test-${process.pid}.zip`); + +afterEach(async () => { + await rm(outPath, { force: true }); + await rm(`${outPath}.partial`, { force: true }); +}); + +describe('downloadBackupToFile', () => { + test('writes a zip with the canonical entry order', async () => { + const result = await downloadBackupToFile({ + manager, + backup, + outPath, + signal: new AbortController().signal, + onProgress: () => {}, + }); + + expect(result.entities).toBe(2); + expect(result.files).toBe(2); + expect(existsSync(`${outPath}.partial`)).toBe(false); + + const { ZipReader, Uint8ArrayReader, TextWriter } = await import( + '@zip.js/zip.js' + ); + const reader = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await readFile(outPath))), + ); + const entries = await reader.getEntries(); + + // Entry order is the restore contract: config first, all entity shards + // before any storage blob. + expect(entries.map((e) => e.filename)).toEqual([ + 'config.json', + 'entities/todos.jsonl', + 'entities/$files.jsonl', + 'files/loc-1', + 'files/loc-2', + ]); + + // Entity shards land decompressed; blobs land verbatim. + const readText = (entry: any) => entry.getData(new TextWriter()); + expect(await readText(entries[0])).toBe('{"schema":{}}'); + expect(await readText(entries[1])).toBe('{"entity":{"id":"1"}}\n'); + expect(await readText(entries[3])).toBe('blob-one'); + expect(await readText(entries[4])).toBe('blob-two'); + await reader.close(); + }); + + test('removes the partial file when a fetch fails', async () => { + const failingManager = { + ...manager, + getFileUrl: async (_backupId: string, name: string) => + `${baseUrl}/missing-${name}`, + }; + + await expect( + downloadBackupToFile({ + manager: failingManager, + backup, + outPath, + signal: new AbortController().signal, + onProgress: () => {}, + }), + ).rejects.toThrow(/Failed to fetch config.json/); + + expect(existsSync(outPath)).toBe(false); + expect(existsSync(`${outPath}.partial`)).toBe(false); + }); + + test('aborting removes the partial file', async () => { + const controller = new AbortController(); + const slowManager = { + ...manager, + streamStorageFiles: async function* () { + yield { + locationId: 'loc-1', + path: 'a.png', + url: `${baseUrl}/blobs/loc-1`, + }; + controller.abort(); + // Give the pipeline a moment to observe the abort mid-drain. + await new Promise((resolve) => setTimeout(resolve, 20)); + }, + }; + + await expect( + downloadBackupToFile({ + manager: slowManager, + backup, + outPath, + signal: controller.signal, + onProgress: () => {}, + }), + ).rejects.toThrow(); + + expect(existsSync(outPath)).toBe(false); + expect(existsSync(`${outPath}.partial`)).toBe(false); + }); +}); diff --git a/client/packages/cli/__tests__/backups.test.ts b/client/packages/cli/__tests__/backups.test.ts new file mode 100644 index 0000000000..cf58354d50 --- /dev/null +++ b/client/packages/cli/__tests__/backups.test.ts @@ -0,0 +1,202 @@ +import { test, expect, describe, vi, beforeEach } from 'vitest'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { Effect, Layer, Logger } from 'effect'; +import { GlobalOpts } from '../src/context/globalOpts.ts'; +import { AuthToken } from '../src/context/authToken.ts'; +import { CurrentApp } from '../src/context/currentApp.ts'; + +vi.mock('../src/index.ts', () => ({})); + +const state = vi.hoisted(() => ({ + manager: undefined as any, + downloadResult: undefined as any, + downloadError: undefined as Error | undefined, + downloadCalls: [] as any[], + promptResponses: [] as unknown[], +})); + +// Mock at the SDK boundary: any `new InstantPlatformApi(...)` returns a stub +// whose `.backups(appId)` is the per-test fake manager. +vi.mock('@instantdb/platform', async (importOriginal) => { + const orig: any = await importOriginal(); + return { + ...orig, + PlatformApi: class { + backups(_appId: string) { + return state.manager; + } + }, + }; +}); + +// The download pipeline is network- and disk-heavy; the command tests only +// care that it's invoked with the right backup and destination. +vi.mock('../src/lib/backupDownload.ts', () => ({ + downloadBackupToFile: vi.fn(async (opts: any) => { + state.downloadCalls.push(opts); + if (state.downloadError) throw state.downloadError; + return state.downloadResult; + }), +})); + +vi.mock('../src/ui/lib.ts', async (importOriginal) => { + const orig: any = await importOriginal(); + return { + ...orig, + renderUnwrap: () => { + if (state.promptResponses.length === 0) { + return Promise.reject(new Error('No prompt response queued')); + } + return Promise.resolve(state.promptResponses.shift()); + }, + }; +}); + +const { backupListCmd } = await import('../src/commands/backup/list.ts'); +const { backupDownloadCmd } = await import( + '../src/commands/backup/download.ts' +); + +let logs: string[] = []; + +const makeBackup = (overrides: any = {}) => ({ + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 1000, + dbSize: 2000, + uncompressedSize: 3000, + description: 'Automated Daily Snapshot', + expiresAt: new Date('2026-08-08T00:00:00Z'), + ...overrides, +}); + +const buildManager = (backups: any[]) => ({ + list: vi.fn(async () => backups), +}); + +const outPath = () => join(tmpdir(), `backup-test-${Date.now()}.zip`); + +const run = (effect: any, opts: { yes: boolean }) => + Effect.runPromise( + effect.pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(GlobalOpts, { yes: opts.yes }), + Layer.succeed(AuthToken, { + getAuthToken: Effect.succeed('test-token'), + getSource: Effect.succeed('env' as const), + setAuthToken: () => Effect.succeed(undefined), + }), + Layer.succeed(CurrentApp, { + appId: 'test-app', + source: 'env' as const, + }), + Logger.replace( + Logger.defaultLogger, + Logger.make(({ message }) => { + logs.push(String(message)); + }), + ), + ), + ), + ), + ); + +beforeEach(() => { + logs = []; + state.manager = buildManager([]); + state.downloadResult = { entities: 3, files: 2, zipBytes: 1234 }; + state.downloadError = undefined; + state.downloadCalls = []; + state.promptResponses = []; +}); + +describe('backup list', () => { + test('renders backups', async () => { + state.manager = buildManager([makeBackup()]); + await run(backupListCmd({}), { yes: true }); + const output = logs.join('\n'); + expect(output).toContain('2026-08-01 00:00 UTC'); + expect(output).toContain('ID: backup-1'); + expect(output).toContain('Description: Automated Daily Snapshot'); + expect(output).toContain('Expires: 2026-08-08 00:00 UTC'); + }); + + test('outputs JSON with --json', async () => { + state.manager = buildManager([makeBackup()]); + await run(backupListCmd({ json: true }), { yes: true }); + const parsed = JSON.parse(logs.join('\n')); + expect(parsed).toHaveLength(1); + expect(parsed[0].id).toBe('backup-1'); + }); + + test('handles no backups', async () => { + await run(backupListCmd({}), { yes: true }); + expect(logs.join('\n')).toContain('No backups yet.'); + }); +}); + +describe('backup download', () => { + test('downloads by id', async () => { + const backup = makeBackup(); + state.manager = buildManager([backup]); + const out = outPath(); + await run(backupDownloadCmd('backup-1', { out }), { yes: true }); + expect(state.downloadCalls).toHaveLength(1); + expect(state.downloadCalls[0].backup).toEqual(backup); + expect(state.downloadCalls[0].outPath).toBe(out); + expect(logs.join('\n')).toContain('Saved 3 namespaces and 2 storage files'); + }); + + test('errors on an unknown id', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('nope', { out: outPath() }), { yes: true }), + ).rejects.toThrow(/No backup found with id nope/); + }); + + test('--latest picks the newest backup', async () => { + const older = makeBackup(); + const newer = makeBackup({ + id: 'backup-2', + backupAt: new Date('2026-08-02T00:00:00Z'), + }); + state.manager = buildManager([older, newer]); + await run(backupDownloadCmd(undefined, { latest: true, out: outPath() }), { + yes: true, + }); + expect(state.downloadCalls[0].backup.id).toBe('backup-2'); + }); + + test('requires an id or --latest when prompts are skipped', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd(undefined, {}), { yes: true }), + ).rejects.toThrow(/Must specify a backup id or --latest/); + }); + + test('prompts for a backup and confirmation interactively', async () => { + const backup = makeBackup(); + state.manager = buildManager([backup]); + // First the backup picker, then the download confirmation. + state.promptResponses = [backup, true]; + await run(backupDownloadCmd(undefined, { out: outPath() }), { + yes: false, + }); + expect(state.downloadCalls).toHaveLength(1); + expect(state.downloadCalls[0].backup).toEqual(backup); + }); + + test('reports a cancelled download', async () => { + state.manager = buildManager([makeBackup()]); + state.downloadError = Object.assign(new Error('aborted'), { + name: 'AbortError', + }); + await run(backupDownloadCmd('backup-1', { out: outPath() }), { + yes: true, + }); + expect(logs.join('\n')).toContain('Download cancelled.'); + }); +}); diff --git a/client/packages/cli/package.json b/client/packages/cli/package.json index 601eea07e9..a2936192c7 100644 --- a/client/packages/cli/package.json +++ b/client/packages/cli/package.json @@ -34,6 +34,7 @@ "@instantdb/core": "workspace:*", "@instantdb/platform": "workspace:*", "@instantdb/version": "workspace:*", + "@zip.js/zip.js": "^2.8.34", "acorn": "^8.15.0", "acorn-typescript": "^1.4.13", "ansi-escapes": "4.3.2", diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts new file mode 100644 index 0000000000..0d272767c6 --- /dev/null +++ b/client/packages/cli/src/commands/backup/download.ts @@ -0,0 +1,212 @@ +import { existsSync } from 'node:fs'; +import path from 'node:path'; +import ansiEscapes from 'ansi-escapes'; +import chalk from 'chalk'; +import { Effect } from 'effect'; +import throttle from 'lodash.throttle'; +import { + backupZipName, + type AppBackup, + type BackupsManager, +} from '@instantdb/platform'; +import type { backupDownloadDef, OptsFromCommand } from '../../index.ts'; +import { BadArgsError } from '../../errors.ts'; +import { GlobalOpts } from '../../context/globalOpts.ts'; +import { PlatformApiError } from '../../context/platformApi.ts'; +import { buildBackupsManager, useBackupsManager } from '../../lib/backups.ts'; +import { + downloadBackupToFile, + type BackupDownloadProgress, + type BackupDownloadResult, +} from '../../lib/backupDownload.ts'; +import { promptOk, runUIEffect } from '../../lib/ui.ts'; +import { UI } from '../../ui/index.ts'; +import { formatBackupDate, formatBytes } from './list.ts'; + +const pickBackup = ( + backups: AppBackup[], + backupId: string | undefined, + opts: { latest?: boolean }, +) => + Effect.gen(function* () { + if (backupId) { + const found = backups.find((b) => b.id === backupId); + if (!found) { + return yield* BadArgsError.make({ + message: `No backup found with id ${backupId}.`, + }); + } + return found; + } + + // The server returns newest first; sort anyway so --latest can't silently + // pick the wrong one. + const sorted = [...backups].sort( + (a, b) => b.backupAt.getTime() - a.backupAt.getTime(), + ); + if (opts.latest) { + return sorted[0]; + } + + const { yes } = yield* GlobalOpts; + if (yes) { + return yield* BadArgsError.make({ + message: 'Must specify a backup id or --latest', + }); + } + + return yield* runUIEffect( + new UI.Select({ + options: sorted.map((backup) => ({ + label: + formatBackupDate(backup.backupAt) + + (backup.description ? ` — ${backup.description}` : '') + + ` ${chalk.dim(`(${backup.id})`)}`, + value: backup, + })), + promptText: 'Select a backup to download:', + }), + ); + }); + +// Single-line progress on a TTY, nothing otherwise. +function makeProgressRenderer() { + const stream = process.stderr; + if (!stream.isTTY) { + return { update: (_p: BackupDownloadProgress) => {}, done: () => {} }; + } + let wrote = false; + const write = (p: BackupDownloadProgress) => { + const parts: string[] = []; + parts.push( + p.entitiesTotal == null + ? 'listing namespaces…' + : `namespaces ${p.entitiesCompleted}/${p.entitiesTotal}`, + ); + if (p.filesTotal !== 0) { + parts.push( + p.filesTotal == null + ? 'listing storage files…' + : `storage files ${p.filesCompleted}/${p.filesTotal}`, + ); + } + let bytes = formatBytes(p.zipBytes); + if (p.bytesTotal != null && p.bytesTotal > 0) { + const pct = Math.min(100, Math.round((p.bytesRead / p.bytesTotal) * 100)); + bytes += ` (${pct}%)`; + } + parts.push(bytes); + if (p.currentEntry) { + parts.push(p.currentEntry); + } + let line = parts.join(' · '); + const width = stream.columns || 80; + if (line.length >= width) { + line = line.slice(0, Math.max(0, width - 2)) + '…'; + } + stream.write(ansiEscapes.eraseLine + ansiEscapes.cursorLeft + line); + wrote = true; + }; + const throttled = throttle(write, 100); + return { + update: throttled, + done: () => { + throttled.cancel(); + if (wrote) { + stream.write(ansiEscapes.eraseLine + ansiEscapes.cursorLeft); + } + }, + }; +} + +// Returns null when the download was cancelled (ctrl-c). A second ctrl-c +// falls through to Node's default handler and kills the process outright. +async function runDownload( + manager: BackupsManager, + backup: AppBackup, + outPath: string, +): Promise { + const controller = new AbortController(); + const onSigint = () => controller.abort(); + process.once('SIGINT', onSigint); + const progress = makeProgressRenderer(); + try { + return await downloadBackupToFile({ + manager, + backup, + outPath, + signal: controller.signal, + onProgress: progress.update, + }); + } catch (e) { + if ((e as { name?: string })?.name === 'AbortError') { + return null; + } + throw e; + } finally { + process.removeListener('SIGINT', onSigint); + progress.done(); + } +} + +export const backupDownloadCmd = Effect.fn(function* ( + backupId: string | undefined, + opts: OptsFromCommand, +) { + const backups = yield* useBackupsManager( + (m) => m.list(), + 'Error listing backups', + ); + + if (backups.length === 0) { + yield* Effect.log('No backups yet.'); + return; + } + + const backup = yield* pickBackup(backups, backupId, opts); + + // Upper bound: everything stored uncompressed. Lower bound: everything + // compressed at a ~4x DEFLATE ratio, best case for text/JSON; storage files + // vary wildly, so the actual zip lands somewhere inside the range. + const backupBytes = backup.uncompressedSize ?? backup.dbSize; + const hasSizes = backupBytes != null && backup.filesSize != null; + const totalBytes = (backupBytes ?? 0) + (backup.filesSize ?? 0); + const estimate = hasSizes + ? ` The zip file will be between ${formatBytes(Math.round(totalBytes / 4))} and ${formatBytes(totalBytes)}, depending on the compression ratio.` + : ''; + + const ok = yield* promptOk({ + promptText: `Download the backup from ${formatBackupDate(backup.backupAt)}?${estimate}`, + }); + if (!ok) return; + + const outPath = path.resolve(opts.out ?? backupZipName(backup)); + if (existsSync(outPath)) { + const overwrite = yield* promptOk({ + promptText: `${path.basename(outPath)} already exists. Overwrite?`, + }); + if (!overwrite) return; + } + + const manager = yield* buildBackupsManager; + yield* Effect.log(`Downloading to ${outPath}`); + + const result = yield* Effect.tryPromise({ + try: () => runDownload(manager, backup, outPath), + catch: (e) => + new PlatformApiError({ message: 'Error downloading backup', cause: e }), + }); + + if (result === null) { + yield* Effect.log('Download cancelled.'); + return; + } + + const filesPart = + result.files > 0 + ? ` and ${result.files.toLocaleString()} storage files` + : ''; + yield* Effect.log( + `Saved ${result.entities.toLocaleString()} namespaces${filesPart} (${formatBytes(result.zipBytes)})`, + ); +}); diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts new file mode 100644 index 0000000000..ececc7952b --- /dev/null +++ b/client/packages/cli/src/commands/backup/list.ts @@ -0,0 +1,66 @@ +import chalk from 'chalk'; +import { Effect } from 'effect'; +import type { AppBackup } from '@instantdb/platform'; +import type { backupListDef, OptsFromCommand } from '../../index.ts'; +import { useBackupsManager } from '../../lib/backups.ts'; + +export const formatBackupDate = (date: Date) => + `${date.toISOString().replace('T', ' ').slice(0, 16)} UTC`; + +export function formatBytes(n: number): string { + // Decimal (1000-based) units with SI labels, to match how macOS/Finder + // reports file sizes. + if (n < 1000) return `${n} B`; + const units = ['KB', 'MB', 'GB', 'TB']; + let i = -1; + let v = n; + do { + v /= 1000; + i++; + } while (v >= 1000 && i < units.length - 1); + const digits = v < 10 ? 2 : v < 100 ? 1 : 0; + return `${v.toFixed(digits)} ${units[i]}`; +} + +export const renderBackup = (backup: AppBackup) => + Effect.gen(function* () { + yield* Effect.log(chalk.cyan(formatBackupDate(backup.backupAt))); + yield* Effect.log(` ID: ${backup.id}`); + if (backup.description) { + yield* Effect.log(` Description: ${backup.description}`); + } + if (backup.dbSize != null) { + yield* Effect.log(` Database size: ${formatBytes(backup.dbSize)}`); + } + if (backup.filesSize != null) { + yield* Effect.log( + ` Storage files size: ${formatBytes(backup.filesSize)}`, + ); + } + if (backup.expiresAt) { + yield* Effect.log(` Expires: ${formatBackupDate(backup.expiresAt)}`); + } + }); + +export const backupListCmd = Effect.fn(function* ( + opts: OptsFromCommand, +) { + const backups = yield* useBackupsManager( + (m) => m.list(), + 'Error listing backups', + ); + + if (opts.json) { + yield* Effect.log(JSON.stringify(backups, null, 2)); + return; + } + + if (backups.length === 0) { + yield* Effect.log('No backups yet.'); + return; + } + + for (const backup of backups) { + yield* renderBackup(backup); + } +}); diff --git a/client/packages/cli/src/index.ts b/client/packages/cli/src/index.ts index be6c5a622d..3916ed02fe 100644 --- a/client/packages/cli/src/index.ts +++ b/client/packages/cli/src/index.ts @@ -51,6 +51,8 @@ import { webhooksEventsResendCmd } from './commands/webhooks/events/resend.ts'; import { emailStatusCmd } from './commands/auth/email/status.ts'; import { verifyCmd } from './commands/auth/email/verify.ts'; import { resendEmailCmd } from './commands/auth/email/resend.ts'; +import { backupListCmd } from './commands/backup/list.ts'; +import { backupDownloadCmd } from './commands/backup/download.ts'; export type OptsFromCommand = C extends Command ? R : never; @@ -640,6 +642,64 @@ export const webhooksEventsPayloadDef = webhooksEvents ); }); +const backup = program + .command('backup') + .description('View and download backups of your app'); + +export const backupListDef = backup + .command('list') + .description('List downloadable backups for an app') + .option( + '-a --app ', + 'App ID to list backups for. Defaults to *_INSTANT_APP_ID in .env', + ) + .option('--json', 'Output backups as JSON') + .action((opts) => { + return runCommandEffect( + backupListCmd(opts).pipe( + Effect.provide( + WithAppLayer({ + coerce: false, + coerceAuth: false, + appId: opts.app, + allowAdminToken: true, + }).pipe(Layer.annotateLogs('silent', !!opts.json)), + ), + ), + ); + }); + +export const backupDownloadDef = backup + .command('download') + .description('Download a backup as a zip file') + .argument( + '[backup-id]', + 'Backup ID to download. Defaults to an interactive picker', + ) + .option( + '-a --app ', + 'App ID to download a backup of. Defaults to *_INSTANT_APP_ID in .env', + ) + .option('--latest', 'Download the most recent backup') + .option( + '-o --out ', + 'Output zip path. Defaults to instant-backup-.zip', + ) + .action((backupId, opts) => { + return runCommandEffect( + backupDownloadCmd(backupId, opts).pipe( + Effect.provide( + WithAppLayer({ + coerce: false, + coerceAuth: false, + appId: opts.app, + allowAdminToken: true, + }), + ), + ), + ); + }); + const authEmail = auth .command('email') .description('Manage custom magic code email templates'); diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts new file mode 100644 index 0000000000..8bd6f3e860 --- /dev/null +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -0,0 +1,320 @@ +import { createWriteStream } from 'node:fs'; +import { rename, unlink } from 'node:fs/promises'; +import { once } from 'node:events'; +import { get as httpGet, type IncomingMessage } from 'node:http'; +import { get as httpsGet } from 'node:https'; +import { Readable, Transform, Writable, type Duplex } from 'node:stream'; +import zlib from 'node:zlib'; +import type { + AppBackup, + AppBackupStorageFile, + BackupsManager, +} from '@instantdb/platform'; + +export type BackupDownloadProgress = { + entitiesCompleted: number; + entitiesTotal: number | null; + filesCompleted: number; + filesTotal: number | null; + // Compressed bytes written to disk so far (the zip's on-disk size). + zipBytes: number; + // Uncompressed bytes read from source bodies, and the backup's known + // uncompressed total — the numerator/denominator for a progress bar. + // bytesTotal is null when the backup row carries no sizes. + bytesRead: number; + bytesTotal: number | null; + // The entry currently being fetched; empty between phases. + currentEntry: string; +}; + +export type BackupDownloadResult = { + entities: number; + files: number; + zipBytes: number; +}; + +// zstd landed in node:zlib in 22.15 / 23.8; on older Nodes the property is +// absent, so feature-detect instead of assuming the type declarations match +// the runtime. +const createZstdDecompress: (() => Duplex) | undefined = ( + zlib as { createZstdDecompress?: () => Duplex } +).createZstdDecompress; + +function fetchStream( + url: string, + signal: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const get = url.startsWith('https:') ? httpsGet : httpGet; + const req = get(url, { signal }, resolve); + req.on('error', reject); + }); +} + +// .pipe doesn't forward errors, so a source failure would otherwise leave the +// destination (and the zip writer reading from it) hanging forever. +function pipe(src: Readable, dst: Duplex): Duplex { + src.on('error', (e) => dst.destroy(e)); + return src.pipe(dst); +} + +/** + * Downloads a backup into a zip file at `outPath`, matching the archive the + * dashboard produces: entries are written in the canonical restore order + * (`config.json`, then the `entities/.jsonl` shards, then + * `files/` storage blobs — all entity files before any storage + * file), zip64 so archives past 4GB stay readable, and disk backpressure so + * memory stays flat regardless of backup size. + * + * Writes to `.partial` and renames on success; a failed or aborted + * download removes the partial file. + */ +export async function downloadBackupToFile(opts: { + manager: BackupsManager; + backup: AppBackup; + outPath: string; + signal: AbortSignal; + onProgress: (progress: BackupDownloadProgress) => void; +}): Promise { + const { manager, backup, outPath, onProgress } = opts; + + // The entity shards are served with `Content-Encoding: zstd`; fail before + // writing anything if this Node can't decompress them. + if (!createZstdDecompress) { + throw new Error( + 'Downloading backups requires Node 22.15 or newer (for zstd support).', + ); + } + + // Internal controller so a zip-pipeline failure also tears down the + // storage-files discovery stream and any in-flight body fetches. + const abortController = new AbortController(); + if (opts.signal.aborted) { + abortController.abort(); + } else { + opts.signal.addEventListener('abort', () => abortController.abort(), { + once: true, + }); + } + const signal = abortController.signal; + + let entitiesCompleted = 0; + let entitiesTotal: number | null = null; + let filesCompleted = 0; + let filesTotal: number | null = null; + let zipBytes = 0; + let bytesRead = 0; + let currentEntry = ''; + const bytesTotal = + backup.uncompressedSize != null + ? backup.uncompressedSize + (backup.filesSize ?? 0) + : null; + + const tick = () => + onProgress({ + entitiesCompleted, + entitiesTotal, + filesCompleted, + filesTotal, + zipBytes, + bytesRead, + bytesTotal, + currentEntry, + }); + + // Throttle by time: a large backup pushes many small chunks and ticking on + // every one is wasted work. Phase changes tick() directly so they're still + // immediate. + const TICK_INTERVAL_MS = 100; + let lastTickAt = 0; + const throttledTick = () => { + const now = Date.now(); + if (now - lastTickAt >= TICK_INTERVAL_MS) { + lastTickAt = now; + tick(); + } + }; + + // Wrap a source body for the zip: decompress if the object is stored + // compressed, and count the uncompressed bytes for progress. + const toEntryStream = (res: IncomingMessage): ReadableStream => { + const encoding = res.headers['content-encoding']; + let stream: Readable = res; + if (encoding === 'zstd') { + stream = pipe(stream, createZstdDecompress()); + } else if (encoding === 'gzip') { + stream = pipe(stream, zlib.createGunzip()); + } else if (encoding) { + throw new Error(`Unsupported content encoding: ${encoding}`); + } + const counter = new Transform({ + transform(chunk: Buffer, _enc, cb) { + bytesRead += chunk.length; + throttledTick(); + cb(null, chunk); + }, + }); + return Readable.toWeb(pipe(stream, counter)) as ReadableStream; + }; + + // Storage-files discovery runs concurrently with the entity phase and is + // drained eagerly into a queue. That isn't just overlap: it closes the + // NDJSON connection quickly instead of holding it open (and at the mercy of + // idle timeouts) while multi-GB blobs download. `queueHead` walks the array + // in place, freeing each slot as it's consumed. + const queue: (AppBackupStorageFile | undefined)[] = []; + let queueHead = 0; + let storageDone = false; + let storageError: Error | null = null; + let waitResolve: (() => void) | null = null; + const notify = () => { + const w = waitResolve; + waitResolve = null; + w?.(); + }; + + // Never rejects: failures land in storageError for the drain loop to throw. + const discovery = (async () => { + try { + for await (const file of manager.streamStorageFiles(backup.id, { + signal, + })) { + queue.push(file); + filesTotal = (filesTotal ?? 0) + 1; + throttledTick(); + notify(); + } + } catch (e) { + // The abort path is expected when the zip pipeline failed and we tore + // the discovery down. + if ((e as { name?: string })?.name !== 'AbortError') { + storageError = e as Error; + } + } finally { + if (filesTotal == null) filesTotal = 0; + storageDone = true; + tick(); + notify(); + } + })(); + + // Entry write order is significant for restore: config.json first, then the + // entities/*.jsonl shards, then files/. In particular ALL entity + // files must be written before ANY storage file. listFiles returns the + // entity files in write order; this generator yields them to completion, + // then drains the storage queue. + type ZipEntry = { name: string; input: ReadableStream }; + const entries = (async function* (): AsyncGenerator { + const files = await manager.listFiles(backup.id); + if (files.length === 0) { + throw new Error('No files found for this backup.'); + } + // config.json isn't a namespace — count only the entities/*.jsonl shards. + entitiesTotal = files.filter((f) => f.name !== 'config.json').length; + tick(); + + for (const f of files) { + currentEntry = f.name; + tick(); + const url = await manager.getFileUrl(backup.id, f.name); + const res = await fetchStream(url, signal); + if (res.statusCode !== 200) { + res.resume(); + throw new Error(`Failed to fetch ${f.name}: HTTP ${res.statusCode}.`); + } + if (f.name !== 'config.json') entitiesCompleted++; + tick(); + yield { name: f.name, input: toEntryStream(res) }; + } + currentEntry = ''; + tick(); + + while (true) { + if (storageError) throw storageError; + let file: AppBackupStorageFile | undefined; + if (queueHead < queue.length) { + file = queue[queueHead]; + queue[queueHead] = undefined; + queueHead++; + } + if (file) { + currentEntry = file.path || file.locationId; + tick(); + const res = await fetchStream(file.url, signal); + if (res.statusCode !== 200) { + res.resume(); + throw new Error( + `Couldn't download storage file "${file.path}" (HTTP ${res.statusCode}).`, + ); + } + filesCompleted++; + tick(); + yield { name: `files/${file.locationId}`, input: toEntryStream(res) }; + } else if (storageDone) { + break; + } else { + await new Promise((resolve) => { + waitResolve = resolve; + }); + } + } + currentEntry = ''; + tick(); + + if (storageError) throw storageError; + })(); + + const partialPath = `${outPath}.partial`; + const fileStream = createWriteStream(partialPath); + const diskWriter = ( + Writable.toWeb(fileStream) as WritableStream + ).getWriter(); + const awaitFileClosed = async () => { + if (!fileStream.closed) await once(fileStream, 'close'); + }; + try { + // Loaded on demand so every other CLI command skips parsing it. + const { ZipWriter } = await import('@zip.js/zip.js'); + // Sink that zip.js writes compressed bytes into: it tallies the on-disk + // size for progress, then forwards to the file. Awaiting the disk write + // propagates backpressure up into zip.js, so a fast source can't outrun + // the disk and balloon memory. + const countingSink = new WritableStream({ + async write(chunk) { + zipBytes += chunk.byteLength; + throttledTick(); + await diskWriter.write(chunk); + }, + async close() { + await diskWriter.close(); + tick(); + }, + async abort(reason) { + await diskWriter.abort(reason); + }, + }); + + // zip64: without it any archive whose central-directory offset passes 4GB + // writes a wrapped 32-bit offset and the zip is unreadable. + const zipWriter = new ZipWriter(countingSink, { zip64: true, signal }); + for await (const entry of entries) { + await zipWriter.add(entry.name, entry.input, { + lastModDate: backup.backupAt, + }); + } + await zipWriter.close(); + await awaitFileClosed(); + await rename(partialPath, outPath); + await discovery; + tick(); + return { entities: entitiesCompleted, files: filesCompleted, zipBytes }; + } catch (e) { + // Tear down the discovery stream and any in-flight body fetches so we + // don't keep pulling from S3, and discard the partial file on disk. + abortController.abort(); + await diskWriter.abort(e).catch(() => {}); + await awaitFileClosed().catch(() => {}); + await unlink(partialPath).catch(() => {}); + throw e; + } +} diff --git a/client/packages/cli/src/lib/backups.ts b/client/packages/cli/src/lib/backups.ts new file mode 100644 index 0000000000..9327d5ccb4 --- /dev/null +++ b/client/packages/cli/src/lib/backups.ts @@ -0,0 +1,33 @@ +import { Effect } from 'effect'; +import type { BackupsManager } from '@instantdb/platform'; +import { CurrentApp } from '../context/currentApp.ts'; +import { PlatformApiError } from '../context/platformApi.ts'; +import { getAuthedPlatformApi } from './platformApi.ts'; + +export const useBackupsManager = ( + fun: (manager: BackupsManager) => Promise, + errorMessage?: string, +) => + Effect.gen(function* () { + const api = yield* getAuthedPlatformApi; + const { appId } = yield* CurrentApp; + return yield* Effect.tryPromise({ + try: () => fun(api.backups(appId)), + catch: (e) => + new PlatformApiError({ + message: errorMessage ?? 'Error using backups api', + cause: e, + }), + }); + }); + +/** + * Yields a `BackupsManager` instance scoped to the current app. Use when you + * need to hold on to the manager outside an Effect (e.g. to drive the + * long-running download pipeline). + */ +export const buildBackupsManager = Effect.gen(function* () { + const api = yield* getAuthedPlatformApi; + const { appId } = yield* CurrentApp; + return api.backups(appId); +}); diff --git a/client/packages/cli/src/lib/platformApi.ts b/client/packages/cli/src/lib/platformApi.ts new file mode 100644 index 0000000000..31bc4c8bcb --- /dev/null +++ b/client/packages/cli/src/lib/platformApi.ts @@ -0,0 +1,11 @@ +import { Effect } from 'effect'; +import { PlatformApi as InstantPlatformApi } from '@instantdb/platform'; +import { AuthToken } from '../context/authToken.ts'; +import { getBaseUrl } from './config.ts'; + +export const getAuthedPlatformApi = Effect.gen(function* () { + const apiURI = yield* getBaseUrl; + const authToken = yield* AuthToken; + const token = yield* authToken.getAuthToken; + return new InstantPlatformApi({ apiURI, auth: { token } }); +}); diff --git a/client/packages/cli/src/lib/webhooks.ts b/client/packages/cli/src/lib/webhooks.ts index cc89da8ed5..044a4f34ce 100644 --- a/client/packages/cli/src/lib/webhooks.ts +++ b/client/packages/cli/src/lib/webhooks.ts @@ -1,22 +1,13 @@ import { Effect } from 'effect'; import { - PlatformApi as InstantPlatformApi, type WebhookAction, type WebhookEventInfo, type WebhooksManager, } from '@instantdb/platform'; -import { AuthToken } from '../context/authToken.ts'; import { CurrentApp } from '../context/currentApp.ts'; import { PlatformApiError } from '../context/platformApi.ts'; import { BadArgsError } from '../errors.ts'; -import { getBaseUrl } from './http.ts'; - -const getAuthedPlatformApi = Effect.gen(function* () { - const apiURI = yield* getBaseUrl; - const authToken = yield* AuthToken; - const token = yield* authToken.getAuthToken; - return new InstantPlatformApi({ apiURI, auth: { token } }); -}); +import { getAuthedPlatformApi } from './platformApi.ts'; export const WEBHOOK_ACTIONS: readonly WebhookAction[] = [ 'create', diff --git a/client/packages/platform/__tests__/src/backups.test.ts b/client/packages/platform/__tests__/src/backups.test.ts new file mode 100644 index 0000000000..cd719686db --- /dev/null +++ b/client/packages/platform/__tests__/src/backups.test.ts @@ -0,0 +1,139 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { + BackupsManager, + backupZipName, + type AppBackupStorageFile, +} from '../../src/backups.ts'; + +const makeManager = () => + new BackupsManager({ + appId: 'app-1', + apiURI: 'http://api.test', + withAuth: (operation) => operation('test-token'), + }); + +const streamOf = (chunks: string[]) => + new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + for (const chunk of chunks) { + controller.enqueue(encoder.encode(chunk)); + } + controller.close(); + }, + }); + +const stubFetchBody = (chunks: string[]) => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(streamOf(chunks), { status: 200 })), + ); +}; + +const collect = async (manager: BackupsManager) => { + const files: AppBackupStorageFile[] = []; + for await (const file of manager.streamStorageFiles('backup-1')) { + files.push(file); + } + return files; +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('streamStorageFiles', () => { + test('yields files and completes on the done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"locationId":"loc-2","path":"b.png","url":"http://s3/loc-2"}\n', + '{"done":true}\n', + ]); + + const files = await collect(makeManager()); + expect(files).toEqual([ + { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + { locationId: 'loc-2', path: 'b.png', url: 'http://s3/loc-2' }, + ]); + }); + + test('handles lines split across chunks', async () => { + stubFetchBody([ + '{"locationId":"loc-1","pa', + 'th":"a.png","url":"http://s3/loc-1"}\n{"done"', + ':true}\n', + ]); + + const files = await collect(makeManager()); + expect(files).toEqual([ + { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + ]); + }); + + test('skips lines without a locationId or url', async () => { + stubFetchBody([ + '{"path":"orphan.png"}\n', + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":true}\n', + ]); + + const files = await collect(makeManager()); + expect(files).toEqual([ + { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + ]); + }); + + test('completes without files when the app has none', async () => { + stubFetchBody(['{"done":true}\n']); + expect(await collect(makeManager())).toEqual([]); + }); + + test('throws when the stream ends without the done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow( + /ended before it finished/, + ); + }); +}); + +describe('list', () => { + test('coerces the server row into an AppBackup', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => + Response.json({ + backups: [ + { + id: 'backup-1', + isn: '42', + backup_at: '2026-08-05T07:12:00Z', + files_size: 100, + db_size: 200, + uncompressed_size: 300, + description: 'Automated Daily Snapshot', + expires_at: '2026-08-12T07:12:00Z', + }, + ], + }), + ), + ); + + const [backup] = await makeManager().list(); + expect(backup).toEqual({ + id: 'backup-1', + isn: '42', + backupAt: new Date('2026-08-05T07:12:00Z'), + filesSize: 100, + dbSize: 200, + uncompressedSize: 300, + description: 'Automated Daily Snapshot', + expiresAt: new Date('2026-08-12T07:12:00Z'), + }); + expect(backupZipName(backup)).toBe( + 'instant-backup-2026-08-05T07-12-00-000Z.zip', + ); + }); +}); diff --git a/client/packages/platform/src/api.ts b/client/packages/platform/src/api.ts index 9649802120..bc08e4be27 100644 --- a/client/packages/platform/src/api.ts +++ b/client/packages/platform/src/api.ts @@ -15,6 +15,7 @@ import { DataAttrDef, } from '@instantdb/core'; import { Webhooks, type WithAuth } from '@instantdb/webhooks'; +import { BackupsManager } from './backups.ts'; import version from './version.ts'; import { attrFwdLabel, @@ -1878,4 +1879,18 @@ export class PlatformApi { withAuth, }); } + + /** + * Returns a {@link BackupsManager} scoped to `appId` for listing an app's + * backups and downloading their contents. Calls are routed through + * {@link withRetry}, so an expired access token is transparently refreshed. + */ + backups(appId: string): BackupsManager { + const withAuth: WithAuth = (operation) => + this.withRetry( + (_apiURI: string, token: string) => operation(token), + [this.#apiURI, this.token()], + ); + return new BackupsManager({ appId, apiURI: this.#apiURI, withAuth }); + } } diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts new file mode 100644 index 0000000000..5658195d2a --- /dev/null +++ b/client/packages/platform/src/backups.ts @@ -0,0 +1,248 @@ +import { InstantAPIError, version as coreVersion } from '@instantdb/core'; +import type { WithAuth } from '@instantdb/webhooks'; +import version from './version.ts'; + +/** A point-in-time snapshot of an app. */ +export type AppBackup = { + /** Unique identifier for the backup. */ + id: string; + /** Instant sequence number the snapshot was taken at. */ + isn: string; + /** When the snapshot was taken. */ + backupAt: Date; + /** Total size in bytes of the app's storage files at backup time, if known. */ + filesSize: number | null; + /** Size in bytes of the app's database at backup time, if known. */ + dbSize: number | null; + /** + * Total uncompressed size in bytes of the backup's entity files (what they + * take up unpacked on disk), if known. + */ + uncompressedSize: number | null; + /** Human-readable label, e.g. "Automated Daily Snapshot". */ + description: string | null; + /** When the backup stops being available for download. */ + expiresAt: Date | null; +}; + +/** + * A file that makes up the backup payload: `config.json` or an + * `entities/.jsonl` shard. + */ +export type AppBackupFile = { + name: string; + /** Size in bytes as stored (the entity shards are stored compressed). */ + size: number; +}; + +/** A storage file captured in a backup, with a presigned download URL. */ +export type AppBackupStorageFile = { + /** + * Stable id of the file's blob. A backup archive stores the blob at + * `files/`. + */ + locationId: string; + /** The path the user uploaded the file to. */ + path: string | null; + /** Presigned URL for the file's contents. Expires after 12 hours. */ + url: string; +}; + +type AppBackupResponse = { + id: string; + isn: string; + backup_at: string; + files_size: number | null; + db_size: number | null; + uncompressed_size: number | null; + description: string | null; + expires_at: string | null; +}; + +function toAppBackup(row: AppBackupResponse): AppBackup { + return { + id: row.id, + isn: row.isn, + backupAt: new Date(row.backup_at), + filesSize: row.files_size, + dbSize: row.db_size, + uncompressedSize: row.uncompressed_size, + description: row.description, + expiresAt: row.expires_at ? new Date(row.expires_at) : null, + }; +} + +/** Suggested filename for a backup archive. */ +export function backupZipName(backup: AppBackup): string { + const safe = backup.backupAt.toISOString().replace(/[:.]/g, '-'); + return `instant-backup-${safe}.zip`; +} + +function authHeaders(token: string): Record { + return { + authorization: `Bearer ${token}`, + 'Instant-Platform-Version': version, + 'Instant-Core-Version': coreVersion, + 'X-Instant-Source': 'platform-sdk', + 'X-Instant-Version': version, + }; +} + +async function apiError(res: Response): Promise { + const body = await res.text(); + try { + return new InstantAPIError({ status: res.status, body: JSON.parse(body) }); + } catch (_e) { + return new InstantAPIError({ + status: res.status, + body: { type: undefined, message: body }, + }); + } +} + +async function* ndjsonLines( + body: ReadableStream, +): AsyncGenerator { + const reader = body.getReader(); + try { + const decoder = new TextDecoder(); + let buf = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + let nl = buf.indexOf('\n'); + while (nl !== -1) { + const line = buf.slice(0, nl).trim(); + buf = buf.slice(nl + 1); + if (line.length > 0) { + yield JSON.parse(line); + } + nl = buf.indexOf('\n'); + } + } + const line = buf.trim(); + if (line.length > 0) { + yield JSON.parse(line); + } + } finally { + // Also releases the connection when the consumer stops iterating early. + await reader.cancel().catch(() => {}); + } +} + +/** + * Read-only API for an app's backups. + * + * A backup archive has a canonical entry order that restore relies on: + * `config.json` first, then every `entities/.jsonl` shard, then the + * `files/` storage blobs. In particular ALL entity files must + * come before ANY storage file, so a restore can process the archive in a + * single streaming pass: `config.json` sets up the schema, the `$files` + * entities register file metadata, and only then can each blob be matched + * to its entity. {@link listFiles} returns the entity files already in + * write order; write the {@link streamStorageFiles} blobs after them. + */ +export class BackupsManager { + #appId: string; + #apiURI: string; + #withAuth: WithAuth; + + constructor(opts: { appId: string; apiURI: string; withAuth: WithAuth }) { + this.#appId = opts.appId; + this.#apiURI = opts.apiURI; + this.#withAuth = opts.withAuth; + } + + #getJson(path: string): Promise { + return this.#withAuth(async (token) => { + const res = await fetch(`${this.#apiURI}${path}`, { + headers: authHeaders(token), + }); + if (res.status !== 200) { + throw await apiError(res); + } + return res.json(); + }); + } + + /** + * Returns the app's downloadable (non-expired) backups, newest first. + */ + async list(): Promise { + const res = await this.#getJson(`/dash/apps/${this.#appId}/backups`); + return ((res.backups || []) as AppBackupResponse[]).map(toAppBackup); + } + + /** + * Returns the backup's entity files (`config.json` and the + * `entities/.jsonl` shards) in archive write order. Storage blobs + * are not included; discover those with {@link streamStorageFiles}. + */ + async listFiles(backupId: string): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups/${backupId}/files`, + ); + return (res.files || []) as AppBackupFile[]; + } + + /** + * Returns a presigned download URL for one of the backup's entity files. + * The URL expires after 1 hour, so fetch it right before downloading. + * + * The entity files are stored zstd-compressed (the response carries + * `Content-Encoding: zstd`); decompress to get the raw JSON/JSONL. + */ + async getFileUrl(backupId: string, name: string): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups/${backupId}/file-url?name=${encodeURIComponent(name)}`, + ); + return res.url as string; + } + + /** + * Streams every storage file captured in the backup, each with a presigned + * download URL. Completes without yielding anything when the app has no + * storage files. + * + * The server ends a healthy stream with a terminal sentinel; if the stream + * closes without it (a server-side failure truncated the listing), this + * throws instead of silently under-reporting files. + */ + async *streamStorageFiles( + backupId: string, + opts?: { signal?: AbortSignal }, + ): AsyncGenerator { + const res = await this.#withAuth(async (token) => { + const res = await fetch( + `${this.#apiURI}/dash/apps/${this.#appId}/backups/${backupId}/storage-files`, + { headers: authHeaders(token), signal: opts?.signal }, + ); + if (res.status !== 200) { + throw await apiError(res); + } + return res; + }); + if (!res.body) { + throw new Error('Storage file listing returned no body.'); + } + let complete = false; + for await (const line of ndjsonLines(res.body)) { + if (line.done) { + complete = true; + break; + } + if (!line.locationId || !line.url) continue; + yield { + locationId: line.locationId, + path: line.path ?? null, + url: line.url, + }; + } + if (!complete) { + throw new Error( + 'Storage file listing ended before it finished. Please retry the download.', + ); + } + } +} diff --git a/client/packages/platform/src/index.ts b/client/packages/platform/src/index.ts index 3181916c50..acced89e02 100644 --- a/client/packages/platform/src/index.ts +++ b/client/packages/platform/src/index.ts @@ -115,6 +115,14 @@ export { type Identifier, } from './migrations.ts'; +export { + BackupsManager, + backupZipName, + type AppBackup, + type AppBackupFile, + type AppBackupStorageFile, +} from './backups.ts'; + export { DEFAULT_OAUTH_CALLBACK_URL, oauthCallbackURL, diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index 6eec6aedc3..05b726b299 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@instantdb/version': specifier: workspace:* version: link:../version + '@zip.js/zip.js': + specifier: ^2.8.34 + version: 2.8.34 acorn: specifier: ^8.15.0 version: 8.15.0 @@ -8960,7 +8963,7 @@ packages: '@xmldom/xmldom@0.7.13': resolution: {integrity: sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g==} engines: {node: '>=10.0.0'} - deprecated: this version has critical issues, please update to the latest version + deprecated: this version is no longer supported, please update to at least 0.8.* '@xmldom/xmldom@0.8.10': resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} @@ -25743,7 +25746,7 @@ snapshots: metro: 0.83.6 metro-config: 0.83.6 metro-core: 0.83.6 - semver: 7.7.4 + semver: 7.8.5 optionalDependencies: '@react-native-community/cli': 15.1.3(typescript@5.9.3) transitivePeerDependencies: @@ -36166,7 +36169,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36213,7 +36216,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36261,7 +36264,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36954,8 +36957,7 @@ snapshots: semver@7.7.4: {} - semver@7.8.5: - optional: true + semver@7.8.5: {} send@0.18.0: dependencies: @@ -38993,7 +38995,7 @@ snapshots: esbuild: 0.27.3 fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 - postcss: 8.5.23 + postcss: 8.5.8 rollup: 4.59.0 tinyglobby: 0.2.15 optionalDependencies: diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index 968e9c9656..5bbdf02f37 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -176,7 +176,7 @@ async function downloadBackup( // files — the user's $files (storage uploads). // entitiesTotal is null until fetchFiles resolves; filesTotal is null // until the discovery NDJSON returns at least one line OR completes - // (404/empty body sets it to 0 so the dialog shows "0 of 0 files"). + // (an empty stream sets it to 0 so the dialog shows "0 of 0 files"). let entitiesCompleted = 0; let entitiesTotal: number | null = null; let filesCompleted = 0; @@ -261,21 +261,18 @@ async function downloadBackup( // It pushes onto storagePending and bumps `storageDiscovered` as URLs // stream in. void (async () => { - // Set once we see the server's terminal `done` sentinel (or a 404, which - // means there's no $files shard at all). If discovery ends without it and - // we weren't aborted, the stream was truncated by a server-side failure. + // Set once we see the server's terminal `done` sentinel. A backup with no + // storage files still gets the sentinel (the server sends it even when + // there's no $files shard), so if discovery ends without it and we weren't + // aborted, the stream was truncated by a server-side failure. A 404 here + // can only mean the backup record itself is missing or expired, so it goes + // through the error path instead of masquerading as "no files". let storageComplete = false; try { const res = await fetch( `${config.apiURI}/dash/apps/${appId}/backups/${backup.id}/storage-files`, { headers: { authorization: `Bearer ${token}` }, signal }, ); - if (res.status === 404) { - filesTotal = 0; - storageComplete = true; - tick(); - return; - } if (!res.ok || !res.body) { throw new Error(`Failed to list storage files: ${res.status}`); } From 6b244a62c96e10558e8cbb399a7ddb73b9322758 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 12:41:20 -0700 Subject: [PATCH 02/18] Trim comment --- client/www/components/dash/BackupDownloadDialog.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index 5bbdf02f37..35f87153cd 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -264,9 +264,7 @@ async function downloadBackup( // Set once we see the server's terminal `done` sentinel. A backup with no // storage files still gets the sentinel (the server sends it even when // there's no $files shard), so if discovery ends without it and we weren't - // aborted, the stream was truncated by a server-side failure. A 404 here - // can only mean the backup record itself is missing or expired, so it goes - // through the error path instead of masquerading as "no files". + // aborted, the stream was truncated by a server-side failure. let storageComplete = false; try { const res = await fetch( From 1f72cc03039d2a4c69ba5f3c8d4935efb873aa47 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 12:46:12 -0700 Subject: [PATCH 03/18] Use BackupsManager in the dashboard download dialog Swaps the dialog's hand-rolled fetch helpers and NDJSON parsing for the shared BackupsManager, so the dashboard and CLI consume the same protocol layer. Adds optional abort signals to the manager's JSON methods so the dialog keeps aborting in-flight API calls on cancel (and the CLI now passes its signal too). --- client/packages/cli/src/lib/backupDownload.ts | 4 +- client/packages/platform/src/backups.ts | 23 +++- .../components/dash/BackupDownloadDialog.tsx | 125 ++++-------------- 3 files changed, 49 insertions(+), 103 deletions(-) diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts index 8bd6f3e860..d698166d54 100644 --- a/client/packages/cli/src/lib/backupDownload.ts +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -205,7 +205,7 @@ export async function downloadBackupToFile(opts: { // then drains the storage queue. type ZipEntry = { name: string; input: ReadableStream }; const entries = (async function* (): AsyncGenerator { - const files = await manager.listFiles(backup.id); + const files = await manager.listFiles(backup.id, { signal }); if (files.length === 0) { throw new Error('No files found for this backup.'); } @@ -216,7 +216,7 @@ export async function downloadBackupToFile(opts: { for (const f of files) { currentEntry = f.name; tick(); - const url = await manager.getFileUrl(backup.id, f.name); + const url = await manager.getFileUrl(backup.id, f.name, { signal }); const res = await fetchStream(url, signal); if (res.statusCode !== 200) { res.resume(); diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts index 5658195d2a..f9f92cae5e 100644 --- a/client/packages/platform/src/backups.ts +++ b/client/packages/platform/src/backups.ts @@ -154,10 +154,11 @@ export class BackupsManager { this.#withAuth = opts.withAuth; } - #getJson(path: string): Promise { + #getJson(path: string, signal?: AbortSignal): Promise { return this.#withAuth(async (token) => { const res = await fetch(`${this.#apiURI}${path}`, { headers: authHeaders(token), + signal, }); if (res.status !== 200) { throw await apiError(res); @@ -169,8 +170,11 @@ export class BackupsManager { /** * Returns the app's downloadable (non-expired) backups, newest first. */ - async list(): Promise { - const res = await this.#getJson(`/dash/apps/${this.#appId}/backups`); + async list(opts?: { signal?: AbortSignal }): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups`, + opts?.signal, + ); return ((res.backups || []) as AppBackupResponse[]).map(toAppBackup); } @@ -179,9 +183,13 @@ export class BackupsManager { * `entities/.jsonl` shards) in archive write order. Storage blobs * are not included; discover those with {@link streamStorageFiles}. */ - async listFiles(backupId: string): Promise { + async listFiles( + backupId: string, + opts?: { signal?: AbortSignal }, + ): Promise { const res = await this.#getJson( `/dash/apps/${this.#appId}/backups/${backupId}/files`, + opts?.signal, ); return (res.files || []) as AppBackupFile[]; } @@ -193,9 +201,14 @@ export class BackupsManager { * The entity files are stored zstd-compressed (the response carries * `Content-Encoding: zstd`); decompress to get the raw JSON/JSONL. */ - async getFileUrl(backupId: string, name: string): Promise { + async getFileUrl( + backupId: string, + name: string, + opts?: { signal?: AbortSignal }, + ): Promise { const res = await this.#getJson( `/dash/apps/${this.#appId}/backups/${backupId}/file-url?name=${encodeURIComponent(name)}`, + opts?.signal, ); return res.url as string; } diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index 35f87153cd..c53a540f75 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -10,8 +10,13 @@ import { } from 'react'; import { ArrowsPointingOutIcon, XMarkIcon } from '@heroicons/react/24/outline'; +import { + BackupsManager, + type AppBackupFile, + type AppBackupStorageFile, +} from '@instantdb/platform'; + import config from '@/lib/config'; -import { jsonFetch } from '@/lib/fetch'; import { messageFromInstantError } from '@/lib/errors'; import { InstantApp, InstantAppBackup, InstantIssue } from '@/lib/types'; @@ -20,8 +25,6 @@ import { Button, Content, Dialog, SubsectionHeading } from '@/components/ui'; import { formatTimestamp } from '@/components/dash/shared'; import { useDarkMode } from '@/components/dash/DarkModeToggle'; -type BackupFile = { name: string; size: number }; - type DownloadProgress = { entitiesCompleted: number; entitiesTotal: number | null; @@ -61,42 +64,6 @@ function formatBytes(n: number): string { return `${v.toFixed(digits)} ${units[i]}`; } -async function fetchFiles( - token: string, - appId: string, - backupId: string, - signal: AbortSignal, -): Promise { - // XXX: Why are you using the authed fetch hook here? - const { files } = (await jsonFetch( - `${config.apiURI}/dash/apps/${appId}/backups/${backupId}/files`, - { headers: { authorization: `Bearer ${token}` }, signal }, - )) as { files: BackupFile[] }; - return files; -} - -async function fetchFileUrl( - token: string, - appId: string, - backupId: string, - name: string, - signal: AbortSignal, -): Promise { - // XXX: Why are you using the authed fetch hook here? - const url = `${config.apiURI}/dash/apps/${appId}/backups/${backupId}/file-url?name=${encodeURIComponent(name)}`; - const { url: signed } = (await jsonFetch(url, { - headers: { authorization: `Bearer ${token}` }, - signal, - })) as { url: string }; - return signed; -} - -type StorageFileLine = { - locationId: string; - path: string; - url: string; -}; - type DownloadResult = | { via: 'picker'; filename: string } | { via: 'browser-default'; filename: string }; @@ -162,6 +129,13 @@ async function downloadBackup( ], }); + // The dash token can't refresh mid-download, so withAuth just supplies it. + const manager = new BackupsManager({ + appId, + apiURI: config.apiURI, + withAuth: (operation) => operation(token), + }); + // ---- Shared progress state ---- // current = files (backup + storage) fully fetched. // total = backup file count + storage files ever discovered. @@ -174,7 +148,7 @@ async function downloadBackup( // Two parallel counters surfaced in the dialog: // entities — the backup payload (config.json + per-etype JSONL shards). // files — the user's $files (storage uploads). - // entitiesTotal is null until fetchFiles resolves; filesTotal is null + // entitiesTotal is null until listFiles resolves; filesTotal is null // until the discovery NDJSON returns at least one line OR completes // (an empty stream sets it to 0 so the dialog shows "0 of 0 files"). let entitiesCompleted = 0; @@ -193,7 +167,7 @@ async function downloadBackup( backup.uncompressed_size != null ? backup.uncompressed_size + (backup.files_size ?? 0) : null; - const storageQueue: Record = {}; + const storageQueue: Record = {}; let queueHead = 0; let queueTail = 0; let storageDone = false; @@ -258,53 +232,20 @@ async function downloadBackup( }; // Kick off storage-files discovery in parallel with the backup file list. - // It pushes onto storagePending and bumps `storageDiscovered` as URLs - // stream in. + // It fills `storageQueue` as URLs stream in. The manager consumes the + // server's terminal `done` sentinel and throws if the stream ends without + // it (a server-side failure truncated the listing), so a partial zip can't + // pass as complete. void (async () => { - // Set once we see the server's terminal `done` sentinel. A backup with no - // storage files still gets the sentinel (the server sends it even when - // there's no $files shard), so if discovery ends without it and we weren't - // aborted, the stream was truncated by a server-side failure. - let storageComplete = false; try { - const res = await fetch( - `${config.apiURI}/dash/apps/${appId}/backups/${backup.id}/storage-files`, - { headers: { authorization: `Bearer ${token}` }, signal }, - ); - if (!res.ok || !res.body) { - throw new Error(`Failed to list storage files: ${res.status}`); - } - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - const consume = (line: string) => { - const trimmed = line.trim(); - if (trimmed.length === 0) return; - const obj = JSON.parse(trimmed) as StorageFileLine & { - done?: boolean; - }; - if (obj.done) { - storageComplete = true; - return; - } - if (!obj.locationId || !obj.url) return; - storageQueue[queueTail++] = obj; + for await (const file of manager.streamStorageFiles(backup.id, { + signal, + })) { + storageQueue[queueTail++] = file; filesTotal = (filesTotal ?? 0) + 1; throttledTick(); notify(); - }; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let nl = buf.indexOf('\n'); - while (nl !== -1) { - consume(buf.slice(0, nl)); - buf = buf.slice(nl + 1); - nl = buf.indexOf('\n'); - } } - consume(buf); } catch (e) { // The abort path is expected when the zip pipeline failed and we // tore the discovery down @@ -312,26 +253,18 @@ async function downloadBackup( storageError = e as Error; } } finally { - // Discovery finished without ever calling consume (empty stream) — + // Discovery finished without yielding anything (no storage files) — // surface "0 of 0" so the dialog doesn't sit on "?" forever. if (filesTotal == null) filesTotal = 0; - // No `done` sentinel and we weren't aborted → the server closed the - // pipe early. Fail loudly instead of building a partial zip. Don't - // clobber a more specific error already caught above. - if (!storageComplete && storageError == null && !signal.aborted) { - storageError = new Error( - 'Storage file listing ended before it finished. Please retry the download.', - ); - } storageDone = true; tick(); notify(); } })(); - let files: BackupFile[]; + let files: AppBackupFile[]; try { - files = await fetchFiles(token, appId, backup.id, signal); + files = await manager.listFiles(backup.id, { signal }); if (files.length === 0) { throw new Error('No files found for this backup.'); } @@ -351,12 +284,12 @@ async function downloadBackup( // entities/*.jsonl shards, then files/${locationId}. In particular ALL entity // files must be written before ANY storage file. This generator preserves // that: it yields the entity `files` (config + entities/*.jsonl, in the order - // fetchFiles returns them) to completion, then drains the storage queue. + // listFiles returns them) to completion, then drains the storage queue. const entries = (async function* (): AsyncGenerator { for (const f of files) { currentEntity = f.name; tick(); - const url = await fetchFileUrl(token, appId, backup.id, f.name, signal); + const url = await manager.getFileUrl(backup.id, f.name, { signal }); const res = await fetch(url, { signal }); if (!res.ok) { throw new Error(`Failed to fetch ${f.name}: ${res.status}`); @@ -376,7 +309,7 @@ async function downloadBackup( while (true) { if (storageError) throw storageError; - let obj: StorageFileLine | undefined; + let obj: AppBackupStorageFile | undefined; if (queueHead < queueTail) { obj = storageQueue[queueHead]; delete storageQueue[queueHead]; From 5a51250bbc13542442ca4e178de74b748a200e1f Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 13:05:50 -0700 Subject: [PATCH 04/18] Address review: strict storage-file records, honest totals on failure The storage-files stream now requires an exact done sentinel and throws on records missing locationId/url instead of silently skipping them (a skipped record would mean a silently incomplete archive). A failed listing keeps the storage total unknown instead of reporting an empty-but-complete phase, and the CLI destroys the response before throwing on an unsupported content encoding. --- client/packages/cli/src/lib/backupDownload.ts | 7 ++++++- .../platform/__tests__/src/backups.test.ts | 14 ++++++++++---- client/packages/platform/src/backups.ts | 16 ++++++++++++++-- .../www/components/dash/BackupDownloadDialog.tsx | 8 ++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts index d698166d54..fb44c3b890 100644 --- a/client/packages/cli/src/lib/backupDownload.ts +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -145,6 +145,7 @@ export async function downloadBackupToFile(opts: { } else if (encoding === 'gzip') { stream = pipe(stream, zlib.createGunzip()); } else if (encoding) { + res.destroy(); throw new Error(`Unsupported content encoding: ${encoding}`); } const counter = new Transform({ @@ -175,6 +176,7 @@ export async function downloadBackupToFile(opts: { // Never rejects: failures land in storageError for the drain loop to throw. const discovery = (async () => { + let discoveryComplete = false; try { for await (const file of manager.streamStorageFiles(backup.id, { signal, @@ -184,6 +186,7 @@ export async function downloadBackupToFile(opts: { throttledTick(); notify(); } + discoveryComplete = true; } catch (e) { // The abort path is expected when the zip pipeline failed and we tore // the discovery down. @@ -191,7 +194,9 @@ export async function downloadBackupToFile(opts: { storageError = e as Error; } } finally { - if (filesTotal == null) filesTotal = 0; + // A failed listing keeps the total unknown rather than reading as an + // empty-but-complete storage phase. + if (discoveryComplete && filesTotal == null) filesTotal = 0; storageDone = true; tick(); notify(); diff --git a/client/packages/platform/__tests__/src/backups.test.ts b/client/packages/platform/__tests__/src/backups.test.ts index cd719686db..d6878caabf 100644 --- a/client/packages/platform/__tests__/src/backups.test.ts +++ b/client/packages/platform/__tests__/src/backups.test.ts @@ -70,17 +70,23 @@ describe('streamStorageFiles', () => { ]); }); - test('skips lines without a locationId or url', async () => { + test('throws on a record without a locationId or url', async () => { stubFetchBody([ '{"path":"orphan.png"}\n', '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', '{"done":true}\n', ]); - const files = await collect(makeManager()); - expect(files).toEqual([ - { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); + }); + + test('only accepts an exact done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":1}\n', ]); + + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); }); test('completes without files when the app has none', async () => { diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts index f9f92cae5e..c0af89a408 100644 --- a/client/packages/platform/src/backups.ts +++ b/client/packages/platform/src/backups.ts @@ -241,11 +241,23 @@ export class BackupsManager { } let complete = false; for await (const line of ndjsonLines(res.body)) { - if (line.done) { + if (line.done === true) { complete = true; break; } - if (!line.locationId || !line.url) continue; + // A file record always carries a locationId and url; anything else is + // corruption, and skipping it would silently omit a file from the + // archive. + if ( + typeof line.locationId !== 'string' || + line.locationId.length === 0 || + typeof line.url !== 'string' || + line.url.length === 0 + ) { + throw new Error( + 'Storage file listing returned a malformed record. Please retry the download.', + ); + } yield { locationId: line.locationId, path: line.path ?? null, diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index c53a540f75..810a5e1e7d 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -237,6 +237,7 @@ async function downloadBackup( // it (a server-side failure truncated the listing), so a partial zip can't // pass as complete. void (async () => { + let discoveryComplete = false; try { for await (const file of manager.streamStorageFiles(backup.id, { signal, @@ -246,6 +247,7 @@ async function downloadBackup( throttledTick(); notify(); } + discoveryComplete = true; } catch (e) { // The abort path is expected when the zip pipeline failed and we // tore the discovery down @@ -254,8 +256,10 @@ async function downloadBackup( } } finally { // Discovery finished without yielding anything (no storage files) — - // surface "0 of 0" so the dialog doesn't sit on "?" forever. - if (filesTotal == null) filesTotal = 0; + // surface "0 of 0" so the dialog doesn't sit on "?" forever. A failed + // listing keeps the total unknown so the error UI doesn't claim an + // empty-but-complete storage phase. + if (discoveryComplete && filesTotal == null) filesTotal = 0; storageDone = true; tick(); notify(); From fa9cf3fff2c3ae27c1f5364052066b95c94326f3 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 13:54:38 -0700 Subject: [PATCH 05/18] Share the download pipeline between the dashboard and CLI --- .../cli/src/commands/backup/download.ts | 17 +- client/packages/cli/src/lib/backupDownload.ts | 325 ++++------------ .../packages/platform/src/backupDownload.ts | 330 ++++++++++++++++ client/packages/platform/src/backups.ts | 45 ++- client/packages/platform/src/index.ts | 10 + .../components/dash/BackupDownloadDialog.tsx | 364 +++--------------- 6 files changed, 511 insertions(+), 580 deletions(-) create mode 100644 client/packages/platform/src/backupDownload.ts diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 0d272767c6..783315d617 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -6,6 +6,7 @@ import { Effect } from 'effect'; import throttle from 'lodash.throttle'; import { backupZipName, + estimateZipSize, type AppBackup, type BackupsManager, } from '@instantdb/platform'; @@ -96,8 +97,9 @@ function makeProgressRenderer() { bytes += ` (${pct}%)`; } parts.push(bytes); - if (p.currentEntry) { - parts.push(p.currentEntry); + const currentEntry = p.currentEntity || p.currentFile; + if (currentEntry) { + parts.push(currentEntry); } let line = parts.join(' · '); const width = stream.columns || 80; @@ -165,14 +167,9 @@ export const backupDownloadCmd = Effect.fn(function* ( const backup = yield* pickBackup(backups, backupId, opts); - // Upper bound: everything stored uncompressed. Lower bound: everything - // compressed at a ~4x DEFLATE ratio, best case for text/JSON; storage files - // vary wildly, so the actual zip lands somewhere inside the range. - const backupBytes = backup.uncompressedSize ?? backup.dbSize; - const hasSizes = backupBytes != null && backup.filesSize != null; - const totalBytes = (backupBytes ?? 0) + (backup.filesSize ?? 0); - const estimate = hasSizes - ? ` The zip file will be between ${formatBytes(Math.round(totalBytes / 4))} and ${formatBytes(totalBytes)}, depending on the compression ratio.` + const sizes = estimateZipSize(backup); + const estimate = sizes + ? ` The zip file will be between ${formatBytes(sizes.min)} and ${formatBytes(sizes.max)}, depending on the compression ratio.` : ''; const ok = yield* promptOk({ diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts index fb44c3b890..286a1e3626 100644 --- a/client/packages/cli/src/lib/backupDownload.ts +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -3,35 +3,21 @@ import { rename, unlink } from 'node:fs/promises'; import { once } from 'node:events'; import { get as httpGet, type IncomingMessage } from 'node:http'; import { get as httpsGet } from 'node:https'; -import { Readable, Transform, Writable, type Duplex } from 'node:stream'; +import { Readable, Writable, type Duplex } from 'node:stream'; import zlib from 'node:zlib'; -import type { - AppBackup, - AppBackupStorageFile, - BackupsManager, +import { + downloadBackupArchive, + type AppBackup, + type BackupArchiveWriter, + type BackupDownloadProgress, + type BackupDownloadResult, + type BackupsManager, } from '@instantdb/platform'; -export type BackupDownloadProgress = { - entitiesCompleted: number; - entitiesTotal: number | null; - filesCompleted: number; - filesTotal: number | null; - // Compressed bytes written to disk so far (the zip's on-disk size). - zipBytes: number; - // Uncompressed bytes read from source bodies, and the backup's known - // uncompressed total — the numerator/denominator for a progress bar. - // bytesTotal is null when the backup row carries no sizes. - bytesRead: number; - bytesTotal: number | null; - // The entry currently being fetched; empty between phases. - currentEntry: string; -}; - -export type BackupDownloadResult = { - entities: number; - files: number; - zipBytes: number; -}; +export type { + BackupDownloadProgress, + BackupDownloadResult, +} from '@instantdb/platform'; // zstd landed in node:zlib in 22.15 / 23.8; on older Nodes the property is // absent, so feature-detect instead of assuming the type declarations match @@ -58,13 +44,49 @@ function pipe(src: Readable, dst: Duplex): Duplex { return src.pipe(dst); } +// Fetches a presigned URL with node:http(s), decompressing explicitly: the +// entity shards are served with `Content-Encoding: zstd` and Node doesn't +// auto-decompress that. downloadBackupToFile refuses to run without zstd +// support, so the assertion below can't fire. +async function fetchBody( + url: string, + signal: AbortSignal, +): Promise> { + const res = await fetchStream(url, signal); + if (res.statusCode !== 200) { + res.resume(); + throw new Error(`HTTP ${res.statusCode}`); + } + const encoding = res.headers['content-encoding']; + let stream: Readable = res; + if (encoding === 'zstd') { + stream = pipe(stream, createZstdDecompress!()); + } else if (encoding === 'gzip') { + stream = pipe(stream, zlib.createGunzip()); + } else if (encoding) { + res.destroy(); + throw new Error(`Unsupported content encoding: ${encoding}`); + } + return Readable.toWeb(stream) as ReadableStream; +} + +async function createZipWriter( + sink: WritableStream, + signal: AbortSignal, +): Promise { + // Loaded on demand so every other CLI command skips parsing it. + const { ZipWriter } = await import('@zip.js/zip.js'); + // zip64: without it any archive whose central-directory offset passes 4GB + // writes a wrapped 32-bit offset and the zip is unreadable. + return new ZipWriter(sink, { zip64: true, signal }); +} + /** - * Downloads a backup into a zip file at `outPath`, matching the archive the - * dashboard produces: entries are written in the canonical restore order - * (`config.json`, then the `entities/.jsonl` shards, then - * `files/` storage blobs — all entity files before any storage - * file), zip64 so archives past 4GB stay readable, and disk backpressure so - * memory stays flat regardless of backup size. + * Downloads a backup into a zip file at `outPath` via the shared + * `downloadBackupArchive` pipeline, supplying the Node-specific pieces: + * presigned URLs are fetched with node:http(s) and decompressed explicitly, + * and the archive streams to disk with backpressure so memory stays flat + * regardless of backup size. * * Writes to `.partial` and renames on success; a failed or aborted * download removes the partial file. @@ -76,8 +98,6 @@ export async function downloadBackupToFile(opts: { signal: AbortSignal; onProgress: (progress: BackupDownloadProgress) => void; }): Promise { - const { manager, backup, outPath, onProgress } = opts; - // The entity shards are served with `Content-Encoding: zstd`; fail before // writing anything if this Node can't decompress them. if (!createZstdDecompress) { @@ -86,238 +106,27 @@ export async function downloadBackupToFile(opts: { ); } - // Internal controller so a zip-pipeline failure also tears down the - // storage-files discovery stream and any in-flight body fetches. - const abortController = new AbortController(); - if (opts.signal.aborted) { - abortController.abort(); - } else { - opts.signal.addEventListener('abort', () => abortController.abort(), { - once: true, - }); - } - const signal = abortController.signal; - - let entitiesCompleted = 0; - let entitiesTotal: number | null = null; - let filesCompleted = 0; - let filesTotal: number | null = null; - let zipBytes = 0; - let bytesRead = 0; - let currentEntry = ''; - const bytesTotal = - backup.uncompressedSize != null - ? backup.uncompressedSize + (backup.filesSize ?? 0) - : null; - - const tick = () => - onProgress({ - entitiesCompleted, - entitiesTotal, - filesCompleted, - filesTotal, - zipBytes, - bytesRead, - bytesTotal, - currentEntry, - }); - - // Throttle by time: a large backup pushes many small chunks and ticking on - // every one is wasted work. Phase changes tick() directly so they're still - // immediate. - const TICK_INTERVAL_MS = 100; - let lastTickAt = 0; - const throttledTick = () => { - const now = Date.now(); - if (now - lastTickAt >= TICK_INTERVAL_MS) { - lastTickAt = now; - tick(); - } - }; - - // Wrap a source body for the zip: decompress if the object is stored - // compressed, and count the uncompressed bytes for progress. - const toEntryStream = (res: IncomingMessage): ReadableStream => { - const encoding = res.headers['content-encoding']; - let stream: Readable = res; - if (encoding === 'zstd') { - stream = pipe(stream, createZstdDecompress()); - } else if (encoding === 'gzip') { - stream = pipe(stream, zlib.createGunzip()); - } else if (encoding) { - res.destroy(); - throw new Error(`Unsupported content encoding: ${encoding}`); - } - const counter = new Transform({ - transform(chunk: Buffer, _enc, cb) { - bytesRead += chunk.length; - throttledTick(); - cb(null, chunk); - }, - }); - return Readable.toWeb(pipe(stream, counter)) as ReadableStream; - }; - - // Storage-files discovery runs concurrently with the entity phase and is - // drained eagerly into a queue. That isn't just overlap: it closes the - // NDJSON connection quickly instead of holding it open (and at the mercy of - // idle timeouts) while multi-GB blobs download. `queueHead` walks the array - // in place, freeing each slot as it's consumed. - const queue: (AppBackupStorageFile | undefined)[] = []; - let queueHead = 0; - let storageDone = false; - let storageError: Error | null = null; - let waitResolve: (() => void) | null = null; - const notify = () => { - const w = waitResolve; - waitResolve = null; - w?.(); - }; - - // Never rejects: failures land in storageError for the drain loop to throw. - const discovery = (async () => { - let discoveryComplete = false; - try { - for await (const file of manager.streamStorageFiles(backup.id, { - signal, - })) { - queue.push(file); - filesTotal = (filesTotal ?? 0) + 1; - throttledTick(); - notify(); - } - discoveryComplete = true; - } catch (e) { - // The abort path is expected when the zip pipeline failed and we tore - // the discovery down. - if ((e as { name?: string })?.name !== 'AbortError') { - storageError = e as Error; - } - } finally { - // A failed listing keeps the total unknown rather than reading as an - // empty-but-complete storage phase. - if (discoveryComplete && filesTotal == null) filesTotal = 0; - storageDone = true; - tick(); - notify(); - } - })(); - - // Entry write order is significant for restore: config.json first, then the - // entities/*.jsonl shards, then files/. In particular ALL entity - // files must be written before ANY storage file. listFiles returns the - // entity files in write order; this generator yields them to completion, - // then drains the storage queue. - type ZipEntry = { name: string; input: ReadableStream }; - const entries = (async function* (): AsyncGenerator { - const files = await manager.listFiles(backup.id, { signal }); - if (files.length === 0) { - throw new Error('No files found for this backup.'); - } - // config.json isn't a namespace — count only the entities/*.jsonl shards. - entitiesTotal = files.filter((f) => f.name !== 'config.json').length; - tick(); - - for (const f of files) { - currentEntry = f.name; - tick(); - const url = await manager.getFileUrl(backup.id, f.name, { signal }); - const res = await fetchStream(url, signal); - if (res.statusCode !== 200) { - res.resume(); - throw new Error(`Failed to fetch ${f.name}: HTTP ${res.statusCode}.`); - } - if (f.name !== 'config.json') entitiesCompleted++; - tick(); - yield { name: f.name, input: toEntryStream(res) }; - } - currentEntry = ''; - tick(); - - while (true) { - if (storageError) throw storageError; - let file: AppBackupStorageFile | undefined; - if (queueHead < queue.length) { - file = queue[queueHead]; - queue[queueHead] = undefined; - queueHead++; - } - if (file) { - currentEntry = file.path || file.locationId; - tick(); - const res = await fetchStream(file.url, signal); - if (res.statusCode !== 200) { - res.resume(); - throw new Error( - `Couldn't download storage file "${file.path}" (HTTP ${res.statusCode}).`, - ); - } - filesCompleted++; - tick(); - yield { name: `files/${file.locationId}`, input: toEntryStream(res) }; - } else if (storageDone) { - break; - } else { - await new Promise((resolve) => { - waitResolve = resolve; - }); - } - } - currentEntry = ''; - tick(); - - if (storageError) throw storageError; - })(); - - const partialPath = `${outPath}.partial`; + const partialPath = `${opts.outPath}.partial`; const fileStream = createWriteStream(partialPath); - const diskWriter = ( - Writable.toWeb(fileStream) as WritableStream - ).getWriter(); const awaitFileClosed = async () => { if (!fileStream.closed) await once(fileStream, 'close'); }; try { - // Loaded on demand so every other CLI command skips parsing it. - const { ZipWriter } = await import('@zip.js/zip.js'); - // Sink that zip.js writes compressed bytes into: it tallies the on-disk - // size for progress, then forwards to the file. Awaiting the disk write - // propagates backpressure up into zip.js, so a fast source can't outrun - // the disk and balloon memory. - const countingSink = new WritableStream({ - async write(chunk) { - zipBytes += chunk.byteLength; - throttledTick(); - await diskWriter.write(chunk); - }, - async close() { - await diskWriter.close(); - tick(); - }, - async abort(reason) { - await diskWriter.abort(reason); - }, + const result = await downloadBackupArchive({ + manager: opts.manager, + backup: opts.backup, + fetchBody, + sink: Writable.toWeb(fileStream) as WritableStream, + createWriter: createZipWriter, + signal: opts.signal, + onProgress: opts.onProgress, }); - - // zip64: without it any archive whose central-directory offset passes 4GB - // writes a wrapped 32-bit offset and the zip is unreadable. - const zipWriter = new ZipWriter(countingSink, { zip64: true, signal }); - for await (const entry of entries) { - await zipWriter.add(entry.name, entry.input, { - lastModDate: backup.backupAt, - }); - } - await zipWriter.close(); await awaitFileClosed(); - await rename(partialPath, outPath); - await discovery; - tick(); - return { entities: entitiesCompleted, files: filesCompleted, zipBytes }; + await rename(partialPath, opts.outPath); + return result; } catch (e) { - // Tear down the discovery stream and any in-flight body fetches so we - // don't keep pulling from S3, and discard the partial file on disk. - abortController.abort(); - await diskWriter.abort(e).catch(() => {}); + // The pipeline already aborted the sink; wait for the fd to close, then + // discard the partial file on disk. await awaitFileClosed().catch(() => {}); await unlink(partialPath).catch(() => {}); throw e; diff --git a/client/packages/platform/src/backupDownload.ts b/client/packages/platform/src/backupDownload.ts new file mode 100644 index 0000000000..bfb373919d --- /dev/null +++ b/client/packages/platform/src/backupDownload.ts @@ -0,0 +1,330 @@ +import type { + AppBackup, + AppBackupStorageFile, + BackupsManager, +} from './backups.ts'; + +export type BackupDownloadProgress = { + entitiesCompleted: number; + entitiesTotal: number | null; + filesCompleted: number; + filesTotal: number | null; + // Compressed bytes written to the sink so far (the zip's on-disk size). + zipBytes: number; + // Uncompressed bytes read from source bodies, and the backup's known + // uncompressed total — the numerator/denominator for a progress bar. + // bytesTotal is null when the backup row carries no sizes. + bytesRead: number; + bytesTotal: number | null; + // The entry currently being fetched in each phase; empty while that phase + // isn't actively fetching, so a finished phase stops claiming a file. + currentEntity: string; + currentFile: string; +}; + +export type BackupDownloadResult = { + entities: number; + files: number; + zipBytes: number; +}; + +/** + * The archive encoder {@link downloadBackupArchive} writes entries through, + * supplied by the caller so this package doesn't depend on a zip + * implementation. zip.js's `ZipWriter` satisfies it structurally, so + * `new ZipWriter(sink, { zip64: true, signal })` works without an adapter. + * + * Implementations must handle archives past 4GB — for zip that means zip64, + * without which the central-directory offsets wrap and the archive is + * silently unreadable. + */ +export type BackupArchiveWriter = { + add( + name: string, + input: ReadableStream, + opts: { lastModDate: Date }, + ): Promise; + close(): Promise; +}; + +export type DownloadBackupArchiveOpts = { + backup: AppBackup; + /** + * Fetches a presigned URL, resolving with the response body and rejecting + * on a non-200 status. Put the status in the message (e.g. `HTTP 403`) — + * it's surfaced to the user alongside the failing entry's name. The entity + * files are served with `Content-Encoding: zstd`; browser fetch decodes + * that transparently, other runtimes must decompress explicitly. + */ + fetchBody: ( + url: string, + signal: AbortSignal, + ) => Promise>; + /** + * Where the archive's bytes go. Closed after the last entry is written; + * aborted when the download fails or is cancelled, so the caller can + * discard partial output. + */ + sink: WritableStream; + /** + * Builds the archive encoder over a sink that already counts progress and + * carries the caller's sink's backpressure. + */ + createWriter: ( + sink: WritableStream, + signal: AbortSignal, + ) => Promise; + signal?: AbortSignal; + onProgress?: (progress: BackupDownloadProgress) => void; +}; + +const isAbortError = (e: unknown): boolean => + (e as { name?: string })?.name === 'AbortError'; + +const errorMessage = (e: unknown): string => + e instanceof Error ? e.message : String(e); + +/** + * Downloads a backup into a single archive written to `opts.sink`: entries + * in the canonical restore order (`config.json`, then the + * `entities/.jsonl` shards, then `files/` storage blobs — + * all entity files before any storage file), with the encoder writing + * through a counting sink that awaits the caller's sink, so a fast source + * can't outrun it and balloon memory. + * + * The runtime-specific pieces are injected: how to fetch a presigned URL + * (`fetchBody`), where the bytes go (`sink`), and the archive encoder + * (`createWriter`). Most callers reach this via + * {@link BackupsManager.downloadArchive}. + */ +export async function downloadBackupArchive( + opts: DownloadBackupArchiveOpts & { + manager: Pick< + BackupsManager, + 'listFiles' | 'getFileUrl' | 'streamStorageFiles' + >; + }, +): Promise { + const { manager, backup, fetchBody, createWriter, onProgress } = opts; + + // Internal controller so a pipeline failure also tears down the + // storage-files discovery stream and any in-flight body fetches. + const abortController = new AbortController(); + if (opts.signal?.aborted) { + abortController.abort(); + } else { + opts.signal?.addEventListener('abort', () => abortController.abort(), { + once: true, + }); + } + const signal = abortController.signal; + + let entitiesCompleted = 0; + let entitiesTotal: number | null = null; + let filesCompleted = 0; + let filesTotal: number | null = null; + let zipBytes = 0; + let bytesRead = 0; + let currentEntity = ''; + let currentFile = ''; + const bytesTotal = + backup.uncompressedSize != null + ? backup.uncompressedSize + (backup.filesSize ?? 0) + : null; + + const tick = () => + onProgress?.({ + entitiesCompleted, + entitiesTotal, + filesCompleted, + filesTotal, + zipBytes, + bytesRead, + bytesTotal, + currentEntity, + currentFile, + }); + + // Throttle by time: a large backup pushes many small chunks and ticking on + // every one is wasted work. Phase changes tick() directly so they're still + // immediate. + const TICK_INTERVAL_MS = 100; + let lastTickAt = 0; + const throttledTick = () => { + const now = Date.now(); + if (now - lastTickAt >= TICK_INTERVAL_MS) { + lastTickAt = now; + tick(); + } + }; + + // Count the uncompressed bytes of a source body for progress as it streams + // into the archive. + const countBytes = ( + body: ReadableStream, + ): ReadableStream => + body.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + bytesRead += chunk.byteLength; + throttledTick(); + controller.enqueue(chunk); + }, + }), + ); + + // Storage-files discovery runs concurrently with the entity phase and is + // drained eagerly into a queue. That isn't just overlap: it closes the + // NDJSON connection quickly instead of holding it open (and at the mercy of + // idle timeouts) while multi-GB blobs download. `queueHead` walks the array + // in place, freeing each slot as it's consumed. + const queue: (AppBackupStorageFile | undefined)[] = []; + let queueHead = 0; + let storageDone = false; + let storageError: Error | null = null; + let waitResolve: (() => void) | null = null; + const notify = () => { + const w = waitResolve; + waitResolve = null; + w?.(); + }; + + // Never rejects: failures land in storageError for the drain loop to throw. + const discovery = (async () => { + let discoveryComplete = false; + try { + for await (const file of manager.streamStorageFiles(backup.id, { + signal, + })) { + queue.push(file); + filesTotal = (filesTotal ?? 0) + 1; + throttledTick(); + notify(); + } + discoveryComplete = true; + } catch (e) { + // The abort path is expected when the pipeline failed and we tore the + // discovery down. + if (!isAbortError(e)) { + storageError = e as Error; + } + } finally { + // A failed listing keeps the total unknown rather than reading as an + // empty-but-complete storage phase. + if (discoveryComplete && filesTotal == null) filesTotal = 0; + storageDone = true; + tick(); + notify(); + } + })(); + + // Entry write order is significant for restore: config.json first, then the + // entities/*.jsonl shards, then files/. In particular ALL entity + // files must be written before ANY storage file. listFiles returns the + // entity files in write order; this generator yields them to completion, + // then drains the storage queue. + type ArchiveEntry = { name: string; input: ReadableStream }; + const entries = (async function* (): AsyncGenerator { + const files = await manager.listFiles(backup.id, { signal }); + if (files.length === 0) { + throw new Error('No files found for this backup.'); + } + // config.json isn't a namespace — count only the entities/*.jsonl shards. + entitiesTotal = files.filter((f) => f.name !== 'config.json').length; + tick(); + + for (const f of files) { + currentEntity = f.name; + tick(); + const url = await manager.getFileUrl(backup.id, f.name, { signal }); + let body: ReadableStream; + try { + body = await fetchBody(url, signal); + } catch (e) { + if (isAbortError(e)) throw e; + throw new Error(`Failed to fetch ${f.name}: ${errorMessage(e)}.`); + } + if (f.name !== 'config.json') entitiesCompleted++; + tick(); + yield { name: f.name, input: countBytes(body) }; + } + currentEntity = ''; + tick(); + + while (true) { + if (storageError) throw storageError; + let file: AppBackupStorageFile | undefined; + if (queueHead < queue.length) { + file = queue[queueHead]; + queue[queueHead] = undefined; + queueHead++; + } + if (file) { + currentFile = file.path || file.locationId; + tick(); + let body: ReadableStream; + try { + body = await fetchBody(file.url, signal); + } catch (e) { + if (isAbortError(e)) throw e; + throw new Error( + `Couldn't download storage file "${file.path}" (${errorMessage(e)}).`, + ); + } + filesCompleted++; + tick(); + yield { name: `files/${file.locationId}`, input: countBytes(body) }; + } else if (storageDone) { + break; + } else { + await new Promise((resolve) => { + waitResolve = resolve; + }); + } + } + currentFile = ''; + tick(); + + if (storageError) throw storageError; + })(); + + const sinkWriter = opts.sink.getWriter(); + try { + // Sink the archive encoder writes into: it tallies the encoded size for + // progress, then forwards to the caller's sink. Awaiting the downstream + // write propagates backpressure up into the encoder, so a fast source + // can't outrun the sink and balloon memory. + const countingSink = new WritableStream({ + async write(chunk) { + zipBytes += chunk.byteLength; + throttledTick(); + await sinkWriter.write(chunk); + }, + async close() { + await sinkWriter.close(); + tick(); + }, + async abort(reason) { + await sinkWriter.abort(reason); + }, + }); + + const writer = await createWriter(countingSink, signal); + for await (const entry of entries) { + await writer.add(entry.name, entry.input, { + lastModDate: backup.backupAt, + }); + } + await writer.close(); + await discovery; + tick(); + return { entities: entitiesCompleted, files: filesCompleted, zipBytes }; + } catch (e) { + // Tear down the discovery stream and any in-flight body fetches so we + // don't keep pulling from S3, and abort the caller's sink so it can + // discard whatever it wrote. + abortController.abort(); + await sinkWriter.abort(e).catch(() => {}); + throw e; + } +} diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts index c0af89a408..7496660947 100644 --- a/client/packages/platform/src/backups.ts +++ b/client/packages/platform/src/backups.ts @@ -1,6 +1,11 @@ import { InstantAPIError, version as coreVersion } from '@instantdb/core'; import type { WithAuth } from '@instantdb/webhooks'; import version from './version.ts'; +import { + downloadBackupArchive, + type BackupDownloadResult, + type DownloadBackupArchiveOpts, +} from './backupDownload.ts'; /** A point-in-time snapshot of an app. */ export type AppBackup = { @@ -59,7 +64,8 @@ type AppBackupResponse = { expires_at: string | null; }; -function toAppBackup(row: AppBackupResponse): AppBackup { +/** Converts a backup row as the server sends it into an {@link AppBackup}. */ +export function toAppBackup(row: AppBackupResponse): AppBackup { return { id: row.id, isn: row.isn, @@ -78,6 +84,24 @@ export function backupZipName(backup: AppBackup): string { return `instant-backup-${safe}.zip`; } +/** + * Estimated size range for a backup's zip archive, or null when the backup + * row carries no sizes. Upper bound: everything stored uncompressed (STORE + * mode and/or files that don't compress). Lower bound: everything compressed + * at a ~4x DEFLATE ratio, best case for text/JSON, but storage files vary + * wildly (raw text compresses well, already-compressed images/videos don't). + * The same divisor applies to both since the file types aren't visible from + * here; the actual zip lands somewhere inside the range. + */ +export function estimateZipSize( + backup: AppBackup, +): { min: number; max: number } | null { + const backupBytes = backup.uncompressedSize ?? backup.dbSize; + if (backupBytes == null || backup.filesSize == null) return null; + const max = backupBytes + backup.filesSize; + return { min: Math.round(max / 4), max }; +} + function authHeaders(token: string): Record { return { authorization: `Bearer ${token}`, @@ -140,8 +164,10 @@ async function* ndjsonLines( * come before ANY storage file, so a restore can process the archive in a * single streaming pass: `config.json` sets up the schema, the `$files` * entities register file metadata, and only then can each blob be matched - * to its entity. {@link listFiles} returns the entity files already in - * write order; write the {@link streamStorageFiles} blobs after them. + * to its entity. {@link downloadArchive} implements that order end to end; + * {@link listFiles} (which returns the entity files already in write order) + * and {@link streamStorageFiles} are the pieces for building a custom + * pipeline. */ export class BackupsManager { #appId: string; @@ -270,4 +296,17 @@ export class BackupsManager { ); } } + + /** + * Downloads the backup into a single zip archive written to `opts.sink`, + * entries in the canonical restore order described above. The caller + * supplies the runtime-specific pieces — how to fetch a presigned URL, + * where the bytes go, and the archive encoder (e.g. zip.js's `ZipWriter`); + * see {@link DownloadBackupArchiveOpts}. + */ + downloadArchive( + opts: DownloadBackupArchiveOpts, + ): Promise { + return downloadBackupArchive({ manager: this, ...opts }); + } } diff --git a/client/packages/platform/src/index.ts b/client/packages/platform/src/index.ts index acced89e02..8b2ed4c2df 100644 --- a/client/packages/platform/src/index.ts +++ b/client/packages/platform/src/index.ts @@ -118,11 +118,21 @@ export { export { BackupsManager, backupZipName, + estimateZipSize, + toAppBackup, type AppBackup, type AppBackupFile, type AppBackupStorageFile, } from './backups.ts'; +export { + downloadBackupArchive, + type DownloadBackupArchiveOpts, + type BackupArchiveWriter, + type BackupDownloadProgress, + type BackupDownloadResult, +} from './backupDownload.ts'; + export { DEFAULT_OAUTH_CALLBACK_URL, oauthCallbackURL, diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index 810a5e1e7d..e33dac13bf 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -12,8 +12,11 @@ import { ArrowsPointingOutIcon, XMarkIcon } from '@heroicons/react/24/outline'; import { BackupsManager, - type AppBackupFile, - type AppBackupStorageFile, + backupZipName, + estimateZipSize, + toAppBackup, + type AppBackup, + type BackupDownloadProgress, } from '@instantdb/platform'; import config from '@/lib/config'; @@ -25,30 +28,12 @@ import { Button, Content, Dialog, SubsectionHeading } from '@/components/ui'; import { formatTimestamp } from '@/components/dash/shared'; import { useDarkMode } from '@/components/dash/DarkModeToggle'; -type DownloadProgress = { - entitiesCompleted: number; - entitiesTotal: number | null; - filesCompleted: number; - filesTotal: number | null; - // Compressed bytes written to disk so far (the zip's on-disk size). - bytes: number; - // Uncompressed bytes read from source bodies, and the backup's known - // uncompressed total — the numerator/denominator for the % progress bar. - // bytesTotal is null when the backup row carries no sizes. - bytesRead: number; - bytesTotal: number | null; - // Empty while we aren't actively fetching from that phase — clear once - // the yield is done so the line stops claiming a file after completion. - currentEntity: string; - currentFile: string; +type DownloadProgress = BackupDownloadProgress & { + // The picker-chosen destination name, folded into progress so the dialog + // and pill can label where the zip is going. outputFilename: string; }; -function backupZipName(backup: InstantAppBackup): string { - const safe = backup.backup_at.replace(/[:.]/g, '-'); - return `instant-backup-${safe}.zip`; -} - function formatBytes(n: number): string { // Decimal (1000-based) units with SI labels, to match how macOS/Finder // reports file sizes so the number lines up with what lands on disk. @@ -72,12 +57,6 @@ type SaveFileHandle = Awaited< ReturnType >; -type ZipEntry = { - name: string; - lastModified: Date; - input: ReadableStream; -}; - async function registerDownloadServiceWorker(): Promise { if (typeof navigator === 'undefined' || !('serviceWorker' in navigator)) { return; @@ -111,7 +90,7 @@ async function downloadBackup( showSaveFilePicker: typeof import('native-file-system-adapter').showSaveFilePicker, token: string, appId: string, - backup: InstantAppBackup, + backup: AppBackup, setProgress: (p: DownloadProgress) => void, abortController: AbortController, ): Promise { @@ -136,268 +115,43 @@ async function downloadBackup( withAuth: (operation) => operation(token), }); - // ---- Shared progress state ---- - // current = files (backup + storage) fully fetched. - // total = backup file count + storage files ever discovered. - // `storageQueue` is a FIFO of yet-to-fetch storage files only — dequeued as - // the generator processes them, so it stays bounded by (discovery rate - - // fetch rate). It's an object with head/tail indices rather than an array - // because discovery streams the whole list in up front: Array.shift() is O(n), - // so draining a large backup would be O(n²). Object access is O(1) and delete - // frees each slot as it's consumed. - // Two parallel counters surfaced in the dialog: - // entities — the backup payload (config.json + per-etype JSONL shards). - // files — the user's $files (storage uploads). - // entitiesTotal is null until listFiles resolves; filesTotal is null - // until the discovery NDJSON returns at least one line OR completes - // (an empty stream sets it to 0 so the dialog shows "0 of 0 files"). - let entitiesCompleted = 0; - let entitiesTotal: number | null = null; - let filesCompleted = 0; - let filesTotal: number | null = null; - let currentEntity = ''; - let currentFile = ''; - let outputFilename = pickerHandle?.name ?? filename; - let zipBytes = 0; - let uncompressedRead = 0; - // Denominator for the % bar: uncompressed entities + storage files, since we - // read (and count) both. Gated on uncompressed_size — no size, no bar — with - // files_size added when present. - const bytesTotal = - backup.uncompressed_size != null - ? backup.uncompressed_size + (backup.files_size ?? 0) - : null; - const storageQueue: Record = {}; - let queueHead = 0; - let queueTail = 0; - let storageDone = false; - let storageError: Error | null = null; - let waitResolve: (() => void) | null = null; - - // The caller-supplied AbortController is shared across every fetch we - // own (the NDJSON discovery stream, backup file body fetches, storage - // file body fetches). The caller can abort it externally (e.g. dialog - // close) and we also abort it from our own catch so the discovery stops - // pulling from S3 if the zip pipeline throws. - const signal = abortController.signal; - - const tick = () => - setProgress({ - entitiesCompleted, - entitiesTotal, - filesCompleted, - filesTotal, - bytes: zipBytes, - bytesRead: uncompressedRead, - bytesTotal, - currentEntity, - currentFile, - outputFilename, - }); - - // Count uncompressed bytes as each source body streams through, for the % - // bar. This is the input size (matches backup.uncompressed_size); zipBytes - // is the compressed output written to disk. - const countRead = ( - body: ReadableStream, - ): ReadableStream => - body.pipeThrough( - new TransformStream({ - transform(chunk, controller) { - uncompressedRead += chunk.byteLength; - controller.enqueue(chunk); - }, - }), - ); - - const notify = () => { - const w = waitResolve; - waitResolve = null; - w?.(); - }; - - // Throttle progress updates by time: a large backup pushes many small chunks - // (and a large listing many discovery lines) and ticking (a React setState) - // on every one would flood re-renders. Ticking at most every 100ms stays - // smooth at any size. Phase changes and completion tick() directly so they're - // still immediate. - const TICK_INTERVAL_MS = 100; - let lastTickAt = 0; - const throttledTick = () => { - const now = Date.now(); - if (now - lastTickAt >= TICK_INTERVAL_MS) { - lastTickAt = now; - tick(); - } - }; - - // Kick off storage-files discovery in parallel with the backup file list. - // It fills `storageQueue` as URLs stream in. The manager consumes the - // server's terminal `done` sentinel and throws if the stream ends without - // it (a server-side failure truncated the listing), so a partial zip can't - // pass as complete. - void (async () => { - let discoveryComplete = false; - try { - for await (const file of manager.streamStorageFiles(backup.id, { - signal, - })) { - storageQueue[queueTail++] = file; - filesTotal = (filesTotal ?? 0) + 1; - throttledTick(); - notify(); - } - discoveryComplete = true; - } catch (e) { - // The abort path is expected when the zip pipeline failed and we - // tore the discovery down - if ((e as { name?: string })?.name !== 'AbortError') { - storageError = e as Error; - } - } finally { - // Discovery finished without yielding anything (no storage files) — - // surface "0 of 0" so the dialog doesn't sit on "?" forever. A failed - // listing keeps the total unknown so the error UI doesn't claim an - // empty-but-complete storage phase. - if (discoveryComplete && filesTotal == null) filesTotal = 0; - storageDone = true; - tick(); - notify(); - } - })(); - - let files: AppBackupFile[]; - try { - files = await manager.listFiles(backup.id, { signal }); - if (files.length === 0) { - throw new Error('No files found for this backup.'); - } - } catch (e) { - // Entity discovery failed before we reached the zip pipeline's own - // teardown. Stop the background storage-files discovery stream so it - // doesn't keep pulling from S3 after we bail. - abortController.abort(); - throw e; - } - // config.json isn't a namespace — count only the entities/*.jsonl shards so - // the "N namespaces" label isn't off by one. - entitiesTotal = files.filter((f) => f.name !== 'config.json').length; - tick(); - - // Entry write order is significant for restore: config first, then the - // entities/*.jsonl shards, then files/${locationId}. In particular ALL entity - // files must be written before ANY storage file. This generator preserves - // that: it yields the entity `files` (config + entities/*.jsonl, in the order - // listFiles returns them) to completion, then drains the storage queue. - const entries = (async function* (): AsyncGenerator { - for (const f of files) { - currentEntity = f.name; - tick(); - const url = await manager.getFileUrl(backup.id, f.name, { signal }); + const outputFilename = pickerHandle?.name ?? filename; + const writable = await pickerHandle.createWritable(); + + // The shared pipeline owns ordering, progress, backpressure, and teardown; + // this call supplies the browser-specific pieces. The caller's + // AbortController reaches every fetch the pipeline makes — the caller + // aborts it externally (e.g. cancel) and the pipeline tears everything + // down itself on failure, so the discovery stream stops pulling from S3 + // either way. + await manager.downloadArchive({ + backup, + // The entity shards' `Content-Encoding: zstd` is decoded transparently + // by the browser's fetch. + fetchBody: async (url, signal) => { const res = await fetch(url, { signal }); if (!res.ok) { - throw new Error(`Failed to fetch ${f.name}: ${res.status}`); - } - if (f.name !== 'config.json') entitiesCompleted++; - tick(); - yield { - name: f.name, - lastModified: new Date(backup.backup_at), - input: countRead(res.body!), - }; - } - // Entity phase done — clear so the dialog stops claiming we're still - // downloading an entity file once we move into storage. - currentEntity = ''; - tick(); - - while (true) { - if (storageError) throw storageError; - let obj: AppBackupStorageFile | undefined; - if (queueHead < queueTail) { - obj = storageQueue[queueHead]; - delete storageQueue[queueHead]; - queueHead++; + throw new Error(`HTTP ${res.status}`); } - if (obj) { - currentFile = obj.path || obj.locationId; - tick(); - const fileRes = await fetch(obj.url, { signal }); - if (!fileRes.ok) { - throw new Error( - `Couldn't download storage file "${obj.path}" (HTTP ${fileRes.status}).`, - ); - } - filesCompleted++; - tick(); - yield { - name: `files/${obj.locationId}`, - lastModified: new Date(backup.backup_at), - input: countRead(fileRes.body!), - }; - } else if (storageDone) { - break; - } else { - await new Promise((resolve) => { - waitResolve = resolve; - }); - } - } - // Storage phase done, clear so the dialog stops showing the last file. - currentFile = ''; - tick(); - - if (storageError) throw storageError; - })(); - - let diskWriter: WritableStreamDefaultWriter | null = null; - try { - // Load the zip encoder on demand (kept out of the shared bundle). Inside the - // try so a chunk-load failure aborts the background discovery too. - const { ZipWriter } = await import('@zip.js/zip.js'); - const writable = await pickerHandle.createWritable(); - const writer = writable.getWriter(); - diskWriter = writer; - // Sink that zip.js writes compressed bytes into: it tallies the on-disk - // (compressed) size for progress, then forwards to the file. Awaiting the - // disk write propagates backpressure up into zip.js, so a fast source can't - // outrun the disk and balloon memory. - const countingSink = new WritableStream({ - async write(chunk) { - zipBytes += chunk.byteLength; - throttledTick(); - await writer.write(chunk); - }, - async close() { - await writer.close(); - tick(); - }, - async abort(reason) { - await writer.abort(reason); - }, - }); - - // zip64: without it any archive whose central-directory offset passes 4GB - // writes a wrapped 32-bit offset and the zip is unreadable. Everything else - // is left at zip.js's defaults: workers + CompressionStream when available - // (compression and CRC32 run off the main thread so a multi-GB backup - // doesn't jank the tab), each degrading to an inline main-thread codec on - // browsers that lack them — correctness (incl. zip64) is unaffected either way. - const zipWriter = new ZipWriter(countingSink, { zip64: true, signal }); - for await (const entry of entries) { - await zipWriter.add(entry.name, entry.input, { - lastModDate: entry.lastModified, - }); - } - await zipWriter.close(); - return { via: 'picker', filename: pickerHandle.name ?? filename }; - } catch (e) { - // Tear down the background discovery and any in-flight body fetches so we - // don't keep pulling from S3, and discard the partial file on disk. - abortController.abort(); - await diskWriter?.abort(e).catch(() => {}); - throw e; - } + return res.body!; + }, + sink: writable, + createWriter: async (sink, signal) => { + // Load the zip encoder on demand (kept out of the shared bundle). + const { ZipWriter } = await import('@zip.js/zip.js'); + // zip64: without it any archive whose central-directory offset passes + // 4GB writes a wrapped 32-bit offset and the zip is unreadable. + // Everything else is left at zip.js's defaults: workers + + // CompressionStream when available (compression and CRC32 run off the + // main thread so a multi-GB backup doesn't jank the tab), each + // degrading to an inline main-thread codec on browsers that lack them — + // correctness (incl. zip64) is unaffected either way. + return new ZipWriter(sink, { zip64: true, signal }); + }, + signal: abortController.signal, + onProgress: (p) => setProgress({ ...p, outputFilename }), + }); + return { via: 'picker', filename: pickerHandle.name ?? filename }; } type DownloadDialogState = @@ -533,6 +287,9 @@ function DownloadInstance({ onDone: () => void; }) { const { darkMode } = useDarkMode(); + // The dash API hands us the raw server row; the platform helpers + // (downloadArchive, backupZipName, estimateZipSize) take the parsed shape. + const appBackup = useMemo(() => toAppBackup(backup), [backup]); const [state, setState] = useState({ kind: 'confirm' }); const abortRef = useRef(null); // Set when the user cancels. The download promise rejects asynchronously with @@ -598,18 +355,7 @@ function DownloadInstance({ // (see handleClose) while the state machine keeps running; dismissing tears // the whole instance down via onDone. - // Upper bound: everything stored uncompressed (STORE mode and/or files - // that don't compress). Lower bound: everything compressed at a ~3x - // DEFLATE ratio, best case for text/JSON, but storage files vary wildly - // (raw text compresses well, already-compressed images/videos don't). - // We apply the same divisor to both since we can't see the file types - // from here; the actual zip will land somewhere inside the range. - const backupBytes = backup.uncompressed_size ?? backup.db_size; - const filesBytes = backup.files_size; - const hasSizes = backupBytes != null && filesBytes != null; - const totalBytes = (backupBytes ?? 0) + (filesBytes ?? 0); - const maxBytes = totalBytes; - const minBytes = Math.round(totalBytes / 4); + const sizeEstimate = estimateZipSize(appBackup); const start = async () => { const showSaveFilePicker = pickerRef.current; @@ -622,7 +368,7 @@ function DownloadInstance({ showSaveFilePicker, token, app.id, - backup, + appBackup, (progress) => { setState((prev) => prev.kind === 'downloading' ? { ...prev, progress } : prev, @@ -705,11 +451,11 @@ function DownloadInstance({ {formatTimestamp(backup.backup_at)}. This will download a zip file containing all entities and all files. - {hasSizes ? ( + {sizeEstimate ? ( The zip file will be between{' '} - {formatBytes(minBytes)} and{' '} - {formatBytes(maxBytes)}, depending on the + {formatBytes(sizeEstimate.min)} and{' '} + {formatBytes(sizeEstimate.max)}, depending on the compression ratio. ) : null} @@ -779,7 +525,7 @@ function DownloadInstance({ {progress?.outputFilename ?? ''} - {formatBytes(progress?.bytes ?? 0)} + {formatBytes(progress?.zipBytes ?? 0)} {pct != null ? ` · ${Math.round(pct)}%` : ''} @@ -826,11 +572,11 @@ function DownloadInstance({ Saved to{' '} - {progress?.outputFilename ?? backupZipName(backup)} + {progress?.outputFilename ?? backupZipName(appBackup)} - {formatBytes(progress?.bytes ?? 0)} + {formatBytes(progress?.zipBytes ?? 0)} @@ -896,7 +642,7 @@ function DownloadInstance({ ) : null} - {progress?.outputFilename ?? backupZipName(backup)} + {progress?.outputFilename ?? backupZipName(appBackup)} {state.kind === 'downloading' && pct != null ? (
From 156bc89023a86a3c58a200a10a0564e617566f1e Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:05:20 -0700 Subject: [PATCH 06/18] Share one formatFileSize between the backup surfaces --- .../cli/src/commands/backup/download.ts | 9 +++---- .../packages/cli/src/commands/backup/list.ts | 21 +++------------- client/packages/platform/src/backups.ts | 18 ++++++++++++++ client/packages/platform/src/index.ts | 1 + .../components/dash/BackupDownloadDialog.tsx | 24 ++++--------------- 5 files changed, 32 insertions(+), 41 deletions(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 783315d617..9c7a7ecbf1 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -7,6 +7,7 @@ import throttle from 'lodash.throttle'; import { backupZipName, estimateZipSize, + formatFileSize, type AppBackup, type BackupsManager, } from '@instantdb/platform'; @@ -22,7 +23,7 @@ import { } from '../../lib/backupDownload.ts'; import { promptOk, runUIEffect } from '../../lib/ui.ts'; import { UI } from '../../ui/index.ts'; -import { formatBackupDate, formatBytes } from './list.ts'; +import { formatBackupDate } from './list.ts'; const pickBackup = ( backups: AppBackup[], @@ -91,7 +92,7 @@ function makeProgressRenderer() { : `storage files ${p.filesCompleted}/${p.filesTotal}`, ); } - let bytes = formatBytes(p.zipBytes); + let bytes = formatFileSize(p.zipBytes); if (p.bytesTotal != null && p.bytesTotal > 0) { const pct = Math.min(100, Math.round((p.bytesRead / p.bytesTotal) * 100)); bytes += ` (${pct}%)`; @@ -169,7 +170,7 @@ export const backupDownloadCmd = Effect.fn(function* ( const sizes = estimateZipSize(backup); const estimate = sizes - ? ` The zip file will be between ${formatBytes(sizes.min)} and ${formatBytes(sizes.max)}, depending on the compression ratio.` + ? ` The zip file will be between ${formatFileSize(sizes.min)} and ${formatFileSize(sizes.max)}, depending on the compression ratio.` : ''; const ok = yield* promptOk({ @@ -204,6 +205,6 @@ export const backupDownloadCmd = Effect.fn(function* ( ? ` and ${result.files.toLocaleString()} storage files` : ''; yield* Effect.log( - `Saved ${result.entities.toLocaleString()} namespaces${filesPart} (${formatBytes(result.zipBytes)})`, + `Saved ${result.entities.toLocaleString()} namespaces${filesPart} (${formatFileSize(result.zipBytes)})`, ); }); diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index ececc7952b..09562bb762 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -1,27 +1,12 @@ import chalk from 'chalk'; import { Effect } from 'effect'; -import type { AppBackup } from '@instantdb/platform'; +import { formatFileSize, type AppBackup } from '@instantdb/platform'; import type { backupListDef, OptsFromCommand } from '../../index.ts'; import { useBackupsManager } from '../../lib/backups.ts'; export const formatBackupDate = (date: Date) => `${date.toISOString().replace('T', ' ').slice(0, 16)} UTC`; -export function formatBytes(n: number): string { - // Decimal (1000-based) units with SI labels, to match how macOS/Finder - // reports file sizes. - if (n < 1000) return `${n} B`; - const units = ['KB', 'MB', 'GB', 'TB']; - let i = -1; - let v = n; - do { - v /= 1000; - i++; - } while (v >= 1000 && i < units.length - 1); - const digits = v < 10 ? 2 : v < 100 ? 1 : 0; - return `${v.toFixed(digits)} ${units[i]}`; -} - export const renderBackup = (backup: AppBackup) => Effect.gen(function* () { yield* Effect.log(chalk.cyan(formatBackupDate(backup.backupAt))); @@ -30,11 +15,11 @@ export const renderBackup = (backup: AppBackup) => yield* Effect.log(` Description: ${backup.description}`); } if (backup.dbSize != null) { - yield* Effect.log(` Database size: ${formatBytes(backup.dbSize)}`); + yield* Effect.log(` Database size: ${formatFileSize(backup.dbSize)}`); } if (backup.filesSize != null) { yield* Effect.log( - ` Storage files size: ${formatBytes(backup.filesSize)}`, + ` Storage files size: ${formatFileSize(backup.filesSize)}`, ); } if (backup.expiresAt) { diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts index 7496660947..dde451f8d9 100644 --- a/client/packages/platform/src/backups.ts +++ b/client/packages/platform/src/backups.ts @@ -84,6 +84,24 @@ export function backupZipName(backup: AppBackup): string { return `instant-backup-${safe}.zip`; } +/** + * Formats a byte count the way macOS/Finder reports file sizes: decimal + * (1000-based) units with SI labels, so the number lines up with what lands + * on disk. + */ +export function formatFileSize(n: number): string { + if (n < 1000) return `${n} B`; + const units = ['KB', 'MB', 'GB', 'TB']; + let i = -1; + let v = n; + do { + v /= 1000; + i++; + } while (v >= 1000 && i < units.length - 1); + const digits = v < 10 ? 2 : v < 100 ? 1 : 0; + return `${v.toFixed(digits)} ${units[i]}`; +} + /** * Estimated size range for a backup's zip archive, or null when the backup * row carries no sizes. Upper bound: everything stored uncompressed (STORE diff --git a/client/packages/platform/src/index.ts b/client/packages/platform/src/index.ts index 8b2ed4c2df..4a112fbc68 100644 --- a/client/packages/platform/src/index.ts +++ b/client/packages/platform/src/index.ts @@ -119,6 +119,7 @@ export { BackupsManager, backupZipName, estimateZipSize, + formatFileSize, toAppBackup, type AppBackup, type AppBackupFile, diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index e33dac13bf..28c408309f 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -14,6 +14,7 @@ import { BackupsManager, backupZipName, estimateZipSize, + formatFileSize, toAppBackup, type AppBackup, type BackupDownloadProgress, @@ -34,21 +35,6 @@ type DownloadProgress = BackupDownloadProgress & { outputFilename: string; }; -function formatBytes(n: number): string { - // Decimal (1000-based) units with SI labels, to match how macOS/Finder - // reports file sizes so the number lines up with what lands on disk. - if (n < 1000) return `${n} B`; - const units = ['KB', 'MB', 'GB', 'TB']; - let i = -1; - let v = n; - do { - v /= 1000; - i++; - } while (v >= 1000 && i < units.length - 1); - const digits = v < 10 ? 2 : v < 100 ? 1 : 0; - return `${v.toFixed(digits)} ${units[i]}`; -} - type DownloadResult = | { via: 'picker'; filename: string } | { via: 'browser-default'; filename: string }; @@ -454,8 +440,8 @@ function DownloadInstance({ {sizeEstimate ? ( The zip file will be between{' '} - {formatBytes(sizeEstimate.min)} and{' '} - {formatBytes(sizeEstimate.max)}, depending on the + {formatFileSize(sizeEstimate.min)} and{' '} + {formatFileSize(sizeEstimate.max)}, depending on the compression ratio. ) : null} @@ -525,7 +511,7 @@ function DownloadInstance({ {progress?.outputFilename ?? ''} - {formatBytes(progress?.zipBytes ?? 0)} + {formatFileSize(progress?.zipBytes ?? 0)} {pct != null ? ` · ${Math.round(pct)}%` : ''}
@@ -576,7 +562,7 @@ function DownloadInstance({ - {formatBytes(progress?.zipBytes ?? 0)} + {formatFileSize(progress?.zipBytes ?? 0)} From 5390786473501b33c70e4fb804e69990e7741f69 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:35:48 -0700 Subject: [PATCH 07/18] Fix the cancellation race and count entries only when fully written A caller abort that landed while no fetch was in flight surfaced only in the storage discovery stream, which swallows AbortError as expected teardown; the pipeline could then close and rename a complete-looking archive that silently omits undiscovered storage files. Check the signal explicitly in the drain loop and before closing the writer. Progress counters now increment after the writer consumes an entry, not when its fetch starts, so "N of M" reflects fully-written files. Also: name pathless storage files by locationId in errors, reject locationIds that could escape the files/ archive prefix, and fix formatFileSize rounding across a unit boundary (999,500 read as 1000 KB). --- .../__tests__/src/backupDownload.test.ts | 193 ++++++++++++++++++ .../platform/__tests__/src/backups.test.ts | 23 +++ .../packages/platform/src/backupDownload.ts | 44 +++- client/packages/platform/src/backups.ts | 14 +- 4 files changed, 264 insertions(+), 10 deletions(-) create mode 100644 client/packages/platform/__tests__/src/backupDownload.test.ts diff --git a/client/packages/platform/__tests__/src/backupDownload.test.ts b/client/packages/platform/__tests__/src/backupDownload.test.ts new file mode 100644 index 0000000000..e9fb6fa99e --- /dev/null +++ b/client/packages/platform/__tests__/src/backupDownload.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, test } from 'vitest'; +import { downloadBackupArchive } from '../../src/backupDownload.ts'; +import type { AppBackup } from '../../src/backups.ts'; + +const backup: AppBackup = { + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 16, + dbSize: 100, + uncompressedSize: 40, + description: null, + expiresAt: null, +}; + +const bodyOf = (text: string) => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }); + +const fetchBody = async (url: string) => bodyOf(`body:${url}`); + +const listFiles = async () => [ + { name: 'config.json', size: 1 }, + { name: 'entities/todos.jsonl', size: 1 }, +]; +const getFileUrl = async (_backupId: string, name: string) => name; + +// Minimal archive writer: records entry names, drains each input, and writes +// a byte through the counting sink so zipBytes moves like a real encoder. +const makeWriter = + (names: string[]) => async (sink: WritableStream) => { + const w = sink.getWriter(); + return { + add: async (name: string, input: ReadableStream) => { + names.push(name); + for await (const _chunk of input) { + // drain + } + await w.write(new Uint8Array([0])); + }, + close: () => w.close(), + }; + }; + +const nullSink = () => new WritableStream({ write() {} }); + +describe('downloadBackupArchive', () => { + test('writes entries in canonical order through the injected writer', async () => { + const names: string[] = []; + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + }, + } as any; + + const result = await downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter: makeWriter(names), + }); + + expect(names).toEqual([ + 'config.json', + 'entities/todos.jsonl', + 'files/loc-1', + ]); + expect(result.entities).toBe(1); + expect(result.files).toBe(1); + expect(result.zipBytes).toBe(3); + }); + + test('counts an entry as completed only after the writer consumed it', async () => { + const seen: Array<{ + name: string; + entitiesCompleted: number; + filesCompleted: number; + }> = []; + let last = { entitiesCompleted: 0, filesCompleted: 0 }; + const createWriter = async (sink: WritableStream) => { + const w = sink.getWriter(); + return { + add: async (name: string, input: ReadableStream) => { + seen.push({ name, ...last }); + for await (const _chunk of input) { + // drain + } + await w.write(new Uint8Array([0])); + }, + close: () => w.close(), + }; + }; + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + }, + } as any; + + await downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter, + onProgress: (p) => { + last = { + entitiesCompleted: p.entitiesCompleted, + filesCompleted: p.filesCompleted, + }; + }, + }); + + // At the moment each add starts, the entry being written isn't counted. + expect(seen).toEqual([ + { name: 'config.json', entitiesCompleted: 0, filesCompleted: 0 }, + { name: 'entities/todos.jsonl', entitiesCompleted: 0, filesCompleted: 0 }, + { name: 'files/loc-1', entitiesCompleted: 1, filesCompleted: 0 }, + ]); + }); + + test('a cancellation while storage discovery is idle rejects instead of completing', async () => { + const controller = new AbortController(); + const manager = { + listFiles, + getFileUrl, + // Yields one file, then holds the stream open like a server with more + // to send, rejecting with AbortError when the download is cancelled — + // the same shape as a real aborted fetch. + streamStorageFiles: ( + _backupId: string, + opts?: { signal?: AbortSignal }, + ) => + (async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + yield await new Promise((_resolve, reject) => { + opts?.signal?.addEventListener('abort', () => + reject( + Object.assign(new Error('aborted'), { name: 'AbortError' }), + ), + ); + }); + })(), + } as any; + + const promise = downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter: makeWriter([]), + signal: controller.signal, + onProgress: (p) => { + // Cancel once the only discovered file is fully written and the + // drain loop is about to go idle. + if (p.filesCompleted === 1) controller.abort(); + }, + }); + + await expect(promise).rejects.toMatchObject({ name: 'AbortError' }); + }); + + test('names a pathless storage file by locationId when its download fails', async () => { + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-9', path: null, url: 'bad-url' }; + }, + } as any; + + await expect( + downloadBackupArchive({ + manager, + backup, + fetchBody: async (url: string) => { + if (url === 'bad-url') throw new Error('HTTP 500'); + return bodyOf('x'); + }, + sink: nullSink(), + createWriter: makeWriter([]), + }), + ).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).'); + }); +}); diff --git a/client/packages/platform/__tests__/src/backups.test.ts b/client/packages/platform/__tests__/src/backups.test.ts index d6878caabf..95e10e0ecc 100644 --- a/client/packages/platform/__tests__/src/backups.test.ts +++ b/client/packages/platform/__tests__/src/backups.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { BackupsManager, backupZipName, + formatFileSize, type AppBackupStorageFile, } from '../../src/backups.ts'; @@ -94,6 +95,15 @@ describe('streamStorageFiles', () => { expect(await collect(makeManager())).toEqual([]); }); + test('rejects a locationId that could escape the archive path', async () => { + stubFetchBody([ + '{"locationId":"../evil","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":true}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); + }); + test('throws when the stream ends without the done sentinel', async () => { stubFetchBody([ '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', @@ -143,3 +153,16 @@ describe('list', () => { ); }); }); + +describe('formatFileSize', () => { + test('formats with decimal units and adaptive precision', () => { + expect(formatFileSize(999)).toBe('999 B'); + expect(formatFileSize(3510)).toBe('3.51 KB'); + expect(formatFileSize(1_540_000_000)).toBe('1.54 GB'); + }); + + test('promotes across the unit boundary when rounding', () => { + expect(formatFileSize(999_500)).toBe('1.00 MB'); + expect(formatFileSize(999_999_500)).toBe('1.00 GB'); + }); +}); diff --git a/client/packages/platform/src/backupDownload.ts b/client/packages/platform/src/backupDownload.ts index 015b50f407..68ac929c31 100644 --- a/client/packages/platform/src/backupDownload.ts +++ b/client/packages/platform/src/backupDownload.ts @@ -223,7 +223,13 @@ export async function downloadBackupArchive( // files must be written before ANY storage file. listFiles returns the // entity files in write order; this generator yields them to completion, // then drains the storage queue. - type ArchiveEntry = { name: string; input: ReadableStream }; + type ArchiveEntry = { + name: string; + input: ReadableStream; + // Fired after the writer finishes consuming the entry, so completion + // counters reflect fully-written files rather than started fetches. + onAdded: () => void; + }; const entries = (async function* (): AsyncGenerator { const files = await manager.listFiles(backup.id, { signal }); if (files.length === 0) { @@ -252,15 +258,25 @@ export async function downloadBackupArchive( if (isAbortError(e)) throw e; throw new Error(`Failed to fetch ${f.name}: ${errorMessage(e)}.`); } - if (f.name !== 'config.json') entitiesCompleted++; - tick(); - yield { name: f.name, input: countBytes(body) }; + yield { + name: f.name, + input: countBytes(body), + onAdded: () => { + if (f.name !== 'config.json') entitiesCompleted++; + tick(); + }, + }; } currentEntity = ''; tick(); while (true) { if (storageError) throw storageError; + // A caller abort while no fetch is in flight surfaces only in the + // discovery stream, which swallows it as expected teardown — check + // explicitly so a cancellation can't read as a complete storage phase + // with files still undiscovered. + signal.throwIfAborted(); let file: AppBackupStorageFile | undefined; if (queueHead < queue.length) { file = queue[queueHead]; @@ -268,7 +284,8 @@ export async function downloadBackupArchive( queueHead++; } if (file) { - currentFile = file.path || file.locationId; + const label = file.path || file.locationId; + currentFile = label; tick(); let body: ReadableStream; try { @@ -276,12 +293,17 @@ export async function downloadBackupArchive( } catch (e) { if (isAbortError(e)) throw e; throw new Error( - `Couldn't download storage file "${file.path}" (${errorMessage(e)}).`, + `Couldn't download storage file "${label}" (${errorMessage(e)}).`, ); } - filesCompleted++; - tick(); - yield { name: `files/${file.locationId}`, input: countBytes(body) }; + yield { + name: `files/${file.locationId}`, + input: countBytes(body), + onAdded: () => { + filesCompleted++; + tick(); + }, + }; } else if (storageDone) { break; } else { @@ -322,7 +344,11 @@ export async function downloadBackupArchive( await writer.add(entry.name, entry.input, { lastModDate: backup.backupAt, }); + entry.onAdded(); } + // A caller abort that lands after the last entry lets the generator + // finish cleanly; don't close and return a complete-looking archive. + signal.throwIfAborted(); await writer.close(); await discovery; tick(); diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts index dde451f8d9..906a233ab3 100644 --- a/client/packages/platform/src/backups.ts +++ b/client/packages/platform/src/backups.ts @@ -98,7 +98,14 @@ export function formatFileSize(n: number): string { v /= 1000; i++; } while (v >= 1000 && i < units.length - 1); - const digits = v < 10 ? 2 : v < 100 ? 1 : 0; + let digits = v < 10 ? 2 : v < 100 ? 1 : 0; + // Rounding can cross a unit boundary (999,500 would read "1000 KB"); + // promote to the next unit instead. + if (Number(v.toFixed(digits)) >= 1000 && i < units.length - 1) { + v /= 1000; + i++; + digits = 2; + } return `${v.toFixed(digits)} ${units[i]}`; } @@ -295,6 +302,11 @@ export class BackupsManager { if ( typeof line.locationId !== 'string' || line.locationId.length === 0 || + // The locationId becomes the archive entry path `files/`; + // reject separators and control characters that could escape it. + /[/\\\u0000-\u001f\u007f]/.test(line.locationId) || + line.locationId === '.' || + line.locationId === '..' || typeof line.url !== 'string' || line.url.length === 0 ) { From 97a39b0ab820fae9bd0003be8c45bfe5b0beff54 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:35:48 -0700 Subject: [PATCH 08/18] Harden the CLI backup download path Reject a backup id combined with --latest and an output path that is a directory before downloading; randomize the .partial suffix and open it exclusively so concurrent or stale runs can't collide; fsync before the final rename; strip control characters from user-controlled descriptions printed to the terminal; and name the auth source in backups errors so a stale INSTANT_APP_ADMIN_TOKEN shadowing a fresh login is diagnosable. --- .../cli/__tests__/backupDownload.test.ts | 20 +++++++++++++------ client/packages/cli/__tests__/backups.test.ts | 18 +++++++++++++++++ .../cli/src/commands/backup/download.ts | 19 +++++++++++++++--- .../packages/cli/src/commands/backup/list.ts | 8 +++++++- client/packages/cli/src/lib/backupDownload.ts | 18 ++++++++++++++--- client/packages/cli/src/lib/backups.ts | 11 +++++++++- 6 files changed, 80 insertions(+), 14 deletions(-) diff --git a/client/packages/cli/__tests__/backupDownload.test.ts b/client/packages/cli/__tests__/backupDownload.test.ts index df09e49bee..0504920ad6 100644 --- a/client/packages/cli/__tests__/backupDownload.test.ts +++ b/client/packages/cli/__tests__/backupDownload.test.ts @@ -1,9 +1,9 @@ import { test, expect, describe, beforeAll, afterAll, afterEach } from 'vitest'; import { createServer, type Server } from 'node:http'; -import { existsSync } from 'node:fs'; +import { existsSync, readdirSync } from 'node:fs'; import { readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { once } from 'node:events'; import zlib from 'node:zlib'; import { downloadBackupToFile } from '../src/lib/backupDownload.ts'; @@ -93,9 +93,17 @@ const manager = { const outPath = join(tmpdir(), `backup-download-test-${process.pid}.zip`); +// The partial file carries a random suffix, so scan for leftovers by prefix. +const partialLeftovers = () => + readdirSync(tmpdir()).filter((f) => + f.startsWith(`${basename(outPath)}.partial`), + ); + afterEach(async () => { await rm(outPath, { force: true }); - await rm(`${outPath}.partial`, { force: true }); + for (const f of partialLeftovers()) { + await rm(join(tmpdir(), f), { force: true }); + } }); describe('downloadBackupToFile', () => { @@ -110,7 +118,7 @@ describe('downloadBackupToFile', () => { expect(result.entities).toBe(2); expect(result.files).toBe(2); - expect(existsSync(`${outPath}.partial`)).toBe(false); + expect(partialLeftovers()).toEqual([]); const { ZipReader, Uint8ArrayReader, TextWriter } = await import( '@zip.js/zip.js' @@ -157,7 +165,7 @@ describe('downloadBackupToFile', () => { ).rejects.toThrow(/Failed to fetch config.json/); expect(existsSync(outPath)).toBe(false); - expect(existsSync(`${outPath}.partial`)).toBe(false); + expect(partialLeftovers()).toEqual([]); }); test('aborting removes the partial file', async () => { @@ -187,6 +195,6 @@ describe('downloadBackupToFile', () => { ).rejects.toThrow(); expect(existsSync(outPath)).toBe(false); - expect(existsSync(`${outPath}.partial`)).toBe(false); + expect(partialLeftovers()).toEqual([]); }); }); diff --git a/client/packages/cli/__tests__/backups.test.ts b/client/packages/cli/__tests__/backups.test.ts index cf58354d50..fde7c92f49 100644 --- a/client/packages/cli/__tests__/backups.test.ts +++ b/client/packages/cli/__tests__/backups.test.ts @@ -150,6 +150,24 @@ describe('backup download', () => { expect(logs.join('\n')).toContain('Saved 3 namespaces and 2 storage files'); }); + test('rejects a backup id combined with --latest', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('backup-1', { latest: true, out: outPath() }), { + yes: true, + }), + ).rejects.toThrow(/not both/); + expect(state.downloadCalls).toHaveLength(0); + }); + + test('rejects an output path that is a directory', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('backup-1', { out: tmpdir() }), { yes: true }), + ).rejects.toThrow(/is a directory/); + expect(state.downloadCalls).toHaveLength(0); + }); + test('errors on an unknown id', async () => { state.manager = buildManager([makeBackup()]); await expect( diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 9c7a7ecbf1..eb5bb1b237 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -1,4 +1,4 @@ -import { existsSync } from 'node:fs'; +import { existsSync, statSync } from 'node:fs'; import path from 'node:path'; import ansiEscapes from 'ansi-escapes'; import chalk from 'chalk'; @@ -23,7 +23,7 @@ import { } from '../../lib/backupDownload.ts'; import { promptOk, runUIEffect } from '../../lib/ui.ts'; import { UI } from '../../ui/index.ts'; -import { formatBackupDate } from './list.ts'; +import { formatBackupDate, stripControlChars } from './list.ts'; const pickBackup = ( backups: AppBackup[], @@ -31,6 +31,11 @@ const pickBackup = ( opts: { latest?: boolean }, ) => Effect.gen(function* () { + if (backupId && opts.latest) { + return yield* BadArgsError.make({ + message: 'Pass either a backup id or --latest, not both.', + }); + } if (backupId) { const found = backups.find((b) => b.id === backupId); if (!found) { @@ -62,7 +67,9 @@ const pickBackup = ( options: sorted.map((backup) => ({ label: formatBackupDate(backup.backupAt) + - (backup.description ? ` — ${backup.description}` : '') + + (backup.description + ? ` — ${stripControlChars(backup.description)}` + : '') + ` ${chalk.dim(`(${backup.id})`)}`, value: backup, })), @@ -180,6 +187,12 @@ export const backupDownloadCmd = Effect.fn(function* ( const outPath = path.resolve(opts.out ?? backupZipName(backup)); if (existsSync(outPath)) { + // Catch this before the download runs, not at the final rename. + if (statSync(outPath).isDirectory()) { + return yield* BadArgsError.make({ + message: `${outPath} is a directory.`, + }); + } const overwrite = yield* promptOk({ promptText: `${path.basename(outPath)} already exists. Overwrite?`, }); diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index 09562bb762..c575a8ae11 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -7,12 +7,18 @@ import { useBackupsManager } from '../../lib/backups.ts'; export const formatBackupDate = (date: Date) => `${date.toISOString().replace('T', ' ').slice(0, 16)} UTC`; +// Backup descriptions are user-controlled text headed for the terminal; +// strip control characters so a crafted value can't inject escape sequences. +export const stripControlChars = (s: string) => s.replace(/\p{Cc}/gu, ''); + export const renderBackup = (backup: AppBackup) => Effect.gen(function* () { yield* Effect.log(chalk.cyan(formatBackupDate(backup.backupAt))); yield* Effect.log(` ID: ${backup.id}`); if (backup.description) { - yield* Effect.log(` Description: ${backup.description}`); + yield* Effect.log( + ` Description: ${stripControlChars(backup.description)}`, + ); } if (backup.dbSize != null) { yield* Effect.log(` Database size: ${formatFileSize(backup.dbSize)}`); diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts index 286a1e3626..114264aded 100644 --- a/client/packages/cli/src/lib/backupDownload.ts +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -1,5 +1,6 @@ import { createWriteStream } from 'node:fs'; -import { rename, unlink } from 'node:fs/promises'; +import { open, rename, unlink } from 'node:fs/promises'; +import { randomBytes } from 'node:crypto'; import { once } from 'node:events'; import { get as httpGet, type IncomingMessage } from 'node:http'; import { get as httpsGet } from 'node:https'; @@ -106,8 +107,11 @@ export async function downloadBackupToFile(opts: { ); } - const partialPath = `${opts.outPath}.partial`; - const fileStream = createWriteStream(partialPath); + // Randomized so a stale partial or a concurrent download of the same + // backup can't collide; 'wx' turns any remaining collision into an error + // instead of silently truncating another run's file. + const partialPath = `${opts.outPath}.partial-${randomBytes(4).toString('hex')}`; + const fileStream = createWriteStream(partialPath, { flags: 'wx' }); const awaitFileClosed = async () => { if (!fileStream.closed) await once(fileStream, 'close'); }; @@ -122,6 +126,14 @@ export async function downloadBackupToFile(opts: { onProgress: opts.onProgress, }); await awaitFileClosed(); + // Flush to disk before the rename so a crash right after can't leave a + // complete-looking zip with unwritten tails. + const fh = await open(partialPath, 'r+'); + try { + await fh.sync(); + } finally { + await fh.close(); + } await rename(partialPath, opts.outPath); return result; } catch (e) { diff --git a/client/packages/cli/src/lib/backups.ts b/client/packages/cli/src/lib/backups.ts index 9327d5ccb4..a2dd3088fd 100644 --- a/client/packages/cli/src/lib/backups.ts +++ b/client/packages/cli/src/lib/backups.ts @@ -1,5 +1,6 @@ import { Effect } from 'effect'; import type { BackupsManager } from '@instantdb/platform'; +import { AuthToken } from '../context/authToken.ts'; import { CurrentApp } from '../context/currentApp.ts'; import { PlatformApiError } from '../context/platformApi.ts'; import { getAuthedPlatformApi } from './platformApi.ts'; @@ -11,11 +12,19 @@ export const useBackupsManager = ( Effect.gen(function* () { const api = yield* getAuthedPlatformApi; const { appId } = yield* CurrentApp; + const authToken = yield* AuthToken; + const source = yield* authToken.getSource; + // An admin token from the environment silently outranks a saved login, + // and the server error alone gives no hint which credential was used. + const hint = + source === 'admin' + ? ' (used the admin token from your environment; if it is stale, remove INSTANT_APP_ADMIN_TOKEN or run `instant-cli login`)' + : ''; return yield* Effect.tryPromise({ try: () => fun(api.backups(appId)), catch: (e) => new PlatformApiError({ - message: errorMessage ?? 'Error using backups api', + message: (errorMessage ?? 'Error using backups api') + hint, cause: e, }), }); From 783f7adaf2265a9259aa313fac20fbf5b8c0270a Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:38:37 -0700 Subject: [PATCH 09/18] Drop the auth-source hint from backups errors Worth doing across the CLI's commands rather than special-cased in backups; punting to a separate PR. --- client/packages/cli/src/lib/backups.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/client/packages/cli/src/lib/backups.ts b/client/packages/cli/src/lib/backups.ts index a2dd3088fd..9327d5ccb4 100644 --- a/client/packages/cli/src/lib/backups.ts +++ b/client/packages/cli/src/lib/backups.ts @@ -1,6 +1,5 @@ import { Effect } from 'effect'; import type { BackupsManager } from '@instantdb/platform'; -import { AuthToken } from '../context/authToken.ts'; import { CurrentApp } from '../context/currentApp.ts'; import { PlatformApiError } from '../context/platformApi.ts'; import { getAuthedPlatformApi } from './platformApi.ts'; @@ -12,19 +11,11 @@ export const useBackupsManager = ( Effect.gen(function* () { const api = yield* getAuthedPlatformApi; const { appId } = yield* CurrentApp; - const authToken = yield* AuthToken; - const source = yield* authToken.getSource; - // An admin token from the environment silently outranks a saved login, - // and the server error alone gives no hint which credential was used. - const hint = - source === 'admin' - ? ' (used the admin token from your environment; if it is stale, remove INSTANT_APP_ADMIN_TOKEN or run `instant-cli login`)' - : ''; return yield* Effect.tryPromise({ try: () => fun(api.backups(appId)), catch: (e) => new PlatformApiError({ - message: (errorMessage ?? 'Error using backups api') + hint, + message: errorMessage ?? 'Error using backups api', cause: e, }), }); From 4057e700744b916bae935e9b6d9c94fb54dab4fe Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:37:59 -0700 Subject: [PATCH 10/18] Render backup list as an aligned table --- client/packages/cli/__tests__/backups.test.ts | 10 +-- .../packages/cli/src/commands/backup/list.ts | 64 +++++++++++++------ 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/client/packages/cli/__tests__/backups.test.ts b/client/packages/cli/__tests__/backups.test.ts index fde7c92f49..697520c7fa 100644 --- a/client/packages/cli/__tests__/backups.test.ts +++ b/client/packages/cli/__tests__/backups.test.ts @@ -114,14 +114,14 @@ beforeEach(() => { }); describe('backup list', () => { - test('renders backups', async () => { + test('renders backups as a table', async () => { state.manager = buildManager([makeBackup()]); await run(backupListCmd({}), { yes: true }); const output = logs.join('\n'); - expect(output).toContain('2026-08-01 00:00 UTC'); - expect(output).toContain('ID: backup-1'); - expect(output).toContain('Description: Automated Daily Snapshot'); - expect(output).toContain('Expires: 2026-08-08 00:00 UTC'); + expect(output).toContain('BACKUP (UTC)'); + expect(output).toContain('2026-08-01 00:00'); + expect(output).toContain('backup-1'); + expect(output).toContain('Automated Daily Snapshot'); }); test('outputs JSON with --json', async () => { diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index c575a8ae11..2bb79f4d7c 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -11,25 +11,45 @@ export const formatBackupDate = (date: Date) => // strip control characters so a crafted value can't inject escape sequences. export const stripControlChars = (s: string) => s.replace(/\p{Cc}/gu, ''); -export const renderBackup = (backup: AppBackup) => +const formatExpiry = (expiresAt: Date): string => { + const hours = Math.round((expiresAt.getTime() - Date.now()) / 3_600_000); + if (hours <= 0) return 'expired'; + if (hours < 48) return `in ${hours}h`; + return `in ${Math.round(hours / 24)}d`; +}; + +// One aligned row per backup, newest first, the way backup CLIs +// conventionally render listings. Dates are UTC (noted in the header) and +// expiry is relative; `--json` carries the precise values. +const renderBackupsTable = (backups: AppBackup[]) => Effect.gen(function* () { - yield* Effect.log(chalk.cyan(formatBackupDate(backup.backupAt))); - yield* Effect.log(` ID: ${backup.id}`); - if (backup.description) { - yield* Effect.log( - ` Description: ${stripControlChars(backup.description)}`, - ); - } - if (backup.dbSize != null) { - yield* Effect.log(` Database size: ${formatFileSize(backup.dbSize)}`); - } - if (backup.filesSize != null) { - yield* Effect.log( - ` Storage files size: ${formatFileSize(backup.filesSize)}`, - ); - } - if (backup.expiresAt) { - yield* Effect.log(` Expires: ${formatBackupDate(backup.expiresAt)}`); + const header = [ + 'BACKUP (UTC)', + 'DB SIZE', + 'STORAGE', + 'EXPIRES', + 'ID', + 'DESCRIPTION', + ]; + const rows = backups.map((backup) => [ + formatBackupDate(backup.backupAt).replace(' UTC', ''), + backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', + backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', + backup.expiresAt ? formatExpiry(backup.expiresAt) : '-', + backup.id, + backup.description ? stripControlChars(backup.description) : '', + ]); + const widths = header.map((h, i) => + Math.max(h.length, ...rows.map((row) => row[i].length)), + ); + const line = (cells: string[]) => + cells + .map((cell, i) => cell.padEnd(widths[i])) + .join(' ') + .trimEnd(); + yield* Effect.log(chalk.dim(line(header))); + for (const row of rows) { + yield* Effect.log(line(row)); } }); @@ -51,7 +71,9 @@ export const backupListCmd = Effect.fn(function* ( return; } - for (const backup of backups) { - yield* renderBackup(backup); - } + // The server returns newest first; sort anyway so the table can't lie. + const sorted = [...backups].sort( + (a, b) => b.backupAt.getTime() - a.backupAt.getTime(), + ); + yield* renderBackupsTable(sorted); }); From ff99512b5d71670b2e1476b3998cbda8a50c8d03 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:42:36 -0700 Subject: [PATCH 11/18] Style the backup table like pscale: id first, relative times --- client/packages/cli/__tests__/backups.test.ts | 5 ++- .../packages/cli/src/commands/backup/list.ts | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/client/packages/cli/__tests__/backups.test.ts b/client/packages/cli/__tests__/backups.test.ts index 697520c7fa..333508fb4b 100644 --- a/client/packages/cli/__tests__/backups.test.ts +++ b/client/packages/cli/__tests__/backups.test.ts @@ -118,10 +118,11 @@ describe('backup list', () => { state.manager = buildManager([makeBackup()]); await run(backupListCmd({}), { yes: true }); const output = logs.join('\n'); - expect(output).toContain('BACKUP (UTC)'); - expect(output).toContain('2026-08-01 00:00'); + expect(output).toContain('CREATED AT'); + expect(output).toContain('EXPIRES AT'); expect(output).toContain('backup-1'); expect(output).toContain('Automated Daily Snapshot'); + expect(output).toMatch(/\d+ (minutes?|hours?|days?) ago/); }); test('outputs JSON with --json', async () => { diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index 2bb79f4d7c..a45d994564 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -11,32 +11,46 @@ export const formatBackupDate = (date: Date) => // strip control characters so a crafted value can't inject escape sequences. export const stripControlChars = (s: string) => s.replace(/\p{Cc}/gu, ''); -const formatExpiry = (expiresAt: Date): string => { - const hours = Math.round((expiresAt.getTime() - Date.now()) / 3_600_000); - if (hours <= 0) return 'expired'; - if (hours < 48) return `in ${hours}h`; - return `in ${Math.round(hours / 24)}d`; +// Relative times in both directions: "3 hours ago", "6 days from now". +const relativeTime = (date: Date): string => { + const diffMs = date.getTime() - Date.now(); + const abs = Math.abs(diffMs); + if (abs < 60_000) return diffMs <= 0 ? 'just now' : 'now'; + const minutes = Math.round(abs / 60_000); + const hours = Math.round(abs / 3_600_000); + const days = Math.round(abs / 86_400_000); + const [count, unit] = + minutes < 60 + ? [minutes, 'minute'] + : hours < 24 + ? [hours, 'hour'] + : [days, 'day']; + const label = `${count} ${unit}${count === 1 ? '' : 's'}`; + return diffMs < 0 ? `${label} ago` : `${label} from now`; }; -// One aligned row per backup, newest first, the way backup CLIs -// conventionally render listings. Dates are UTC (noted in the header) and -// expiry is relative; `--json` carries the precise values. +// One aligned row per backup, newest first, in the style of pscale's backup +// listing: id first, relative times. `--json` carries the precise values. const renderBackupsTable = (backups: AppBackup[]) => Effect.gen(function* () { const header = [ - 'BACKUP (UTC)', + 'ID', + 'CREATED AT', 'DB SIZE', 'STORAGE', - 'EXPIRES', - 'ID', + 'EXPIRES AT', 'DESCRIPTION', ]; const rows = backups.map((backup) => [ - formatBackupDate(backup.backupAt).replace(' UTC', ''), + backup.id, + relativeTime(backup.backupAt), backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', - backup.expiresAt ? formatExpiry(backup.expiresAt) : '-', - backup.id, + backup.expiresAt + ? backup.expiresAt.getTime() <= Date.now() + ? 'expired' + : relativeTime(backup.expiresAt) + : '-', backup.description ? stripControlChars(backup.description) : '', ]); const widths = header.map((h, i) => @@ -48,6 +62,7 @@ const renderBackupsTable = (backups: AppBackup[]) => .join(' ') .trimEnd(); yield* Effect.log(chalk.dim(line(header))); + yield* Effect.log(chalk.dim(widths.map((w) => '-'.repeat(w)).join(' '))); for (const row of rows) { yield* Effect.log(line(row)); } From 95dfdce75f5fb771511124265a26a22e482b1dea Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 14:52:45 -0700 Subject: [PATCH 12/18] Confirm downloads with the backup table and reuse the UI spinner --- .../cli/src/commands/backup/download.ts | 156 +++++++++--------- .../packages/cli/src/commands/backup/list.ts | 6 +- client/packages/cli/src/ui/lib.ts | 2 +- 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index eb5bb1b237..3e651bf4bb 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -1,9 +1,7 @@ import { existsSync, statSync } from 'node:fs'; import path from 'node:path'; -import ansiEscapes from 'ansi-escapes'; import chalk from 'chalk'; import { Effect } from 'effect'; -import throttle from 'lodash.throttle'; import { backupZipName, estimateZipSize, @@ -22,8 +20,13 @@ import { type BackupDownloadResult, } from '../../lib/backupDownload.ts'; import { promptOk, runUIEffect } from '../../lib/ui.ts'; +import { onTerminate, renderUnwrap } from '../../ui/lib.ts'; import { UI } from '../../ui/index.ts'; -import { formatBackupDate, stripControlChars } from './list.ts'; +import { + formatBackupDate, + renderBackupsTable, + stripControlChars, +} from './list.ts'; const pickBackup = ( backups: AppBackup[], @@ -78,59 +81,45 @@ const pickBackup = ( ); }); -// Single-line progress on a TTY, nothing otherwise. -function makeProgressRenderer() { - const stream = process.stderr; - if (!stream.isTTY) { - return { update: (_p: BackupDownloadProgress) => {}, done: () => {} }; - } - let wrote = false; - const write = (p: BackupDownloadProgress) => { - const parts: string[] = []; +// One compact line for the spinner as the pipeline ticks. +function progressLine(p: BackupDownloadProgress): string { + const parts: string[] = []; + parts.push( + p.entitiesTotal == null + ? 'listing namespaces…' + : `namespaces ${p.entitiesCompleted}/${p.entitiesTotal}`, + ); + if (p.filesTotal !== 0) { parts.push( - p.entitiesTotal == null - ? 'listing namespaces…' - : `namespaces ${p.entitiesCompleted}/${p.entitiesTotal}`, + p.filesTotal == null + ? 'listing storage files…' + : `storage files ${p.filesCompleted}/${p.filesTotal}`, ); - if (p.filesTotal !== 0) { - parts.push( - p.filesTotal == null - ? 'listing storage files…' - : `storage files ${p.filesCompleted}/${p.filesTotal}`, - ); - } - let bytes = formatFileSize(p.zipBytes); - if (p.bytesTotal != null && p.bytesTotal > 0) { - const pct = Math.min(100, Math.round((p.bytesRead / p.bytesTotal) * 100)); - bytes += ` (${pct}%)`; - } - parts.push(bytes); - const currentEntry = p.currentEntity || p.currentFile; - if (currentEntry) { - parts.push(currentEntry); - } - let line = parts.join(' · '); - const width = stream.columns || 80; - if (line.length >= width) { - line = line.slice(0, Math.max(0, width - 2)) + '…'; - } - stream.write(ansiEscapes.eraseLine + ansiEscapes.cursorLeft + line); - wrote = true; - }; - const throttled = throttle(write, 100); - return { - update: throttled, - done: () => { - throttled.cancel(); - if (wrote) { - stream.write(ansiEscapes.eraseLine + ansiEscapes.cursorLeft); - } - }, - }; + } + let bytes = formatFileSize(p.zipBytes); + if (p.bytesTotal != null && p.bytesTotal > 0) { + const pct = Math.min(100, Math.round((p.bytesRead / p.bytesTotal) * 100)); + bytes += ` (${pct}%)`; + } + parts.push(bytes); + const currentEntry = p.currentEntity || p.currentFile; + if (currentEntry) { + parts.push(currentEntry); + } + let line = parts.join(' · '); + // The spinner prefixes a frame glyph; truncate so the line can't wrap. + const width = (process.stdout.columns || 80) - 4; + if (line.length > width) { + line = line.slice(0, Math.max(0, width - 1)) + '…'; + } + return line; } -// Returns null when the download was cancelled (ctrl-c). A second ctrl-c -// falls through to Node's default handler and kills the process outright. +// Returns null when the download was cancelled. While the spinner is +// attached the terminal is raw, so ctrl-c arrives through the UI's +// terminate hook rather than SIGINT; both routes abort the same controller +// and the pipeline removes its partial file before we return. Outside the +// spinner (non-TTY runs), SIGINT covers it. async function runDownload( manager: BackupsManager, backup: AppBackup, @@ -139,23 +128,38 @@ async function runDownload( const controller = new AbortController(); const onSigint = () => controller.abort(); process.once('SIGINT', onSigint); - const progress = makeProgressRenderer(); + onTerminate(() => controller.abort()); try { - return await downloadBackupToFile({ + let spinner: UI.Spinner | null = null; + // Settled into a sentinel so the spinner always disappears cleanly and + // cancellation/error output stays with the command below. + const settled = downloadBackupToFile({ manager, backup, outPath, signal: controller.signal, - onProgress: progress.update, - }); - } catch (e) { - if ((e as { name?: string })?.name === 'AbortError') { - return null; + onProgress: (p) => spinner?.updateText(progressLine(p)), + }).then( + (result) => ({ result, error: null as unknown }), + (error: unknown) => ({ result: null, error }), + ); + if (process.stdout.isTTY) { + spinner = new UI.Spinner({ + promise: settled, + workingText: 'Preparing download…', + disappearWhenDone: true, + }); + await renderUnwrap(spinner); + } + const { result, error } = await settled; + if (error) { + if ((error as { name?: string })?.name === 'AbortError') return null; + throw error; } - throw e; + return result; } finally { process.removeListener('SIGINT', onSigint); - progress.done(); + onTerminate(undefined); } } @@ -175,24 +179,27 @@ export const backupDownloadCmd = Effect.fn(function* ( const backup = yield* pickBackup(backups, backupId, opts); + const outPath = path.resolve(opts.out ?? backupZipName(backup)); + // Catch this before the download runs, not at the final rename. + if (existsSync(outPath) && statSync(outPath).isDirectory()) { + return yield* BadArgsError.make({ + message: `${outPath} is a directory.`, + }); + } + + yield* renderBackupsTable([backup]); const sizes = estimateZipSize(backup); - const estimate = sizes - ? ` The zip file will be between ${formatFileSize(sizes.min)} and ${formatFileSize(sizes.max)}, depending on the compression ratio.` - : ''; + if (sizes) { + yield* Effect.log( + `The zip file will be between ${formatFileSize(sizes.min)} and ${formatFileSize(sizes.max)}, depending on the compression ratio.`, + ); + } + yield* Effect.log(chalk.dim(`Saving to ${outPath} (pass -o to change).`)); - const ok = yield* promptOk({ - promptText: `Download the backup from ${formatBackupDate(backup.backupAt)}?${estimate}`, - }); + const ok = yield* promptOk({ promptText: 'Download this backup?' }); if (!ok) return; - const outPath = path.resolve(opts.out ?? backupZipName(backup)); if (existsSync(outPath)) { - // Catch this before the download runs, not at the final rename. - if (statSync(outPath).isDirectory()) { - return yield* BadArgsError.make({ - message: `${outPath} is a directory.`, - }); - } const overwrite = yield* promptOk({ promptText: `${path.basename(outPath)} already exists. Overwrite?`, }); @@ -200,7 +207,6 @@ export const backupDownloadCmd = Effect.fn(function* ( } const manager = yield* buildBackupsManager; - yield* Effect.log(`Downloading to ${outPath}`); const result = yield* Effect.tryPromise({ try: () => runDownload(manager, backup, outPath), diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index a45d994564..8f65d7d836 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -29,9 +29,9 @@ const relativeTime = (date: Date): string => { return diffMs < 0 ? `${label} ago` : `${label} from now`; }; -// One aligned row per backup, newest first, in the style of pscale's backup -// listing: id first, relative times. `--json` carries the precise values. -const renderBackupsTable = (backups: AppBackup[]) => +// One aligned row per backup, newest first: id first, relative times. +// `--json` carries the precise values. +export const renderBackupsTable = (backups: AppBackup[]) => Effect.gen(function* () { const header = [ 'ID', diff --git a/client/packages/cli/src/ui/lib.ts b/client/packages/cli/src/ui/lib.ts index 5610a6a8bc..f92a65e8bc 100644 --- a/client/packages/cli/src/ui/lib.ts +++ b/client/packages/cli/src/ui/lib.ts @@ -372,7 +372,7 @@ let terminateHandler: | undefined; export function onTerminate( - callback: (stdin: ReadStream, stdout: WriteStream) => void | undefined, + callback: ((stdin: ReadStream, stdout: WriteStream) => void) | undefined, ) { terminateHandler = callback; } From cbe35d7a7947161e71c84e04a2ad8413e84d9dbb Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:00:10 -0700 Subject: [PATCH 13/18] Align the backup picker options like the table --- .../cli/src/commands/backup/download.ts | 25 +++++++++++-------- .../packages/cli/src/commands/backup/list.ts | 2 +- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 3e651bf4bb..2442117557 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -22,11 +22,7 @@ import { import { promptOk, runUIEffect } from '../../lib/ui.ts'; import { onTerminate, renderUnwrap } from '../../ui/lib.ts'; import { UI } from '../../ui/index.ts'; -import { - formatBackupDate, - renderBackupsTable, - stripControlChars, -} from './list.ts'; +import { relativeTime, renderBackupsTable, stripControlChars } from './list.ts'; const pickBackup = ( backups: AppBackup[], @@ -65,14 +61,23 @@ const pickBackup = ( }); } + // Aligned like the `backup list` table so the options scan as columns: + // created, sizes, expiry, description, id. + const cells = sorted.map((backup) => [ + relativeTime(backup.backupAt), + backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', + backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', + backup.expiresAt ? `expires ${relativeTime(backup.expiresAt)}` : '', + backup.description ? stripControlChars(backup.description) : '', + ]); + const widths = cells[0].map((_, i) => + Math.max(...cells.map((row) => row[i].length)), + ); return yield* runUIEffect( new UI.Select({ - options: sorted.map((backup) => ({ + options: sorted.map((backup, idx) => ({ label: - formatBackupDate(backup.backupAt) + - (backup.description - ? ` — ${stripControlChars(backup.description)}` - : '') + + cells[idx].map((cell, i) => cell.padEnd(widths[i])).join(' ') + ` ${chalk.dim(`(${backup.id})`)}`, value: backup, })), diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts index 8f65d7d836..d7a8893bb0 100644 --- a/client/packages/cli/src/commands/backup/list.ts +++ b/client/packages/cli/src/commands/backup/list.ts @@ -12,7 +12,7 @@ export const formatBackupDate = (date: Date) => export const stripControlChars = (s: string) => s.replace(/\p{Cc}/gu, ''); // Relative times in both directions: "3 hours ago", "6 days from now". -const relativeTime = (date: Date): string => { +export const relativeTime = (date: Date): string => { const diffMs = date.getTime() - Date.now(); const abs = Math.abs(diffMs); if (abs < 60_000) return diffMs <= 0 ? 'just now' : 'now'; From 8a97ccdf27c69aca58919a89afc2d9146da9cc87 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:01:38 -0700 Subject: [PATCH 14/18] Put the picker columns in table order --- client/packages/cli/src/commands/backup/download.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 2442117557..c3d61fa755 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -61,9 +61,10 @@ const pickBackup = ( }); } - // Aligned like the `backup list` table so the options scan as columns: - // created, sizes, expiry, description, id. + // Aligned columns in the same order as the `backup list` table: id, + // created, sizes, expiry, description. const cells = sorted.map((backup) => [ + backup.id, relativeTime(backup.backupAt), backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', @@ -76,9 +77,7 @@ const pickBackup = ( return yield* runUIEffect( new UI.Select({ options: sorted.map((backup, idx) => ({ - label: - cells[idx].map((cell, i) => cell.padEnd(widths[i])).join(' ') + - ` ${chalk.dim(`(${backup.id})`)}`, + label: cells[idx].map((cell, i) => cell.padEnd(widths[i])).join(' '), value: backup, })), promptText: 'Select a backup to download:', From 57a6dfc1c040017e46202556967c30e756fbd287 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:02:35 -0700 Subject: [PATCH 15/18] Label the picker's created time --- client/packages/cli/src/commands/backup/download.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index c3d61fa755..3825825fad 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -65,7 +65,7 @@ const pickBackup = ( // created, sizes, expiry, description. const cells = sorted.map((backup) => [ backup.id, - relativeTime(backup.backupAt), + `created ${relativeTime(backup.backupAt)}`, backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', backup.expiresAt ? `expires ${relativeTime(backup.expiresAt)}` : '', From 5f463101a46a613f4c7758d16bc8f9318009da48 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:03:49 -0700 Subject: [PATCH 16/18] Keep the backup picker lean --- client/packages/cli/src/commands/backup/download.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts index 3825825fad..1ea5a624b7 100644 --- a/client/packages/cli/src/commands/backup/download.ts +++ b/client/packages/cli/src/commands/backup/download.ts @@ -61,14 +61,12 @@ const pickBackup = ( }); } - // Aligned columns in the same order as the `backup list` table: id, - // created, sizes, expiry, description. + // The picker stays lean: id, age, and description are what you choose + // by, in the table's column order. Sizes and expiry show up in the + // confirm table right after, where headers explain them. const cells = sorted.map((backup) => [ backup.id, `created ${relativeTime(backup.backupAt)}`, - backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', - backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', - backup.expiresAt ? `expires ${relativeTime(backup.expiresAt)}` : '', backup.description ? stripControlChars(backup.description) : '', ]); const widths = cells[0].map((_, i) => From 0a93755cf8d3805f07fb8d22efa60968b74fe365 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:08:45 -0700 Subject: [PATCH 17/18] Make BackupsManager.downloadArchive the only pipeline entry point --- .../cli/__tests__/backupDownload.test.ts | 6 ++++++ client/packages/cli/src/lib/backupDownload.ts | 19 +++++++++---------- client/packages/platform/src/index.ts | 1 - 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/client/packages/cli/__tests__/backupDownload.test.ts b/client/packages/cli/__tests__/backupDownload.test.ts index 0504920ad6..71f9a2c751 100644 --- a/client/packages/cli/__tests__/backupDownload.test.ts +++ b/client/packages/cli/__tests__/backupDownload.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os'; import { basename, join } from 'node:path'; import { once } from 'node:events'; import zlib from 'node:zlib'; +import { BackupsManager } from '@instantdb/platform'; import { downloadBackupToFile } from '../src/lib/backupDownload.ts'; // Exercises the real pipeline end-to-end against a local HTTP server: zstd @@ -89,6 +90,11 @@ const manager = { yield { locationId: f.locationId, path: f.path, url: f.url() }; } }, + // Borrow the real method so the test drives the production pipeline + // against this fake's endpoints. + downloadArchive(opts: unknown) { + return (BackupsManager.prototype.downloadArchive as any).call(this, opts); + }, } as any; const outPath = join(tmpdir(), `backup-download-test-${process.pid}.zip`); diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts index 114264aded..ffaefa6aca 100644 --- a/client/packages/cli/src/lib/backupDownload.ts +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -6,13 +6,12 @@ import { get as httpGet, type IncomingMessage } from 'node:http'; import { get as httpsGet } from 'node:https'; import { Readable, Writable, type Duplex } from 'node:stream'; import zlib from 'node:zlib'; -import { - downloadBackupArchive, - type AppBackup, - type BackupArchiveWriter, - type BackupDownloadProgress, - type BackupDownloadResult, - type BackupsManager, +import type { + AppBackup, + BackupArchiveWriter, + BackupDownloadProgress, + BackupDownloadResult, + BackupsManager, } from '@instantdb/platform'; export type { @@ -84,7 +83,8 @@ async function createZipWriter( /** * Downloads a backup into a zip file at `outPath` via the shared - * `downloadBackupArchive` pipeline, supplying the Node-specific pieces: + * `BackupsManager.downloadArchive` pipeline, supplying the Node-specific + * pieces: * presigned URLs are fetched with node:http(s) and decompressed explicitly, * and the archive streams to disk with backpressure so memory stays flat * regardless of backup size. @@ -116,8 +116,7 @@ export async function downloadBackupToFile(opts: { if (!fileStream.closed) await once(fileStream, 'close'); }; try { - const result = await downloadBackupArchive({ - manager: opts.manager, + const result = await opts.manager.downloadArchive({ backup: opts.backup, fetchBody, sink: Writable.toWeb(fileStream) as WritableStream, diff --git a/client/packages/platform/src/index.ts b/client/packages/platform/src/index.ts index 4a112fbc68..34143e8b52 100644 --- a/client/packages/platform/src/index.ts +++ b/client/packages/platform/src/index.ts @@ -127,7 +127,6 @@ export { } from './backups.ts'; export { - downloadBackupArchive, type DownloadBackupArchiveOpts, type BackupArchiveWriter, type BackupDownloadProgress, From fe6fd8fb4ad79cc9f77945cd175d536827cc68c4 Mon Sep 17 00:00:00 2001 From: stopachka Date: Wed, 5 Aug 2026 15:38:59 -0700 Subject: [PATCH 18/18] Bump version to v1.0.61 --- client/packages/version/src/version.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/packages/version/src/version.ts b/client/packages/version/src/version.ts index 063bd17103..348189d998 100644 --- a/client/packages/version/src/version.ts +++ b/client/packages/version/src/version.ts @@ -2,6 +2,6 @@ // Update the version here and merge your code to main to // publish a new version of all of the packages to npm. -const version = 'v1.0.60'; +const version = 'v1.0.61'; export { version };