diff --git a/client/packages/cli/__tests__/backupDownload.test.ts b/client/packages/cli/__tests__/backupDownload.test.ts new file mode 100644 index 0000000000..71f9a2c751 --- /dev/null +++ b/client/packages/cli/__tests__/backupDownload.test.ts @@ -0,0 +1,206 @@ +import { test, expect, describe, beforeAll, afterAll, afterEach } from 'vitest'; +import { createServer, type Server } from 'node:http'; +import { existsSync, readdirSync } from 'node:fs'; +import { readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { once } from 'node:events'; +import zlib from 'node:zlib'; +import { BackupsManager } from '@instantdb/platform'; +import { downloadBackupToFile } from '../src/lib/backupDownload.ts'; + +// Exercises the real pipeline end-to-end against a local HTTP server: zstd +// decompression of entity shards, canonical entry order (config.json, then +// entities/*.jsonl, then files/), and the partial-file rename. + +// Not yet in this @types/node version, same as createZstdDecompress in the +// pipeline itself. +const zstd = (s: string): Buffer => + (zlib as any).zstdCompressSync(Buffer.from(s)); + +const bodies: Record = { + '/config.json': { body: zstd('{"schema":{}}'), encoding: 'zstd' }, + '/entities/todos.jsonl': { + body: zstd('{"entity":{"id":"1"}}\n'), + encoding: 'zstd', + }, + '/entities/$files.jsonl': { + body: zstd('{"entity":{"location-id":"loc-1"}}\n'), + encoding: 'zstd', + }, + '/blobs/loc-1': { body: Buffer.from('blob-one') }, + '/blobs/loc-2': { body: Buffer.from('blob-two') }, +}; + +let server: Server; +let baseUrl: string; + +beforeAll(async () => { + server = createServer((req, res) => { + const found = bodies[req.url ?? '']; + if (!found) { + res.writeHead(404).end(); + return; + } + const headers: Record = {}; + if (found.encoding) headers['content-encoding'] = found.encoding; + res.writeHead(200, headers).end(found.body); + }); + server.listen(0); + await once(server, 'listening'); + const address = server.address(); + if (typeof address === 'string' || address === null) { + throw new Error('Expected a TCP address'); + } + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterAll(() => { + server.close(); +}); + +const backup = { + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 16, + dbSize: 100, + uncompressedSize: 40, + description: null, + expiresAt: new Date('2026-08-08T00:00:00Z'), +}; + +const storageFiles = [ + { locationId: 'loc-1', path: 'a.png', url: () => `${baseUrl}/blobs/loc-1` }, + { locationId: 'loc-2', path: 'b.png', url: () => `${baseUrl}/blobs/loc-2` }, +]; + +const manager = { + listFiles: async (_backupId: string) => [ + { name: 'config.json', size: 10 }, + { name: 'entities/todos.jsonl', size: 10 }, + { name: 'entities/$files.jsonl', size: 10 }, + ], + getFileUrl: async (_backupId: string, name: string) => `${baseUrl}/${name}`, + streamStorageFiles: async function* ( + _backupId: string, + _opts?: { signal?: AbortSignal }, + ) { + for (const f of storageFiles) { + yield { locationId: f.locationId, path: f.path, url: f.url() }; + } + }, + // Borrow the real method so the test drives the production pipeline + // against this fake's endpoints. + downloadArchive(opts: unknown) { + return (BackupsManager.prototype.downloadArchive as any).call(this, opts); + }, +} as any; + +const outPath = join(tmpdir(), `backup-download-test-${process.pid}.zip`); + +// The partial file carries a random suffix, so scan for leftovers by prefix. +const partialLeftovers = () => + readdirSync(tmpdir()).filter((f) => + f.startsWith(`${basename(outPath)}.partial`), + ); + +afterEach(async () => { + await rm(outPath, { force: true }); + for (const f of partialLeftovers()) { + await rm(join(tmpdir(), f), { force: true }); + } +}); + +describe('downloadBackupToFile', () => { + test('writes a zip with the canonical entry order', async () => { + const result = await downloadBackupToFile({ + manager, + backup, + outPath, + signal: new AbortController().signal, + onProgress: () => {}, + }); + + expect(result.entities).toBe(2); + expect(result.files).toBe(2); + expect(partialLeftovers()).toEqual([]); + + const { ZipReader, Uint8ArrayReader, TextWriter } = await import( + '@zip.js/zip.js' + ); + const reader = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await readFile(outPath))), + ); + const entries = await reader.getEntries(); + + // Entry order is the restore contract: config first, all entity shards + // before any storage blob. + expect(entries.map((e) => e.filename)).toEqual([ + 'config.json', + 'entities/todos.jsonl', + 'entities/$files.jsonl', + 'files/loc-1', + 'files/loc-2', + ]); + + // Entity shards land decompressed; blobs land verbatim. + const readText = (entry: any) => entry.getData(new TextWriter()); + expect(await readText(entries[0])).toBe('{"schema":{}}'); + expect(await readText(entries[1])).toBe('{"entity":{"id":"1"}}\n'); + expect(await readText(entries[3])).toBe('blob-one'); + expect(await readText(entries[4])).toBe('blob-two'); + await reader.close(); + }); + + test('removes the partial file when a fetch fails', async () => { + const failingManager = { + ...manager, + getFileUrl: async (_backupId: string, name: string) => + `${baseUrl}/missing-${name}`, + }; + + await expect( + downloadBackupToFile({ + manager: failingManager, + backup, + outPath, + signal: new AbortController().signal, + onProgress: () => {}, + }), + ).rejects.toThrow(/Failed to fetch config.json/); + + expect(existsSync(outPath)).toBe(false); + expect(partialLeftovers()).toEqual([]); + }); + + test('aborting removes the partial file', async () => { + const controller = new AbortController(); + const slowManager = { + ...manager, + streamStorageFiles: async function* () { + yield { + locationId: 'loc-1', + path: 'a.png', + url: `${baseUrl}/blobs/loc-1`, + }; + controller.abort(); + // Give the pipeline a moment to observe the abort mid-drain. + await new Promise((resolve) => setTimeout(resolve, 20)); + }, + }; + + await expect( + downloadBackupToFile({ + manager: slowManager, + backup, + outPath, + signal: controller.signal, + onProgress: () => {}, + }), + ).rejects.toThrow(); + + expect(existsSync(outPath)).toBe(false); + expect(partialLeftovers()).toEqual([]); + }); +}); diff --git a/client/packages/cli/__tests__/backups.test.ts b/client/packages/cli/__tests__/backups.test.ts new file mode 100644 index 0000000000..333508fb4b --- /dev/null +++ b/client/packages/cli/__tests__/backups.test.ts @@ -0,0 +1,221 @@ +import { test, expect, describe, vi, beforeEach } from 'vitest'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { Effect, Layer, Logger } from 'effect'; +import { GlobalOpts } from '../src/context/globalOpts.ts'; +import { AuthToken } from '../src/context/authToken.ts'; +import { CurrentApp } from '../src/context/currentApp.ts'; + +vi.mock('../src/index.ts', () => ({})); + +const state = vi.hoisted(() => ({ + manager: undefined as any, + downloadResult: undefined as any, + downloadError: undefined as Error | undefined, + downloadCalls: [] as any[], + promptResponses: [] as unknown[], +})); + +// Mock at the SDK boundary: any `new InstantPlatformApi(...)` returns a stub +// whose `.backups(appId)` is the per-test fake manager. +vi.mock('@instantdb/platform', async (importOriginal) => { + const orig: any = await importOriginal(); + return { + ...orig, + PlatformApi: class { + backups(_appId: string) { + return state.manager; + } + }, + }; +}); + +// The download pipeline is network- and disk-heavy; the command tests only +// care that it's invoked with the right backup and destination. +vi.mock('../src/lib/backupDownload.ts', () => ({ + downloadBackupToFile: vi.fn(async (opts: any) => { + state.downloadCalls.push(opts); + if (state.downloadError) throw state.downloadError; + return state.downloadResult; + }), +})); + +vi.mock('../src/ui/lib.ts', async (importOriginal) => { + const orig: any = await importOriginal(); + return { + ...orig, + renderUnwrap: () => { + if (state.promptResponses.length === 0) { + return Promise.reject(new Error('No prompt response queued')); + } + return Promise.resolve(state.promptResponses.shift()); + }, + }; +}); + +const { backupListCmd } = await import('../src/commands/backup/list.ts'); +const { backupDownloadCmd } = await import( + '../src/commands/backup/download.ts' +); + +let logs: string[] = []; + +const makeBackup = (overrides: any = {}) => ({ + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 1000, + dbSize: 2000, + uncompressedSize: 3000, + description: 'Automated Daily Snapshot', + expiresAt: new Date('2026-08-08T00:00:00Z'), + ...overrides, +}); + +const buildManager = (backups: any[]) => ({ + list: vi.fn(async () => backups), +}); + +const outPath = () => join(tmpdir(), `backup-test-${Date.now()}.zip`); + +const run = (effect: any, opts: { yes: boolean }) => + Effect.runPromise( + effect.pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(GlobalOpts, { yes: opts.yes }), + Layer.succeed(AuthToken, { + getAuthToken: Effect.succeed('test-token'), + getSource: Effect.succeed('env' as const), + setAuthToken: () => Effect.succeed(undefined), + }), + Layer.succeed(CurrentApp, { + appId: 'test-app', + source: 'env' as const, + }), + Logger.replace( + Logger.defaultLogger, + Logger.make(({ message }) => { + logs.push(String(message)); + }), + ), + ), + ), + ), + ); + +beforeEach(() => { + logs = []; + state.manager = buildManager([]); + state.downloadResult = { entities: 3, files: 2, zipBytes: 1234 }; + state.downloadError = undefined; + state.downloadCalls = []; + state.promptResponses = []; +}); + +describe('backup list', () => { + test('renders backups as a table', async () => { + state.manager = buildManager([makeBackup()]); + await run(backupListCmd({}), { yes: true }); + const output = logs.join('\n'); + expect(output).toContain('CREATED AT'); + expect(output).toContain('EXPIRES AT'); + expect(output).toContain('backup-1'); + expect(output).toContain('Automated Daily Snapshot'); + expect(output).toMatch(/\d+ (minutes?|hours?|days?) ago/); + }); + + test('outputs JSON with --json', async () => { + state.manager = buildManager([makeBackup()]); + await run(backupListCmd({ json: true }), { yes: true }); + const parsed = JSON.parse(logs.join('\n')); + expect(parsed).toHaveLength(1); + expect(parsed[0].id).toBe('backup-1'); + }); + + test('handles no backups', async () => { + await run(backupListCmd({}), { yes: true }); + expect(logs.join('\n')).toContain('No backups yet.'); + }); +}); + +describe('backup download', () => { + test('downloads by id', async () => { + const backup = makeBackup(); + state.manager = buildManager([backup]); + const out = outPath(); + await run(backupDownloadCmd('backup-1', { out }), { yes: true }); + expect(state.downloadCalls).toHaveLength(1); + expect(state.downloadCalls[0].backup).toEqual(backup); + expect(state.downloadCalls[0].outPath).toBe(out); + expect(logs.join('\n')).toContain('Saved 3 namespaces and 2 storage files'); + }); + + test('rejects a backup id combined with --latest', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('backup-1', { latest: true, out: outPath() }), { + yes: true, + }), + ).rejects.toThrow(/not both/); + expect(state.downloadCalls).toHaveLength(0); + }); + + test('rejects an output path that is a directory', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('backup-1', { out: tmpdir() }), { yes: true }), + ).rejects.toThrow(/is a directory/); + expect(state.downloadCalls).toHaveLength(0); + }); + + test('errors on an unknown id', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd('nope', { out: outPath() }), { yes: true }), + ).rejects.toThrow(/No backup found with id nope/); + }); + + test('--latest picks the newest backup', async () => { + const older = makeBackup(); + const newer = makeBackup({ + id: 'backup-2', + backupAt: new Date('2026-08-02T00:00:00Z'), + }); + state.manager = buildManager([older, newer]); + await run(backupDownloadCmd(undefined, { latest: true, out: outPath() }), { + yes: true, + }); + expect(state.downloadCalls[0].backup.id).toBe('backup-2'); + }); + + test('requires an id or --latest when prompts are skipped', async () => { + state.manager = buildManager([makeBackup()]); + await expect( + run(backupDownloadCmd(undefined, {}), { yes: true }), + ).rejects.toThrow(/Must specify a backup id or --latest/); + }); + + test('prompts for a backup and confirmation interactively', async () => { + const backup = makeBackup(); + state.manager = buildManager([backup]); + // First the backup picker, then the download confirmation. + state.promptResponses = [backup, true]; + await run(backupDownloadCmd(undefined, { out: outPath() }), { + yes: false, + }); + expect(state.downloadCalls).toHaveLength(1); + expect(state.downloadCalls[0].backup).toEqual(backup); + }); + + test('reports a cancelled download', async () => { + state.manager = buildManager([makeBackup()]); + state.downloadError = Object.assign(new Error('aborted'), { + name: 'AbortError', + }); + await run(backupDownloadCmd('backup-1', { out: outPath() }), { + yes: true, + }); + expect(logs.join('\n')).toContain('Download cancelled.'); + }); +}); diff --git a/client/packages/cli/package.json b/client/packages/cli/package.json index 601eea07e9..a2936192c7 100644 --- a/client/packages/cli/package.json +++ b/client/packages/cli/package.json @@ -34,6 +34,7 @@ "@instantdb/core": "workspace:*", "@instantdb/platform": "workspace:*", "@instantdb/version": "workspace:*", + "@zip.js/zip.js": "^2.8.34", "acorn": "^8.15.0", "acorn-typescript": "^1.4.13", "ansi-escapes": "4.3.2", diff --git a/client/packages/cli/src/commands/backup/download.ts b/client/packages/cli/src/commands/backup/download.ts new file mode 100644 index 0000000000..1ea5a624b7 --- /dev/null +++ b/client/packages/cli/src/commands/backup/download.ts @@ -0,0 +1,231 @@ +import { existsSync, statSync } from 'node:fs'; +import path from 'node:path'; +import chalk from 'chalk'; +import { Effect } from 'effect'; +import { + backupZipName, + estimateZipSize, + formatFileSize, + type AppBackup, + type BackupsManager, +} from '@instantdb/platform'; +import type { backupDownloadDef, OptsFromCommand } from '../../index.ts'; +import { BadArgsError } from '../../errors.ts'; +import { GlobalOpts } from '../../context/globalOpts.ts'; +import { PlatformApiError } from '../../context/platformApi.ts'; +import { buildBackupsManager, useBackupsManager } from '../../lib/backups.ts'; +import { + downloadBackupToFile, + type BackupDownloadProgress, + type BackupDownloadResult, +} from '../../lib/backupDownload.ts'; +import { promptOk, runUIEffect } from '../../lib/ui.ts'; +import { onTerminate, renderUnwrap } from '../../ui/lib.ts'; +import { UI } from '../../ui/index.ts'; +import { relativeTime, renderBackupsTable, stripControlChars } from './list.ts'; + +const pickBackup = ( + backups: AppBackup[], + backupId: string | undefined, + opts: { latest?: boolean }, +) => + Effect.gen(function* () { + if (backupId && opts.latest) { + return yield* BadArgsError.make({ + message: 'Pass either a backup id or --latest, not both.', + }); + } + if (backupId) { + const found = backups.find((b) => b.id === backupId); + if (!found) { + return yield* BadArgsError.make({ + message: `No backup found with id ${backupId}.`, + }); + } + return found; + } + + // The server returns newest first; sort anyway so --latest can't silently + // pick the wrong one. + const sorted = [...backups].sort( + (a, b) => b.backupAt.getTime() - a.backupAt.getTime(), + ); + if (opts.latest) { + return sorted[0]; + } + + const { yes } = yield* GlobalOpts; + if (yes) { + return yield* BadArgsError.make({ + message: 'Must specify a backup id or --latest', + }); + } + + // The picker stays lean: id, age, and description are what you choose + // by, in the table's column order. Sizes and expiry show up in the + // confirm table right after, where headers explain them. + const cells = sorted.map((backup) => [ + backup.id, + `created ${relativeTime(backup.backupAt)}`, + backup.description ? stripControlChars(backup.description) : '', + ]); + const widths = cells[0].map((_, i) => + Math.max(...cells.map((row) => row[i].length)), + ); + return yield* runUIEffect( + new UI.Select({ + options: sorted.map((backup, idx) => ({ + label: cells[idx].map((cell, i) => cell.padEnd(widths[i])).join(' '), + value: backup, + })), + promptText: 'Select a backup to download:', + }), + ); + }); + +// One compact line for the spinner as the pipeline ticks. +function progressLine(p: BackupDownloadProgress): string { + const parts: string[] = []; + parts.push( + p.entitiesTotal == null + ? 'listing namespaces…' + : `namespaces ${p.entitiesCompleted}/${p.entitiesTotal}`, + ); + if (p.filesTotal !== 0) { + parts.push( + p.filesTotal == null + ? 'listing storage files…' + : `storage files ${p.filesCompleted}/${p.filesTotal}`, + ); + } + let bytes = formatFileSize(p.zipBytes); + if (p.bytesTotal != null && p.bytesTotal > 0) { + const pct = Math.min(100, Math.round((p.bytesRead / p.bytesTotal) * 100)); + bytes += ` (${pct}%)`; + } + parts.push(bytes); + const currentEntry = p.currentEntity || p.currentFile; + if (currentEntry) { + parts.push(currentEntry); + } + let line = parts.join(' · '); + // The spinner prefixes a frame glyph; truncate so the line can't wrap. + const width = (process.stdout.columns || 80) - 4; + if (line.length > width) { + line = line.slice(0, Math.max(0, width - 1)) + '…'; + } + return line; +} + +// Returns null when the download was cancelled. While the spinner is +// attached the terminal is raw, so ctrl-c arrives through the UI's +// terminate hook rather than SIGINT; both routes abort the same controller +// and the pipeline removes its partial file before we return. Outside the +// spinner (non-TTY runs), SIGINT covers it. +async function runDownload( + manager: BackupsManager, + backup: AppBackup, + outPath: string, +): Promise { + const controller = new AbortController(); + const onSigint = () => controller.abort(); + process.once('SIGINT', onSigint); + onTerminate(() => controller.abort()); + try { + let spinner: UI.Spinner | null = null; + // Settled into a sentinel so the spinner always disappears cleanly and + // cancellation/error output stays with the command below. + const settled = downloadBackupToFile({ + manager, + backup, + outPath, + signal: controller.signal, + onProgress: (p) => spinner?.updateText(progressLine(p)), + }).then( + (result) => ({ result, error: null as unknown }), + (error: unknown) => ({ result: null, error }), + ); + if (process.stdout.isTTY) { + spinner = new UI.Spinner({ + promise: settled, + workingText: 'Preparing download…', + disappearWhenDone: true, + }); + await renderUnwrap(spinner); + } + const { result, error } = await settled; + if (error) { + if ((error as { name?: string })?.name === 'AbortError') return null; + throw error; + } + return result; + } finally { + process.removeListener('SIGINT', onSigint); + onTerminate(undefined); + } +} + +export const backupDownloadCmd = Effect.fn(function* ( + backupId: string | undefined, + opts: OptsFromCommand, +) { + const backups = yield* useBackupsManager( + (m) => m.list(), + 'Error listing backups', + ); + + if (backups.length === 0) { + yield* Effect.log('No backups yet.'); + return; + } + + const backup = yield* pickBackup(backups, backupId, opts); + + const outPath = path.resolve(opts.out ?? backupZipName(backup)); + // Catch this before the download runs, not at the final rename. + if (existsSync(outPath) && statSync(outPath).isDirectory()) { + return yield* BadArgsError.make({ + message: `${outPath} is a directory.`, + }); + } + + yield* renderBackupsTable([backup]); + const sizes = estimateZipSize(backup); + if (sizes) { + yield* Effect.log( + `The zip file will be between ${formatFileSize(sizes.min)} and ${formatFileSize(sizes.max)}, depending on the compression ratio.`, + ); + } + yield* Effect.log(chalk.dim(`Saving to ${outPath} (pass -o to change).`)); + + const ok = yield* promptOk({ promptText: 'Download this backup?' }); + if (!ok) return; + + if (existsSync(outPath)) { + const overwrite = yield* promptOk({ + promptText: `${path.basename(outPath)} already exists. Overwrite?`, + }); + if (!overwrite) return; + } + + const manager = yield* buildBackupsManager; + + const result = yield* Effect.tryPromise({ + try: () => runDownload(manager, backup, outPath), + catch: (e) => + new PlatformApiError({ message: 'Error downloading backup', cause: e }), + }); + + if (result === null) { + yield* Effect.log('Download cancelled.'); + return; + } + + const filesPart = + result.files > 0 + ? ` and ${result.files.toLocaleString()} storage files` + : ''; + yield* Effect.log( + `Saved ${result.entities.toLocaleString()} namespaces${filesPart} (${formatFileSize(result.zipBytes)})`, + ); +}); diff --git a/client/packages/cli/src/commands/backup/list.ts b/client/packages/cli/src/commands/backup/list.ts new file mode 100644 index 0000000000..d7a8893bb0 --- /dev/null +++ b/client/packages/cli/src/commands/backup/list.ts @@ -0,0 +1,94 @@ +import chalk from 'chalk'; +import { Effect } from 'effect'; +import { formatFileSize, type AppBackup } from '@instantdb/platform'; +import type { backupListDef, OptsFromCommand } from '../../index.ts'; +import { useBackupsManager } from '../../lib/backups.ts'; + +export const formatBackupDate = (date: Date) => + `${date.toISOString().replace('T', ' ').slice(0, 16)} UTC`; + +// Backup descriptions are user-controlled text headed for the terminal; +// strip control characters so a crafted value can't inject escape sequences. +export const stripControlChars = (s: string) => s.replace(/\p{Cc}/gu, ''); + +// Relative times in both directions: "3 hours ago", "6 days from now". +export const relativeTime = (date: Date): string => { + const diffMs = date.getTime() - Date.now(); + const abs = Math.abs(diffMs); + if (abs < 60_000) return diffMs <= 0 ? 'just now' : 'now'; + const minutes = Math.round(abs / 60_000); + const hours = Math.round(abs / 3_600_000); + const days = Math.round(abs / 86_400_000); + const [count, unit] = + minutes < 60 + ? [minutes, 'minute'] + : hours < 24 + ? [hours, 'hour'] + : [days, 'day']; + const label = `${count} ${unit}${count === 1 ? '' : 's'}`; + return diffMs < 0 ? `${label} ago` : `${label} from now`; +}; + +// One aligned row per backup, newest first: id first, relative times. +// `--json` carries the precise values. +export const renderBackupsTable = (backups: AppBackup[]) => + Effect.gen(function* () { + const header = [ + 'ID', + 'CREATED AT', + 'DB SIZE', + 'STORAGE', + 'EXPIRES AT', + 'DESCRIPTION', + ]; + const rows = backups.map((backup) => [ + backup.id, + relativeTime(backup.backupAt), + backup.dbSize != null ? formatFileSize(backup.dbSize) : '-', + backup.filesSize != null ? formatFileSize(backup.filesSize) : '-', + backup.expiresAt + ? backup.expiresAt.getTime() <= Date.now() + ? 'expired' + : relativeTime(backup.expiresAt) + : '-', + backup.description ? stripControlChars(backup.description) : '', + ]); + const widths = header.map((h, i) => + Math.max(h.length, ...rows.map((row) => row[i].length)), + ); + const line = (cells: string[]) => + cells + .map((cell, i) => cell.padEnd(widths[i])) + .join(' ') + .trimEnd(); + yield* Effect.log(chalk.dim(line(header))); + yield* Effect.log(chalk.dim(widths.map((w) => '-'.repeat(w)).join(' '))); + for (const row of rows) { + yield* Effect.log(line(row)); + } + }); + +export const backupListCmd = Effect.fn(function* ( + opts: OptsFromCommand, +) { + const backups = yield* useBackupsManager( + (m) => m.list(), + 'Error listing backups', + ); + + if (opts.json) { + yield* Effect.log(JSON.stringify(backups, null, 2)); + return; + } + + if (backups.length === 0) { + yield* Effect.log('No backups yet.'); + return; + } + + // The server returns newest first; sort anyway so the table can't lie. + const sorted = [...backups].sort( + (a, b) => b.backupAt.getTime() - a.backupAt.getTime(), + ); + yield* renderBackupsTable(sorted); +}); diff --git a/client/packages/cli/src/index.ts b/client/packages/cli/src/index.ts index be6c5a622d..3916ed02fe 100644 --- a/client/packages/cli/src/index.ts +++ b/client/packages/cli/src/index.ts @@ -51,6 +51,8 @@ import { webhooksEventsResendCmd } from './commands/webhooks/events/resend.ts'; import { emailStatusCmd } from './commands/auth/email/status.ts'; import { verifyCmd } from './commands/auth/email/verify.ts'; import { resendEmailCmd } from './commands/auth/email/resend.ts'; +import { backupListCmd } from './commands/backup/list.ts'; +import { backupDownloadCmd } from './commands/backup/download.ts'; export type OptsFromCommand = C extends Command ? R : never; @@ -640,6 +642,64 @@ export const webhooksEventsPayloadDef = webhooksEvents ); }); +const backup = program + .command('backup') + .description('View and download backups of your app'); + +export const backupListDef = backup + .command('list') + .description('List downloadable backups for an app') + .option( + '-a --app ', + 'App ID to list backups for. Defaults to *_INSTANT_APP_ID in .env', + ) + .option('--json', 'Output backups as JSON') + .action((opts) => { + return runCommandEffect( + backupListCmd(opts).pipe( + Effect.provide( + WithAppLayer({ + coerce: false, + coerceAuth: false, + appId: opts.app, + allowAdminToken: true, + }).pipe(Layer.annotateLogs('silent', !!opts.json)), + ), + ), + ); + }); + +export const backupDownloadDef = backup + .command('download') + .description('Download a backup as a zip file') + .argument( + '[backup-id]', + 'Backup ID to download. Defaults to an interactive picker', + ) + .option( + '-a --app ', + 'App ID to download a backup of. Defaults to *_INSTANT_APP_ID in .env', + ) + .option('--latest', 'Download the most recent backup') + .option( + '-o --out ', + 'Output zip path. Defaults to instant-backup-.zip', + ) + .action((backupId, opts) => { + return runCommandEffect( + backupDownloadCmd(backupId, opts).pipe( + Effect.provide( + WithAppLayer({ + coerce: false, + coerceAuth: false, + appId: opts.app, + allowAdminToken: true, + }), + ), + ), + ); + }); + const authEmail = auth .command('email') .description('Manage custom magic code email templates'); diff --git a/client/packages/cli/src/lib/backupDownload.ts b/client/packages/cli/src/lib/backupDownload.ts new file mode 100644 index 0000000000..ffaefa6aca --- /dev/null +++ b/client/packages/cli/src/lib/backupDownload.ts @@ -0,0 +1,145 @@ +import { createWriteStream } from 'node:fs'; +import { open, rename, unlink } from 'node:fs/promises'; +import { randomBytes } from 'node:crypto'; +import { once } from 'node:events'; +import { get as httpGet, type IncomingMessage } from 'node:http'; +import { get as httpsGet } from 'node:https'; +import { Readable, Writable, type Duplex } from 'node:stream'; +import zlib from 'node:zlib'; +import type { + AppBackup, + BackupArchiveWriter, + BackupDownloadProgress, + BackupDownloadResult, + BackupsManager, +} from '@instantdb/platform'; + +export type { + BackupDownloadProgress, + BackupDownloadResult, +} from '@instantdb/platform'; + +// zstd landed in node:zlib in 22.15 / 23.8; on older Nodes the property is +// absent, so feature-detect instead of assuming the type declarations match +// the runtime. +const createZstdDecompress: (() => Duplex) | undefined = ( + zlib as { createZstdDecompress?: () => Duplex } +).createZstdDecompress; + +function fetchStream( + url: string, + signal: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const get = url.startsWith('https:') ? httpsGet : httpGet; + const req = get(url, { signal }, resolve); + req.on('error', reject); + }); +} + +// .pipe doesn't forward errors, so a source failure would otherwise leave the +// destination (and the zip writer reading from it) hanging forever. +function pipe(src: Readable, dst: Duplex): Duplex { + src.on('error', (e) => dst.destroy(e)); + return src.pipe(dst); +} + +// Fetches a presigned URL with node:http(s), decompressing explicitly: the +// entity shards are served with `Content-Encoding: zstd` and Node doesn't +// auto-decompress that. downloadBackupToFile refuses to run without zstd +// support, so the assertion below can't fire. +async function fetchBody( + url: string, + signal: AbortSignal, +): Promise> { + const res = await fetchStream(url, signal); + if (res.statusCode !== 200) { + res.resume(); + throw new Error(`HTTP ${res.statusCode}`); + } + const encoding = res.headers['content-encoding']; + let stream: Readable = res; + if (encoding === 'zstd') { + stream = pipe(stream, createZstdDecompress!()); + } else if (encoding === 'gzip') { + stream = pipe(stream, zlib.createGunzip()); + } else if (encoding) { + res.destroy(); + throw new Error(`Unsupported content encoding: ${encoding}`); + } + return Readable.toWeb(stream) as ReadableStream; +} + +async function createZipWriter( + sink: WritableStream, + signal: AbortSignal, +): Promise { + // Loaded on demand so every other CLI command skips parsing it. + const { ZipWriter } = await import('@zip.js/zip.js'); + // zip64: without it any archive whose central-directory offset passes 4GB + // writes a wrapped 32-bit offset and the zip is unreadable. + return new ZipWriter(sink, { zip64: true, signal }); +} + +/** + * Downloads a backup into a zip file at `outPath` via the shared + * `BackupsManager.downloadArchive` pipeline, supplying the Node-specific + * pieces: + * presigned URLs are fetched with node:http(s) and decompressed explicitly, + * and the archive streams to disk with backpressure so memory stays flat + * regardless of backup size. + * + * Writes to `.partial` and renames on success; a failed or aborted + * download removes the partial file. + */ +export async function downloadBackupToFile(opts: { + manager: BackupsManager; + backup: AppBackup; + outPath: string; + signal: AbortSignal; + onProgress: (progress: BackupDownloadProgress) => void; +}): Promise { + // The entity shards are served with `Content-Encoding: zstd`; fail before + // writing anything if this Node can't decompress them. + if (!createZstdDecompress) { + throw new Error( + 'Downloading backups requires Node 22.15 or newer (for zstd support).', + ); + } + + // Randomized so a stale partial or a concurrent download of the same + // backup can't collide; 'wx' turns any remaining collision into an error + // instead of silently truncating another run's file. + const partialPath = `${opts.outPath}.partial-${randomBytes(4).toString('hex')}`; + const fileStream = createWriteStream(partialPath, { flags: 'wx' }); + const awaitFileClosed = async () => { + if (!fileStream.closed) await once(fileStream, 'close'); + }; + try { + const result = await opts.manager.downloadArchive({ + backup: opts.backup, + fetchBody, + sink: Writable.toWeb(fileStream) as WritableStream, + createWriter: createZipWriter, + signal: opts.signal, + onProgress: opts.onProgress, + }); + await awaitFileClosed(); + // Flush to disk before the rename so a crash right after can't leave a + // complete-looking zip with unwritten tails. + const fh = await open(partialPath, 'r+'); + try { + await fh.sync(); + } finally { + await fh.close(); + } + await rename(partialPath, opts.outPath); + return result; + } catch (e) { + // The pipeline already aborted the sink; wait for the fd to close, then + // discard the partial file on disk. + await awaitFileClosed().catch(() => {}); + await unlink(partialPath).catch(() => {}); + throw e; + } +} diff --git a/client/packages/cli/src/lib/backups.ts b/client/packages/cli/src/lib/backups.ts new file mode 100644 index 0000000000..9327d5ccb4 --- /dev/null +++ b/client/packages/cli/src/lib/backups.ts @@ -0,0 +1,33 @@ +import { Effect } from 'effect'; +import type { BackupsManager } from '@instantdb/platform'; +import { CurrentApp } from '../context/currentApp.ts'; +import { PlatformApiError } from '../context/platformApi.ts'; +import { getAuthedPlatformApi } from './platformApi.ts'; + +export const useBackupsManager = ( + fun: (manager: BackupsManager) => Promise, + errorMessage?: string, +) => + Effect.gen(function* () { + const api = yield* getAuthedPlatformApi; + const { appId } = yield* CurrentApp; + return yield* Effect.tryPromise({ + try: () => fun(api.backups(appId)), + catch: (e) => + new PlatformApiError({ + message: errorMessage ?? 'Error using backups api', + cause: e, + }), + }); + }); + +/** + * Yields a `BackupsManager` instance scoped to the current app. Use when you + * need to hold on to the manager outside an Effect (e.g. to drive the + * long-running download pipeline). + */ +export const buildBackupsManager = Effect.gen(function* () { + const api = yield* getAuthedPlatformApi; + const { appId } = yield* CurrentApp; + return api.backups(appId); +}); diff --git a/client/packages/cli/src/lib/platformApi.ts b/client/packages/cli/src/lib/platformApi.ts new file mode 100644 index 0000000000..31bc4c8bcb --- /dev/null +++ b/client/packages/cli/src/lib/platformApi.ts @@ -0,0 +1,11 @@ +import { Effect } from 'effect'; +import { PlatformApi as InstantPlatformApi } from '@instantdb/platform'; +import { AuthToken } from '../context/authToken.ts'; +import { getBaseUrl } from './config.ts'; + +export const getAuthedPlatformApi = Effect.gen(function* () { + const apiURI = yield* getBaseUrl; + const authToken = yield* AuthToken; + const token = yield* authToken.getAuthToken; + return new InstantPlatformApi({ apiURI, auth: { token } }); +}); diff --git a/client/packages/cli/src/lib/webhooks.ts b/client/packages/cli/src/lib/webhooks.ts index cc89da8ed5..044a4f34ce 100644 --- a/client/packages/cli/src/lib/webhooks.ts +++ b/client/packages/cli/src/lib/webhooks.ts @@ -1,22 +1,13 @@ import { Effect } from 'effect'; import { - PlatformApi as InstantPlatformApi, type WebhookAction, type WebhookEventInfo, type WebhooksManager, } from '@instantdb/platform'; -import { AuthToken } from '../context/authToken.ts'; import { CurrentApp } from '../context/currentApp.ts'; import { PlatformApiError } from '../context/platformApi.ts'; import { BadArgsError } from '../errors.ts'; -import { getBaseUrl } from './http.ts'; - -const getAuthedPlatformApi = Effect.gen(function* () { - const apiURI = yield* getBaseUrl; - const authToken = yield* AuthToken; - const token = yield* authToken.getAuthToken; - return new InstantPlatformApi({ apiURI, auth: { token } }); -}); +import { getAuthedPlatformApi } from './platformApi.ts'; export const WEBHOOK_ACTIONS: readonly WebhookAction[] = [ 'create', diff --git a/client/packages/cli/src/ui/lib.ts b/client/packages/cli/src/ui/lib.ts index 5610a6a8bc..f92a65e8bc 100644 --- a/client/packages/cli/src/ui/lib.ts +++ b/client/packages/cli/src/ui/lib.ts @@ -372,7 +372,7 @@ let terminateHandler: | undefined; export function onTerminate( - callback: (stdin: ReadStream, stdout: WriteStream) => void | undefined, + callback: ((stdin: ReadStream, stdout: WriteStream) => void) | undefined, ) { terminateHandler = callback; } diff --git a/client/packages/platform/__tests__/src/backupDownload.test.ts b/client/packages/platform/__tests__/src/backupDownload.test.ts new file mode 100644 index 0000000000..e9fb6fa99e --- /dev/null +++ b/client/packages/platform/__tests__/src/backupDownload.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, test } from 'vitest'; +import { downloadBackupArchive } from '../../src/backupDownload.ts'; +import type { AppBackup } from '../../src/backups.ts'; + +const backup: AppBackup = { + id: 'backup-1', + isn: '1', + backupAt: new Date('2026-08-01T00:00:00Z'), + filesSize: 16, + dbSize: 100, + uncompressedSize: 40, + description: null, + expiresAt: null, +}; + +const bodyOf = (text: string) => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }); + +const fetchBody = async (url: string) => bodyOf(`body:${url}`); + +const listFiles = async () => [ + { name: 'config.json', size: 1 }, + { name: 'entities/todos.jsonl', size: 1 }, +]; +const getFileUrl = async (_backupId: string, name: string) => name; + +// Minimal archive writer: records entry names, drains each input, and writes +// a byte through the counting sink so zipBytes moves like a real encoder. +const makeWriter = + (names: string[]) => async (sink: WritableStream) => { + const w = sink.getWriter(); + return { + add: async (name: string, input: ReadableStream) => { + names.push(name); + for await (const _chunk of input) { + // drain + } + await w.write(new Uint8Array([0])); + }, + close: () => w.close(), + }; + }; + +const nullSink = () => new WritableStream({ write() {} }); + +describe('downloadBackupArchive', () => { + test('writes entries in canonical order through the injected writer', async () => { + const names: string[] = []; + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + }, + } as any; + + const result = await downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter: makeWriter(names), + }); + + expect(names).toEqual([ + 'config.json', + 'entities/todos.jsonl', + 'files/loc-1', + ]); + expect(result.entities).toBe(1); + expect(result.files).toBe(1); + expect(result.zipBytes).toBe(3); + }); + + test('counts an entry as completed only after the writer consumed it', async () => { + const seen: Array<{ + name: string; + entitiesCompleted: number; + filesCompleted: number; + }> = []; + let last = { entitiesCompleted: 0, filesCompleted: 0 }; + const createWriter = async (sink: WritableStream) => { + const w = sink.getWriter(); + return { + add: async (name: string, input: ReadableStream) => { + seen.push({ name, ...last }); + for await (const _chunk of input) { + // drain + } + await w.write(new Uint8Array([0])); + }, + close: () => w.close(), + }; + }; + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + }, + } as any; + + await downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter, + onProgress: (p) => { + last = { + entitiesCompleted: p.entitiesCompleted, + filesCompleted: p.filesCompleted, + }; + }, + }); + + // At the moment each add starts, the entry being written isn't counted. + expect(seen).toEqual([ + { name: 'config.json', entitiesCompleted: 0, filesCompleted: 0 }, + { name: 'entities/todos.jsonl', entitiesCompleted: 0, filesCompleted: 0 }, + { name: 'files/loc-1', entitiesCompleted: 1, filesCompleted: 0 }, + ]); + }); + + test('a cancellation while storage discovery is idle rejects instead of completing', async () => { + const controller = new AbortController(); + const manager = { + listFiles, + getFileUrl, + // Yields one file, then holds the stream open like a server with more + // to send, rejecting with AbortError when the download is cancelled — + // the same shape as a real aborted fetch. + streamStorageFiles: ( + _backupId: string, + opts?: { signal?: AbortSignal }, + ) => + (async function* () { + yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' }; + yield await new Promise((_resolve, reject) => { + opts?.signal?.addEventListener('abort', () => + reject( + Object.assign(new Error('aborted'), { name: 'AbortError' }), + ), + ); + }); + })(), + } as any; + + const promise = downloadBackupArchive({ + manager, + backup, + fetchBody, + sink: nullSink(), + createWriter: makeWriter([]), + signal: controller.signal, + onProgress: (p) => { + // Cancel once the only discovered file is fully written and the + // drain loop is about to go idle. + if (p.filesCompleted === 1) controller.abort(); + }, + }); + + await expect(promise).rejects.toMatchObject({ name: 'AbortError' }); + }); + + test('names a pathless storage file by locationId when its download fails', async () => { + const manager = { + listFiles, + getFileUrl, + streamStorageFiles: async function* () { + yield { locationId: 'loc-9', path: null, url: 'bad-url' }; + }, + } as any; + + await expect( + downloadBackupArchive({ + manager, + backup, + fetchBody: async (url: string) => { + if (url === 'bad-url') throw new Error('HTTP 500'); + return bodyOf('x'); + }, + sink: nullSink(), + createWriter: makeWriter([]), + }), + ).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).'); + }); +}); diff --git a/client/packages/platform/__tests__/src/backups.test.ts b/client/packages/platform/__tests__/src/backups.test.ts new file mode 100644 index 0000000000..95e10e0ecc --- /dev/null +++ b/client/packages/platform/__tests__/src/backups.test.ts @@ -0,0 +1,168 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { + BackupsManager, + backupZipName, + formatFileSize, + type AppBackupStorageFile, +} from '../../src/backups.ts'; + +const makeManager = () => + new BackupsManager({ + appId: 'app-1', + apiURI: 'http://api.test', + withAuth: (operation) => operation('test-token'), + }); + +const streamOf = (chunks: string[]) => + new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + for (const chunk of chunks) { + controller.enqueue(encoder.encode(chunk)); + } + controller.close(); + }, + }); + +const stubFetchBody = (chunks: string[]) => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(streamOf(chunks), { status: 200 })), + ); +}; + +const collect = async (manager: BackupsManager) => { + const files: AppBackupStorageFile[] = []; + for await (const file of manager.streamStorageFiles('backup-1')) { + files.push(file); + } + return files; +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('streamStorageFiles', () => { + test('yields files and completes on the done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"locationId":"loc-2","path":"b.png","url":"http://s3/loc-2"}\n', + '{"done":true}\n', + ]); + + const files = await collect(makeManager()); + expect(files).toEqual([ + { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + { locationId: 'loc-2', path: 'b.png', url: 'http://s3/loc-2' }, + ]); + }); + + test('handles lines split across chunks', async () => { + stubFetchBody([ + '{"locationId":"loc-1","pa', + 'th":"a.png","url":"http://s3/loc-1"}\n{"done"', + ':true}\n', + ]); + + const files = await collect(makeManager()); + expect(files).toEqual([ + { locationId: 'loc-1', path: 'a.png', url: 'http://s3/loc-1' }, + ]); + }); + + test('throws on a record without a locationId or url', async () => { + stubFetchBody([ + '{"path":"orphan.png"}\n', + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":true}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); + }); + + test('only accepts an exact done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":1}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); + }); + + test('completes without files when the app has none', async () => { + stubFetchBody(['{"done":true}\n']); + expect(await collect(makeManager())).toEqual([]); + }); + + test('rejects a locationId that could escape the archive path', async () => { + stubFetchBody([ + '{"locationId":"../evil","path":"a.png","url":"http://s3/loc-1"}\n', + '{"done":true}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow(/malformed record/); + }); + + test('throws when the stream ends without the done sentinel', async () => { + stubFetchBody([ + '{"locationId":"loc-1","path":"a.png","url":"http://s3/loc-1"}\n', + ]); + + await expect(collect(makeManager())).rejects.toThrow( + /ended before it finished/, + ); + }); +}); + +describe('list', () => { + test('coerces the server row into an AppBackup', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => + Response.json({ + backups: [ + { + id: 'backup-1', + isn: '42', + backup_at: '2026-08-05T07:12:00Z', + files_size: 100, + db_size: 200, + uncompressed_size: 300, + description: 'Automated Daily Snapshot', + expires_at: '2026-08-12T07:12:00Z', + }, + ], + }), + ), + ); + + const [backup] = await makeManager().list(); + expect(backup).toEqual({ + id: 'backup-1', + isn: '42', + backupAt: new Date('2026-08-05T07:12:00Z'), + filesSize: 100, + dbSize: 200, + uncompressedSize: 300, + description: 'Automated Daily Snapshot', + expiresAt: new Date('2026-08-12T07:12:00Z'), + }); + expect(backupZipName(backup)).toBe( + 'instant-backup-2026-08-05T07-12-00-000Z.zip', + ); + }); +}); + +describe('formatFileSize', () => { + test('formats with decimal units and adaptive precision', () => { + expect(formatFileSize(999)).toBe('999 B'); + expect(formatFileSize(3510)).toBe('3.51 KB'); + expect(formatFileSize(1_540_000_000)).toBe('1.54 GB'); + }); + + test('promotes across the unit boundary when rounding', () => { + expect(formatFileSize(999_500)).toBe('1.00 MB'); + expect(formatFileSize(999_999_500)).toBe('1.00 GB'); + }); +}); diff --git a/client/packages/platform/src/api.ts b/client/packages/platform/src/api.ts index 9649802120..bc08e4be27 100644 --- a/client/packages/platform/src/api.ts +++ b/client/packages/platform/src/api.ts @@ -15,6 +15,7 @@ import { DataAttrDef, } from '@instantdb/core'; import { Webhooks, type WithAuth } from '@instantdb/webhooks'; +import { BackupsManager } from './backups.ts'; import version from './version.ts'; import { attrFwdLabel, @@ -1878,4 +1879,18 @@ export class PlatformApi { withAuth, }); } + + /** + * Returns a {@link BackupsManager} scoped to `appId` for listing an app's + * backups and downloading their contents. Calls are routed through + * {@link withRetry}, so an expired access token is transparently refreshed. + */ + backups(appId: string): BackupsManager { + const withAuth: WithAuth = (operation) => + this.withRetry( + (_apiURI: string, token: string) => operation(token), + [this.#apiURI, this.token()], + ); + return new BackupsManager({ appId, apiURI: this.#apiURI, withAuth }); + } } diff --git a/client/packages/platform/src/backupDownload.ts b/client/packages/platform/src/backupDownload.ts new file mode 100644 index 0000000000..68ac929c31 --- /dev/null +++ b/client/packages/platform/src/backupDownload.ts @@ -0,0 +1,364 @@ +import type { + AppBackup, + AppBackupStorageFile, + BackupsManager, +} from './backups.ts'; + +export type BackupDownloadProgress = { + entitiesCompleted: number; + entitiesTotal: number | null; + filesCompleted: number; + filesTotal: number | null; + // Compressed bytes written to the sink so far (the zip's on-disk size). + zipBytes: number; + // Uncompressed bytes read from source bodies, and the backup's known + // uncompressed total — the numerator/denominator for a progress bar. + // bytesTotal is null when the backup row carries no sizes. + bytesRead: number; + bytesTotal: number | null; + // The entry currently being fetched in each phase; empty while that phase + // isn't actively fetching, so a finished phase stops claiming a file. + currentEntity: string; + currentFile: string; +}; + +export type BackupDownloadResult = { + entities: number; + files: number; + zipBytes: number; +}; + +/** + * The archive encoder {@link downloadBackupArchive} writes entries through, + * supplied by the caller so this package doesn't depend on a zip + * implementation. zip.js's `ZipWriter` satisfies it structurally, so + * `new ZipWriter(sink, { zip64: true, signal })` works without an adapter. + * + * Implementations must handle archives past 4GB — for zip that means zip64, + * without which the central-directory offsets wrap and the archive is + * silently unreadable. + */ +export type BackupArchiveWriter = { + add( + name: string, + input: ReadableStream, + opts: { lastModDate: Date }, + ): Promise; + close(): Promise; +}; + +export type DownloadBackupArchiveOpts = { + backup: AppBackup; + /** + * Fetches a presigned URL, resolving with the response body and rejecting + * on a non-200 status. Put the status in the message (e.g. `HTTP 403`) — + * it's surfaced to the user alongside the failing entry's name. The entity + * files are served with `Content-Encoding: zstd`; browser fetch decodes + * that transparently, other runtimes must decompress explicitly. + */ + fetchBody: ( + url: string, + signal: AbortSignal, + ) => Promise>; + /** + * Where the archive's bytes go. Closed after the last entry is written; + * aborted when the download fails or is cancelled, so the caller can + * discard partial output. + */ + sink: WritableStream; + /** + * Builds the archive encoder over a sink that already counts progress and + * carries the caller's sink's backpressure. + */ + createWriter: ( + sink: WritableStream, + signal: AbortSignal, + ) => Promise; + signal?: AbortSignal; + onProgress?: (progress: BackupDownloadProgress) => void; +}; + +const isAbortError = (e: unknown): boolean => + (e as { name?: string })?.name === 'AbortError'; + +const errorMessage = (e: unknown): string => + e instanceof Error ? e.message : String(e); + +/** + * Downloads a backup into a single archive written to `opts.sink`: entries + * in the canonical restore order (`config.json`, then the + * `entities/.jsonl` shards, then `files/` storage blobs — + * all entity files before any storage file), with the encoder writing + * through a counting sink that awaits the caller's sink, so a fast source + * can't outrun it and balloon memory. + * + * The runtime-specific pieces are injected: how to fetch a presigned URL + * (`fetchBody`), where the bytes go (`sink`), and the archive encoder + * (`createWriter`). Most callers reach this via + * {@link BackupsManager.downloadArchive}. + */ +export async function downloadBackupArchive( + opts: DownloadBackupArchiveOpts & { + manager: Pick< + BackupsManager, + 'listFiles' | 'getFileUrl' | 'streamStorageFiles' + >; + }, +): Promise { + const { manager, backup, fetchBody, createWriter, onProgress } = opts; + + // Internal controller so a pipeline failure also tears down the + // storage-files discovery stream and any in-flight body fetches. + const abortController = new AbortController(); + if (opts.signal?.aborted) { + abortController.abort(); + } else { + opts.signal?.addEventListener('abort', () => abortController.abort(), { + once: true, + }); + } + const signal = abortController.signal; + + let entitiesCompleted = 0; + let entitiesTotal: number | null = null; + let filesCompleted = 0; + let filesTotal: number | null = null; + let zipBytes = 0; + let bytesRead = 0; + let currentEntity = ''; + let currentFile = ''; + const bytesTotal = + backup.uncompressedSize != null + ? backup.uncompressedSize + (backup.filesSize ?? 0) + : null; + + const tick = () => + onProgress?.({ + entitiesCompleted, + entitiesTotal, + filesCompleted, + filesTotal, + zipBytes, + bytesRead, + bytesTotal, + currentEntity, + currentFile, + }); + + // Throttle by time: a large backup pushes many small chunks and ticking on + // every one is wasted work. Phase changes tick() directly so they're still + // immediate. + const TICK_INTERVAL_MS = 100; + let lastTickAt = 0; + const throttledTick = () => { + const now = Date.now(); + if (now - lastTickAt >= TICK_INTERVAL_MS) { + lastTickAt = now; + tick(); + } + }; + + // Count the uncompressed bytes of a source body for progress as it streams + // into the archive. + const countBytes = ( + body: ReadableStream, + ): ReadableStream => + body.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + bytesRead += chunk.byteLength; + throttledTick(); + controller.enqueue(chunk); + }, + }), + ); + + // Storage-files discovery runs concurrently with the entity phase and is + // drained eagerly into a queue. That isn't just overlap: it closes the + // NDJSON connection quickly instead of holding it open (and at the mercy of + // idle timeouts) while multi-GB blobs download. `queueHead` walks the array + // in place, freeing each slot as it's consumed. + const queue: (AppBackupStorageFile | undefined)[] = []; + let queueHead = 0; + let storageDone = false; + let storageError: Error | null = null; + let waitResolve: (() => void) | null = null; + const notify = () => { + const w = waitResolve; + waitResolve = null; + w?.(); + }; + + // Never rejects: failures land in storageError for the drain loop to throw. + const discovery = (async () => { + let discoveryComplete = false; + try { + for await (const file of manager.streamStorageFiles(backup.id, { + signal, + })) { + queue.push(file); + filesTotal = (filesTotal ?? 0) + 1; + throttledTick(); + notify(); + } + discoveryComplete = true; + } catch (e) { + // The abort path is expected when the pipeline failed and we tore the + // discovery down. + if (!isAbortError(e)) { + storageError = e as Error; + } + } finally { + // A failed listing keeps the total unknown rather than reading as an + // empty-but-complete storage phase. + if (discoveryComplete && filesTotal == null) filesTotal = 0; + storageDone = true; + tick(); + notify(); + } + })(); + + // Entry write order is significant for restore: config.json first, then the + // entities/*.jsonl shards, then files/. In particular ALL entity + // files must be written before ANY storage file. listFiles returns the + // entity files in write order; this generator yields them to completion, + // then drains the storage queue. + type ArchiveEntry = { + name: string; + input: ReadableStream; + // Fired after the writer finishes consuming the entry, so completion + // counters reflect fully-written files rather than started fetches. + onAdded: () => void; + }; + const entries = (async function* (): AsyncGenerator { + const files = await manager.listFiles(backup.id, { signal }); + if (files.length === 0) { + throw new Error('No files found for this backup.'); + } + // We write entries in the order the server returns them, and restore + // requires config.json to be the first entry. Fail loudly rather than + // build a zip that can't be restored. + if (files[0].name !== 'config.json') { + throw new Error( + `Backup files came back in an unexpected order (expected config.json first, got "${files[0].name}").`, + ); + } + // config.json isn't a namespace — count only the entities/*.jsonl shards. + entitiesTotal = files.filter((f) => f.name !== 'config.json').length; + tick(); + + for (const f of files) { + currentEntity = f.name; + tick(); + const url = await manager.getFileUrl(backup.id, f.name, { signal }); + let body: ReadableStream; + try { + body = await fetchBody(url, signal); + } catch (e) { + if (isAbortError(e)) throw e; + throw new Error(`Failed to fetch ${f.name}: ${errorMessage(e)}.`); + } + yield { + name: f.name, + input: countBytes(body), + onAdded: () => { + if (f.name !== 'config.json') entitiesCompleted++; + tick(); + }, + }; + } + currentEntity = ''; + tick(); + + while (true) { + if (storageError) throw storageError; + // A caller abort while no fetch is in flight surfaces only in the + // discovery stream, which swallows it as expected teardown — check + // explicitly so a cancellation can't read as a complete storage phase + // with files still undiscovered. + signal.throwIfAborted(); + let file: AppBackupStorageFile | undefined; + if (queueHead < queue.length) { + file = queue[queueHead]; + queue[queueHead] = undefined; + queueHead++; + } + if (file) { + const label = file.path || file.locationId; + currentFile = label; + tick(); + let body: ReadableStream; + try { + body = await fetchBody(file.url, signal); + } catch (e) { + if (isAbortError(e)) throw e; + throw new Error( + `Couldn't download storage file "${label}" (${errorMessage(e)}).`, + ); + } + yield { + name: `files/${file.locationId}`, + input: countBytes(body), + onAdded: () => { + filesCompleted++; + tick(); + }, + }; + } else if (storageDone) { + break; + } else { + await new Promise((resolve) => { + waitResolve = resolve; + }); + } + } + currentFile = ''; + tick(); + + if (storageError) throw storageError; + })(); + + const sinkWriter = opts.sink.getWriter(); + try { + // Sink the archive encoder writes into: it tallies the encoded size for + // progress, then forwards to the caller's sink. Awaiting the downstream + // write propagates backpressure up into the encoder, so a fast source + // can't outrun the sink and balloon memory. + const countingSink = new WritableStream({ + async write(chunk) { + zipBytes += chunk.byteLength; + throttledTick(); + await sinkWriter.write(chunk); + }, + async close() { + await sinkWriter.close(); + tick(); + }, + async abort(reason) { + await sinkWriter.abort(reason); + }, + }); + + const writer = await createWriter(countingSink, signal); + for await (const entry of entries) { + await writer.add(entry.name, entry.input, { + lastModDate: backup.backupAt, + }); + entry.onAdded(); + } + // A caller abort that lands after the last entry lets the generator + // finish cleanly; don't close and return a complete-looking archive. + signal.throwIfAborted(); + await writer.close(); + await discovery; + tick(); + return { entities: entitiesCompleted, files: filesCompleted, zipBytes }; + } catch (e) { + // Tear down the discovery stream and any in-flight body fetches so we + // don't keep pulling from S3, and abort the caller's sink so it can + // discard whatever it wrote. + abortController.abort(); + await sinkWriter.abort(e).catch(() => {}); + throw e; + } +} diff --git a/client/packages/platform/src/backups.ts b/client/packages/platform/src/backups.ts new file mode 100644 index 0000000000..906a233ab3 --- /dev/null +++ b/client/packages/platform/src/backups.ts @@ -0,0 +1,342 @@ +import { InstantAPIError, version as coreVersion } from '@instantdb/core'; +import type { WithAuth } from '@instantdb/webhooks'; +import version from './version.ts'; +import { + downloadBackupArchive, + type BackupDownloadResult, + type DownloadBackupArchiveOpts, +} from './backupDownload.ts'; + +/** A point-in-time snapshot of an app. */ +export type AppBackup = { + /** Unique identifier for the backup. */ + id: string; + /** Instant sequence number the snapshot was taken at. */ + isn: string; + /** When the snapshot was taken. */ + backupAt: Date; + /** Total size in bytes of the app's storage files at backup time, if known. */ + filesSize: number | null; + /** Size in bytes of the app's database at backup time, if known. */ + dbSize: number | null; + /** + * Total uncompressed size in bytes of the backup's entity files (what they + * take up unpacked on disk), if known. + */ + uncompressedSize: number | null; + /** Human-readable label, e.g. "Automated Daily Snapshot". */ + description: string | null; + /** When the backup stops being available for download. */ + expiresAt: Date | null; +}; + +/** + * A file that makes up the backup payload: `config.json` or an + * `entities/.jsonl` shard. + */ +export type AppBackupFile = { + name: string; + /** Size in bytes as stored (the entity shards are stored compressed). */ + size: number; +}; + +/** A storage file captured in a backup, with a presigned download URL. */ +export type AppBackupStorageFile = { + /** + * Stable id of the file's blob. A backup archive stores the blob at + * `files/`. + */ + locationId: string; + /** The path the user uploaded the file to. */ + path: string | null; + /** Presigned URL for the file's contents. Expires after 12 hours. */ + url: string; +}; + +type AppBackupResponse = { + id: string; + isn: string; + backup_at: string; + files_size: number | null; + db_size: number | null; + uncompressed_size: number | null; + description: string | null; + expires_at: string | null; +}; + +/** Converts a backup row as the server sends it into an {@link AppBackup}. */ +export function toAppBackup(row: AppBackupResponse): AppBackup { + return { + id: row.id, + isn: row.isn, + backupAt: new Date(row.backup_at), + filesSize: row.files_size, + dbSize: row.db_size, + uncompressedSize: row.uncompressed_size, + description: row.description, + expiresAt: row.expires_at ? new Date(row.expires_at) : null, + }; +} + +/** Suggested filename for a backup archive. */ +export function backupZipName(backup: AppBackup): string { + const safe = backup.backupAt.toISOString().replace(/[:.]/g, '-'); + return `instant-backup-${safe}.zip`; +} + +/** + * Formats a byte count the way macOS/Finder reports file sizes: decimal + * (1000-based) units with SI labels, so the number lines up with what lands + * on disk. + */ +export function formatFileSize(n: number): string { + if (n < 1000) return `${n} B`; + const units = ['KB', 'MB', 'GB', 'TB']; + let i = -1; + let v = n; + do { + v /= 1000; + i++; + } while (v >= 1000 && i < units.length - 1); + let digits = v < 10 ? 2 : v < 100 ? 1 : 0; + // Rounding can cross a unit boundary (999,500 would read "1000 KB"); + // promote to the next unit instead. + if (Number(v.toFixed(digits)) >= 1000 && i < units.length - 1) { + v /= 1000; + i++; + digits = 2; + } + return `${v.toFixed(digits)} ${units[i]}`; +} + +/** + * Estimated size range for a backup's zip archive, or null when the backup + * row carries no sizes. Upper bound: everything stored uncompressed (STORE + * mode and/or files that don't compress). Lower bound: everything compressed + * at a ~4x DEFLATE ratio, best case for text/JSON, but storage files vary + * wildly (raw text compresses well, already-compressed images/videos don't). + * The same divisor applies to both since the file types aren't visible from + * here; the actual zip lands somewhere inside the range. + */ +export function estimateZipSize( + backup: AppBackup, +): { min: number; max: number } | null { + const backupBytes = backup.uncompressedSize ?? backup.dbSize; + if (backupBytes == null || backup.filesSize == null) return null; + const max = backupBytes + backup.filesSize; + return { min: Math.round(max / 4), max }; +} + +function authHeaders(token: string): Record { + return { + authorization: `Bearer ${token}`, + 'Instant-Platform-Version': version, + 'Instant-Core-Version': coreVersion, + 'X-Instant-Source': 'platform-sdk', + 'X-Instant-Version': version, + }; +} + +async function apiError(res: Response): Promise { + const body = await res.text(); + try { + return new InstantAPIError({ status: res.status, body: JSON.parse(body) }); + } catch (_e) { + return new InstantAPIError({ + status: res.status, + body: { type: undefined, message: body }, + }); + } +} + +async function* ndjsonLines( + body: ReadableStream, +): AsyncGenerator { + const reader = body.getReader(); + try { + const decoder = new TextDecoder(); + let buf = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + let nl = buf.indexOf('\n'); + while (nl !== -1) { + const line = buf.slice(0, nl).trim(); + buf = buf.slice(nl + 1); + if (line.length > 0) { + yield JSON.parse(line); + } + nl = buf.indexOf('\n'); + } + } + const line = buf.trim(); + if (line.length > 0) { + yield JSON.parse(line); + } + } finally { + // Also releases the connection when the consumer stops iterating early. + await reader.cancel().catch(() => {}); + } +} + +/** + * Read-only API for an app's backups. + * + * A backup archive has a canonical entry order that restore relies on: + * `config.json` first, then every `entities/.jsonl` shard, then the + * `files/` storage blobs. In particular ALL entity files must + * come before ANY storage file, so a restore can process the archive in a + * single streaming pass: `config.json` sets up the schema, the `$files` + * entities register file metadata, and only then can each blob be matched + * to its entity. {@link downloadArchive} implements that order end to end; + * {@link listFiles} (which returns the entity files already in write order) + * and {@link streamStorageFiles} are the pieces for building a custom + * pipeline. + */ +export class BackupsManager { + #appId: string; + #apiURI: string; + #withAuth: WithAuth; + + constructor(opts: { appId: string; apiURI: string; withAuth: WithAuth }) { + this.#appId = opts.appId; + this.#apiURI = opts.apiURI; + this.#withAuth = opts.withAuth; + } + + #getJson(path: string, signal?: AbortSignal): Promise { + return this.#withAuth(async (token) => { + const res = await fetch(`${this.#apiURI}${path}`, { + headers: authHeaders(token), + signal, + }); + if (res.status !== 200) { + throw await apiError(res); + } + return res.json(); + }); + } + + /** + * Returns the app's downloadable (non-expired) backups, newest first. + */ + async list(opts?: { signal?: AbortSignal }): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups`, + opts?.signal, + ); + return ((res.backups || []) as AppBackupResponse[]).map(toAppBackup); + } + + /** + * Returns the backup's entity files (`config.json` and the + * `entities/.jsonl` shards) in archive write order. Storage blobs + * are not included; discover those with {@link streamStorageFiles}. + */ + async listFiles( + backupId: string, + opts?: { signal?: AbortSignal }, + ): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups/${backupId}/files`, + opts?.signal, + ); + return (res.files || []) as AppBackupFile[]; + } + + /** + * Returns a presigned download URL for one of the backup's entity files. + * The URL expires after 1 hour, so fetch it right before downloading. + * + * The entity files are stored zstd-compressed (the response carries + * `Content-Encoding: zstd`); decompress to get the raw JSON/JSONL. + */ + async getFileUrl( + backupId: string, + name: string, + opts?: { signal?: AbortSignal }, + ): Promise { + const res = await this.#getJson( + `/dash/apps/${this.#appId}/backups/${backupId}/file-url?name=${encodeURIComponent(name)}`, + opts?.signal, + ); + return res.url as string; + } + + /** + * Streams every storage file captured in the backup, each with a presigned + * download URL. Completes without yielding anything when the app has no + * storage files. + * + * The server ends a healthy stream with a terminal sentinel; if the stream + * closes without it (a server-side failure truncated the listing), this + * throws instead of silently under-reporting files. + */ + async *streamStorageFiles( + backupId: string, + opts?: { signal?: AbortSignal }, + ): AsyncGenerator { + const res = await this.#withAuth(async (token) => { + const res = await fetch( + `${this.#apiURI}/dash/apps/${this.#appId}/backups/${backupId}/storage-files`, + { headers: authHeaders(token), signal: opts?.signal }, + ); + if (res.status !== 200) { + throw await apiError(res); + } + return res; + }); + if (!res.body) { + throw new Error('Storage file listing returned no body.'); + } + let complete = false; + for await (const line of ndjsonLines(res.body)) { + if (line.done === true) { + complete = true; + break; + } + // A file record always carries a locationId and url; anything else is + // corruption, and skipping it would silently omit a file from the + // archive. + if ( + typeof line.locationId !== 'string' || + line.locationId.length === 0 || + // The locationId becomes the archive entry path `files/`; + // reject separators and control characters that could escape it. + /[/\\\u0000-\u001f\u007f]/.test(line.locationId) || + line.locationId === '.' || + line.locationId === '..' || + typeof line.url !== 'string' || + line.url.length === 0 + ) { + throw new Error( + 'Storage file listing returned a malformed record. Please retry the download.', + ); + } + yield { + locationId: line.locationId, + path: line.path ?? null, + url: line.url, + }; + } + if (!complete) { + throw new Error( + 'Storage file listing ended before it finished. Please retry the download.', + ); + } + } + + /** + * Downloads the backup into a single zip archive written to `opts.sink`, + * entries in the canonical restore order described above. The caller + * supplies the runtime-specific pieces — how to fetch a presigned URL, + * where the bytes go, and the archive encoder (e.g. zip.js's `ZipWriter`); + * see {@link DownloadBackupArchiveOpts}. + */ + downloadArchive( + opts: DownloadBackupArchiveOpts, + ): Promise { + return downloadBackupArchive({ manager: this, ...opts }); + } +} diff --git a/client/packages/platform/src/index.ts b/client/packages/platform/src/index.ts index 3181916c50..34143e8b52 100644 --- a/client/packages/platform/src/index.ts +++ b/client/packages/platform/src/index.ts @@ -115,6 +115,24 @@ export { type Identifier, } from './migrations.ts'; +export { + BackupsManager, + backupZipName, + estimateZipSize, + formatFileSize, + toAppBackup, + type AppBackup, + type AppBackupFile, + type AppBackupStorageFile, +} from './backups.ts'; + +export { + type DownloadBackupArchiveOpts, + type BackupArchiveWriter, + type BackupDownloadProgress, + type BackupDownloadResult, +} from './backupDownload.ts'; + export { DEFAULT_OAUTH_CALLBACK_URL, oauthCallbackURL, diff --git a/client/packages/version/src/version.ts b/client/packages/version/src/version.ts index 063bd17103..348189d998 100644 --- a/client/packages/version/src/version.ts +++ b/client/packages/version/src/version.ts @@ -2,6 +2,6 @@ // Update the version here and merge your code to main to // publish a new version of all of the packages to npm. -const version = 'v1.0.60'; +const version = 'v1.0.61'; export { version }; diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index 6eec6aedc3..05b726b299 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@instantdb/version': specifier: workspace:* version: link:../version + '@zip.js/zip.js': + specifier: ^2.8.34 + version: 2.8.34 acorn: specifier: ^8.15.0 version: 8.15.0 @@ -8960,7 +8963,7 @@ packages: '@xmldom/xmldom@0.7.13': resolution: {integrity: sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g==} engines: {node: '>=10.0.0'} - deprecated: this version has critical issues, please update to the latest version + deprecated: this version is no longer supported, please update to at least 0.8.* '@xmldom/xmldom@0.8.10': resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} @@ -25743,7 +25746,7 @@ snapshots: metro: 0.83.6 metro-config: 0.83.6 metro-core: 0.83.6 - semver: 7.7.4 + semver: 7.8.5 optionalDependencies: '@react-native-community/cli': 15.1.3(typescript@5.9.3) transitivePeerDependencies: @@ -36166,7 +36169,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36213,7 +36216,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36261,7 +36264,7 @@ snapshots: react-refresh: 0.14.2 regenerator-runtime: 0.13.11 scheduler: 0.26.0 - semver: 7.7.4 + semver: 7.8.5 stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 ws: 6.2.3 @@ -36954,8 +36957,7 @@ snapshots: semver@7.7.4: {} - semver@7.8.5: - optional: true + semver@7.8.5: {} send@0.18.0: dependencies: @@ -38993,7 +38995,7 @@ snapshots: esbuild: 0.27.3 fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 - postcss: 8.5.23 + postcss: 8.5.8 rollup: 4.59.0 tinyglobby: 0.2.15 optionalDependencies: diff --git a/client/www/components/dash/BackupDownloadDialog.tsx b/client/www/components/dash/BackupDownloadDialog.tsx index e70a65d746..b4e2f5ac6c 100644 --- a/client/www/components/dash/BackupDownloadDialog.tsx +++ b/client/www/components/dash/BackupDownloadDialog.tsx @@ -10,8 +10,17 @@ import { } from 'react'; import { ArrowsPointingOutIcon, XMarkIcon } from '@heroicons/react/24/outline'; +import { + BackupsManager, + backupZipName, + estimateZipSize, + formatFileSize, + toAppBackup, + type AppBackup, + type BackupDownloadProgress, +} from '@instantdb/platform'; + import config from '@/lib/config'; -import { jsonFetch } from '@/lib/fetch'; import { messageFromInstantError } from '@/lib/errors'; import { InstantApp, InstantAppBackup, InstantIssue } from '@/lib/types'; @@ -20,83 +29,12 @@ import { Button, Content, Dialog, SubsectionHeading } from '@/components/ui'; import { formatTimestamp } from '@/components/dash/shared'; import { useDarkMode } from '@/components/dash/DarkModeToggle'; -type BackupFile = { name: string; size: number }; - -type DownloadProgress = { - entitiesCompleted: number; - entitiesTotal: number | null; - filesCompleted: number; - filesTotal: number | null; - // Compressed bytes written to disk so far (the zip's on-disk size). - bytes: number; - // Uncompressed bytes read from source bodies, and the backup's known - // uncompressed total — the numerator/denominator for the % progress bar. - // bytesTotal is null when the backup row carries no sizes. - bytesRead: number; - bytesTotal: number | null; - // Empty while we aren't actively fetching from that phase — clear once - // the yield is done so the line stops claiming a file after completion. - currentEntity: string; - currentFile: string; +type DownloadProgress = BackupDownloadProgress & { + // The picker-chosen destination name, folded into progress so the dialog + // and pill can label where the zip is going. outputFilename: string; }; -function backupZipName(backup: InstantAppBackup): string { - const safe = backup.backup_at.replace(/[:.]/g, '-'); - return `instant-backup-${safe}.zip`; -} - -function formatBytes(n: number): string { - // Decimal (1000-based) units with SI labels, to match how macOS/Finder - // reports file sizes so the number lines up with what lands on disk. - if (n < 1000) return `${n} B`; - const units = ['KB', 'MB', 'GB', 'TB']; - let i = -1; - let v = n; - do { - v /= 1000; - i++; - } while (v >= 1000 && i < units.length - 1); - const digits = v < 10 ? 2 : v < 100 ? 1 : 0; - return `${v.toFixed(digits)} ${units[i]}`; -} - -async function fetchFiles( - token: string, - appId: string, - backupId: string, - signal: AbortSignal, -): Promise { - // XXX: Why are you using the authed fetch hook here? - const { files } = (await jsonFetch( - `${config.apiURI}/dash/apps/${appId}/backups/${backupId}/files`, - { headers: { authorization: `Bearer ${token}` }, signal }, - )) as { files: BackupFile[] }; - return files; -} - -async function fetchFileUrl( - token: string, - appId: string, - backupId: string, - name: string, - signal: AbortSignal, -): Promise { - // XXX: Why are you using the authed fetch hook here? - const url = `${config.apiURI}/dash/apps/${appId}/backups/${backupId}/file-url?name=${encodeURIComponent(name)}`; - const { url: signed } = (await jsonFetch(url, { - headers: { authorization: `Bearer ${token}` }, - signal, - })) as { url: string }; - return signed; -} - -type StorageFileLine = { - locationId: string; - path: string; - url: string; -}; - type DownloadResult = | { via: 'picker'; filename: string } | { via: 'browser-default'; filename: string }; @@ -105,12 +43,6 @@ type SaveFileHandle = Awaited< ReturnType >; -type ZipEntry = { - name: string; - lastModified: Date; - input: ReadableStream; -}; - async function registerDownloadServiceWorker(): Promise { if (typeof navigator === 'undefined' || !('serviceWorker' in navigator)) { return; @@ -144,7 +76,7 @@ async function downloadBackup( showSaveFilePicker: typeof import('native-file-system-adapter').showSaveFilePicker, token: string, appId: string, - backup: InstantAppBackup, + backup: AppBackup, setProgress: (p: DownloadProgress) => void, abortController: AbortController, ): Promise { @@ -162,318 +94,50 @@ async function downloadBackup( ], }); - // ---- Shared progress state ---- - // current = files (backup + storage) fully fetched. - // total = backup file count + storage files ever discovered. - // `storageQueue` is a FIFO of yet-to-fetch storage files only — dequeued as - // the generator processes them, so it stays bounded by (discovery rate - - // fetch rate). It's an object with head/tail indices rather than an array - // because discovery streams the whole list in up front: Array.shift() is O(n), - // so draining a large backup would be O(n²). Object access is O(1) and delete - // frees each slot as it's consumed. - // Two parallel counters surfaced in the dialog: - // entities — the backup payload (config.json + per-etype JSONL shards). - // files — the user's $files (storage uploads). - // entitiesTotal is null until fetchFiles resolves; filesTotal is null - // until the discovery NDJSON returns at least one line OR completes - // (404/empty body sets it to 0 so the dialog shows "0 of 0 files"). - let entitiesCompleted = 0; - let entitiesTotal: number | null = null; - let filesCompleted = 0; - let filesTotal: number | null = null; - let currentEntity = ''; - let currentFile = ''; - let outputFilename = pickerHandle?.name ?? filename; - let zipBytes = 0; - let uncompressedRead = 0; - // Denominator for the % bar: uncompressed entities + storage files, since we - // read (and count) both. Gated on uncompressed_size — no size, no bar — with - // files_size added when present. - const bytesTotal = - backup.uncompressed_size != null - ? backup.uncompressed_size + (backup.files_size ?? 0) - : null; - const storageQueue: Record = {}; - let queueHead = 0; - let queueTail = 0; - let storageDone = false; - let storageError: Error | null = null; - let waitResolve: (() => void) | null = null; - - // The caller-supplied AbortController is shared across every fetch we - // own (the NDJSON discovery stream, backup file body fetches, storage - // file body fetches). The caller can abort it externally (e.g. dialog - // close) and we also abort it from our own catch so the discovery stops - // pulling from S3 if the zip pipeline throws. - const signal = abortController.signal; - - const tick = () => - setProgress({ - entitiesCompleted, - entitiesTotal, - filesCompleted, - filesTotal, - bytes: zipBytes, - bytesRead: uncompressedRead, - bytesTotal, - currentEntity, - currentFile, - outputFilename, - }); - - // Count uncompressed bytes as each source body streams through, for the % - // bar. This is the input size (matches backup.uncompressed_size); zipBytes - // is the compressed output written to disk. - const countRead = ( - body: ReadableStream, - ): ReadableStream => - body.pipeThrough( - new TransformStream({ - transform(chunk, controller) { - uncompressedRead += chunk.byteLength; - controller.enqueue(chunk); - }, - }), - ); - - const notify = () => { - const w = waitResolve; - waitResolve = null; - w?.(); - }; - - // Throttle progress updates by time: a large backup pushes many small chunks - // (and a large listing many discovery lines) and ticking (a React setState) - // on every one would flood re-renders. Ticking at most every 100ms stays - // smooth at any size. Phase changes and completion tick() directly so they're - // still immediate. - const TICK_INTERVAL_MS = 100; - let lastTickAt = 0; - const throttledTick = () => { - const now = Date.now(); - if (now - lastTickAt >= TICK_INTERVAL_MS) { - lastTickAt = now; - tick(); - } - }; - - // Kick off storage-files discovery in parallel with the backup file list. - // It pushes onto storagePending and bumps `storageDiscovered` as URLs - // stream in. - void (async () => { - // Set once we see the server's terminal `done` sentinel (or a 404, which - // means there's no $files shard at all). If discovery ends without it and - // we weren't aborted, the stream was truncated by a server-side failure. - let storageComplete = false; - try { - const res = await fetch( - `${config.apiURI}/dash/apps/${appId}/backups/${backup.id}/storage-files`, - { headers: { authorization: `Bearer ${token}` }, signal }, - ); - if (res.status === 404) { - filesTotal = 0; - storageComplete = true; - tick(); - return; - } - if (!res.ok || !res.body) { - throw new Error(`Failed to list storage files: ${res.status}`); - } - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - const consume = (line: string) => { - const trimmed = line.trim(); - if (trimmed.length === 0) return; - const obj = JSON.parse(trimmed) as StorageFileLine & { - done?: boolean; - }; - if (obj.done) { - storageComplete = true; - return; - } - if (!obj.locationId || !obj.url) return; - storageQueue[queueTail++] = obj; - filesTotal = (filesTotal ?? 0) + 1; - throttledTick(); - notify(); - }; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let nl = buf.indexOf('\n'); - while (nl !== -1) { - consume(buf.slice(0, nl)); - buf = buf.slice(nl + 1); - nl = buf.indexOf('\n'); - } - } - consume(buf); - } catch (e) { - // The abort path is expected when the zip pipeline failed and we - // tore the discovery down - if ((e as { name?: string })?.name !== 'AbortError') { - storageError = e as Error; - } - } finally { - // Discovery finished without ever calling consume (empty stream) — - // surface "0 of 0" so the dialog doesn't sit on "?" forever. - if (filesTotal == null) filesTotal = 0; - // No `done` sentinel and we weren't aborted → the server closed the - // pipe early. Fail loudly instead of building a partial zip. Don't - // clobber a more specific error already caught above. - if (!storageComplete && storageError == null && !signal.aborted) { - storageError = new Error( - 'Storage file listing ended before it finished. Please retry the download.', - ); - } - storageDone = true; - tick(); - notify(); - } - })(); + // The dash token can't refresh mid-download, so withAuth just supplies it. + const manager = new BackupsManager({ + appId, + apiURI: config.apiURI, + withAuth: (operation) => operation(token), + }); - let files: BackupFile[]; - try { - files = await fetchFiles(token, appId, backup.id, signal); - if (files.length === 0) { - throw new Error('No files found for this backup.'); - } - // We write entries in the order the server returns them, and restore - // requires config.json to be the first entry. Fail loudly rather than - // build a zip that can't be restored. - if (files[0].name !== 'config.json') { - throw new Error( - `Backup files came back in an unexpected order (expected config.json first, got "${files[0].name}").`, - ); - } - } catch (e) { - // Entity discovery failed before we reached the zip pipeline's own - // teardown. Stop the background storage-files discovery stream so it - // doesn't keep pulling from S3 after we bail. - abortController.abort(); - throw e; - } - // config.json isn't a namespace — count only the entities/*.jsonl shards so - // the "N namespaces" label isn't off by one. - entitiesTotal = files.filter((f) => f.name !== 'config.json').length; - tick(); - - // Entry write order is significant for restore: config first, then the - // entities/*.jsonl shards, then files/${locationId}. In particular ALL entity - // files must be written before ANY storage file. This generator preserves - // that: it yields the entity `files` (config + entities/*.jsonl, in the order - // fetchFiles returns them) to completion, then drains the storage queue. - const entries = (async function* (): AsyncGenerator { - for (const f of files) { - currentEntity = f.name; - tick(); - const url = await fetchFileUrl(token, appId, backup.id, f.name, signal); + const outputFilename = pickerHandle?.name ?? filename; + const writable = await pickerHandle.createWritable(); + + // The shared pipeline owns ordering, progress, backpressure, and teardown; + // this call supplies the browser-specific pieces. The caller's + // AbortController reaches every fetch the pipeline makes — the caller + // aborts it externally (e.g. cancel) and the pipeline tears everything + // down itself on failure, so the discovery stream stops pulling from S3 + // either way. + await manager.downloadArchive({ + backup, + // The entity shards' `Content-Encoding: zstd` is decoded transparently + // by the browser's fetch. + fetchBody: async (url, signal) => { const res = await fetch(url, { signal }); if (!res.ok) { - throw new Error(`Failed to fetch ${f.name}: ${res.status}`); - } - if (f.name !== 'config.json') entitiesCompleted++; - tick(); - yield { - name: f.name, - lastModified: new Date(backup.backup_at), - input: countRead(res.body!), - }; - } - // Entity phase done — clear so the dialog stops claiming we're still - // downloading an entity file once we move into storage. - currentEntity = ''; - tick(); - - while (true) { - if (storageError) throw storageError; - let obj: StorageFileLine | undefined; - if (queueHead < queueTail) { - obj = storageQueue[queueHead]; - delete storageQueue[queueHead]; - queueHead++; + throw new Error(`HTTP ${res.status}`); } - if (obj) { - currentFile = obj.path || obj.locationId; - tick(); - const fileRes = await fetch(obj.url, { signal }); - if (!fileRes.ok) { - throw new Error( - `Couldn't download storage file "${obj.path}" (HTTP ${fileRes.status}).`, - ); - } - filesCompleted++; - tick(); - yield { - name: `files/${obj.locationId}`, - lastModified: new Date(backup.backup_at), - input: countRead(fileRes.body!), - }; - } else if (storageDone) { - break; - } else { - await new Promise((resolve) => { - waitResolve = resolve; - }); - } - } - // Storage phase done, clear so the dialog stops showing the last file. - currentFile = ''; - tick(); - - if (storageError) throw storageError; - })(); - - let diskWriter: WritableStreamDefaultWriter | null = null; - try { - // Load the zip encoder on demand (kept out of the shared bundle). Inside the - // try so a chunk-load failure aborts the background discovery too. - const { ZipWriter } = await import('@zip.js/zip.js'); - const writable = await pickerHandle.createWritable(); - const writer = writable.getWriter(); - diskWriter = writer; - // Sink that zip.js writes compressed bytes into: it tallies the on-disk - // (compressed) size for progress, then forwards to the file. Awaiting the - // disk write propagates backpressure up into zip.js, so a fast source can't - // outrun the disk and balloon memory. - const countingSink = new WritableStream({ - async write(chunk) { - zipBytes += chunk.byteLength; - throttledTick(); - await writer.write(chunk); - }, - async close() { - await writer.close(); - tick(); - }, - async abort(reason) { - await writer.abort(reason); - }, - }); - - // zip64: without it any archive whose central-directory offset passes 4GB - // writes a wrapped 32-bit offset and the zip is unreadable. Everything else - // is left at zip.js's defaults: workers + CompressionStream when available - // (compression and CRC32 run off the main thread so a multi-GB backup - // doesn't jank the tab), each degrading to an inline main-thread codec on - // browsers that lack them — correctness (incl. zip64) is unaffected either way. - const zipWriter = new ZipWriter(countingSink, { zip64: true, signal }); - for await (const entry of entries) { - await zipWriter.add(entry.name, entry.input, { - lastModDate: entry.lastModified, - }); - } - await zipWriter.close(); - return { via: 'picker', filename: pickerHandle.name ?? filename }; - } catch (e) { - // Tear down the background discovery and any in-flight body fetches so we - // don't keep pulling from S3, and discard the partial file on disk. - abortController.abort(); - await diskWriter?.abort(e).catch(() => {}); - throw e; - } + return res.body!; + }, + sink: writable, + createWriter: async (sink, signal) => { + // Load the zip encoder on demand (kept out of the shared bundle). + const { ZipWriter } = await import('@zip.js/zip.js'); + // zip64: without it any archive whose central-directory offset passes + // 4GB writes a wrapped 32-bit offset and the zip is unreadable. + // Everything else is left at zip.js's defaults: workers + + // CompressionStream when available (compression and CRC32 run off the + // main thread so a multi-GB backup doesn't jank the tab), each + // degrading to an inline main-thread codec on browsers that lack them — + // correctness (incl. zip64) is unaffected either way. + return new ZipWriter(sink, { zip64: true, signal }); + }, + signal: abortController.signal, + onProgress: (p) => setProgress({ ...p, outputFilename }), + }); + return { via: 'picker', filename: pickerHandle.name ?? filename }; } type DownloadDialogState = @@ -609,6 +273,9 @@ function DownloadInstance({ onDone: () => void; }) { const { darkMode } = useDarkMode(); + // The dash API hands us the raw server row; the platform helpers + // (downloadArchive, backupZipName, estimateZipSize) take the parsed shape. + const appBackup = useMemo(() => toAppBackup(backup), [backup]); const [state, setState] = useState({ kind: 'confirm' }); const abortRef = useRef(null); // Set when the user cancels. The download promise rejects asynchronously with @@ -674,18 +341,7 @@ function DownloadInstance({ // (see handleClose) while the state machine keeps running; dismissing tears // the whole instance down via onDone. - // Upper bound: everything stored uncompressed (STORE mode and/or files - // that don't compress). Lower bound: everything compressed at a ~3x - // DEFLATE ratio, best case for text/JSON, but storage files vary wildly - // (raw text compresses well, already-compressed images/videos don't). - // We apply the same divisor to both since we can't see the file types - // from here; the actual zip will land somewhere inside the range. - const backupBytes = backup.uncompressed_size ?? backup.db_size; - const filesBytes = backup.files_size; - const hasSizes = backupBytes != null && filesBytes != null; - const totalBytes = (backupBytes ?? 0) + (filesBytes ?? 0); - const maxBytes = totalBytes; - const minBytes = Math.round(totalBytes / 4); + const sizeEstimate = estimateZipSize(appBackup); const start = async () => { const showSaveFilePicker = pickerRef.current; @@ -698,7 +354,7 @@ function DownloadInstance({ showSaveFilePicker, token, app.id, - backup, + appBackup, (progress) => { setState((prev) => prev.kind === 'downloading' ? { ...prev, progress } : prev, @@ -781,12 +437,12 @@ function DownloadInstance({ {formatTimestamp(backup.backup_at)}. This will download a zip file containing all entities and all files. - {hasSizes ? ( + {sizeEstimate ? ( The zip file will be between{' '} - {formatBytes(minBytes)} and{' '} - {formatBytes(maxBytes)}, depending on the - compression ratio. + {formatFileSize(sizeEstimate.min)} and{' '} + {formatFileSize(sizeEstimate.max)}, depending on + the compression ratio. ) : null} {pickerError ? ( @@ -855,7 +511,7 @@ function DownloadInstance({ {progress?.outputFilename ?? ''} - {formatBytes(progress?.bytes ?? 0)} + {formatFileSize(progress?.zipBytes ?? 0)} {pct != null ? ` · ${Math.round(pct)}%` : ''} @@ -902,11 +558,11 @@ function DownloadInstance({ Saved to{' '} - {progress?.outputFilename ?? backupZipName(backup)} + {progress?.outputFilename ?? backupZipName(appBackup)} - {formatBytes(progress?.bytes ?? 0)} + {formatFileSize(progress?.zipBytes ?? 0)} @@ -972,7 +628,7 @@ function DownloadInstance({ ) : null} - {progress?.outputFilename ?? backupZipName(backup)} + {progress?.outputFilename ?? backupZipName(appBackup)} {state.kind === 'downloading' && pct != null ? (