diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 7e35c0db04e..d5ca0781234 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -72,6 +72,13 @@ explicit path or endpoint and do not need it. argument strings and mutability as inferred candidates; this performs no chain lookup or target execution. See the [EVM bytecode guide](https://github.com/morluto/rea/blob/main/docs/evm-bytecode.md). +- Explicit local SQLite database snapshot: `inspect_sqlite_database`. Inspect + with Node.js 24.x >=24.15 or 26+ providing native SQLite limits. Read schema + first or select an exact ordinary table and `row_limit` when rows are + needed. Preserve both database and existing sibling WAL identity; row samples + report truncation and have unspecified order. Use a quiescent snapshot rather + than assuming a live database/WAL pair was copied atomically. See the + [SQLite guide](https://github.com/morluto/rea/blob/main/docs/sqlite-databases.md). - Explicit Linux ELF file for offline layout, symbols, relocations or static mitigation evidence: `inspect_binary_layout`. This target-free operation uses caller-supplied pwntools without opening a disassembler database. Preserve its diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 0a281067663..3d2829a28ab 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -1,5 +1,17 @@ # Native, managed, and packaged artifacts +## SQLite snapshots + +Use `inspect_sqlite_database` with an explicit local database snapshot for its +schema and optional rows from one exact ordinary table. It requires Node.js +24.x >=24.15 or 26+ with the native `DatabaseSync.limits` API. It also consumes the +existing sibling `-wal`; retain both original digests when citing its result. +Choose `row_limit` when a finite sample is useful and keep truncation and +unspecified row order explicit. Virtual tables, shadow tables and views are +schema facts, not row-read targets. REA inspects private copies and leaves +the source files untouched; obtain a coherent quiescent database/WAL snapshot +before inspection. See [SQLite snapshots](https://github.com/morluto/rea/blob/main/docs/sqlite-databases.md). + ## Recorded Linux crashes Use `inspect_recorded_crash` with an explicit Linux x86-64 ELF core path to diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b55d8dcaf7e..346709c6d43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,11 @@ jobs: if-no-files-found: error retention-days: 7 + sqlite-runtime: + needs: [changes, build] + if: needs.changes.outputs.code == 'true' + uses: ./.github/workflows/sqlite-runtime.yml + test: name: Test (coverage + timing) needs: [changes, test-shard] @@ -561,6 +566,7 @@ jobs: static, build, test-shard, + sqlite-runtime, test, conformance-fixtures, windows-native-build, diff --git a/.github/workflows/sqlite-runtime.yml b/.github/workflows/sqlite-runtime.yml new file mode 100644 index 00000000000..8b3ba107f27 --- /dev/null +++ b/.github/workflows/sqlite-runtime.yml @@ -0,0 +1,43 @@ +name: SQLite runtime compatibility + +on: + workflow_call: + +permissions: + contents: read + +env: + HUSKY: "0" + +jobs: + sqlite: + name: SQLite runtime compatibility (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Exercise unavailable-runtime diagnostics, the first supported release, + # and new patches on both supported lines without hand-editing versions. + # The repository-pinned Node 24 version also runs the complete test suite. + node: ["22.x", "24.15.0", "24.x", "26.x"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + check-latest: true + cache: npm + - run: npm ci + - uses: actions/download-artifact@v4 + with: + name: rea-dist + path: dist + - name: Generate test artifacts from the downloaded runtime + run: | + node scripts/generate-mcp-tool-catalog.mjs + node scripts/generate-skill-metadata.mjs + node scripts/generate-product-catalog.mjs + node scripts/generate-completion-ledger.mjs + - name: Verify SQLite discovery, inspection and cleanup + run: npm run test:sqlite:run diff --git a/README.md b/README.md index 8bdfb2739ed..36488a33395 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ Additional tools and host support depend on the target: | Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](https://rea.tools/guides/native/) | | Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | | EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | +| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | | Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | | JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; [application analysis](https://rea.tools/guides/javascript/) | | Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](https://rea.tools/guides/browser/) | diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e1c184993f6..1e00f35466e 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -41,6 +41,7 @@ export default defineConfig({ { text: "Android applications", link: "/android-analysis" }, { text: "Apple applications", link: "/apple-application-analysis" }, { text: "Firmware", link: "/firmware-analysis" }, + { text: "SQLite databases", link: "/sqlite-databases" }, { text: "Reconstruction readiness", link: "/reconstruction-readiness", diff --git a/docs/cli.md b/docs/cli.md index 7f35eb40d54..3df30219c71 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -37,6 +37,14 @@ module page. See [JavaScript application workflows](javascript-application-workflows.md) for views, feature traces and comparisons that take the same saved Evidence. +## SQLite snapshots + +For a local database snapshot, use `rea inspect-sqlite-database PATH --json`. +Add `--table NAME --row-limit 20` to read a bounded sample from an ordinary +table. This command does not require a native analysis provider. See +[SQLite database snapshots](sqlite-databases.md) for database/WAL identity, +source-file handling and row-value semantics. + ## Native analysis Configure [Hopper or Ghidra](installation.md#hopper), or the diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md new file mode 100644 index 00000000000..a098d5c1eb7 --- /dev/null +++ b/docs/sqlite-databases.md @@ -0,0 +1,121 @@ +# SQLite database snapshots + +`inspect_sqlite_database` / `inspect-sqlite-database` inspects one explicitly +selected local SQLite database snapshot without an active native analysis +target. Use it to understand application storage, configuration or retained +database evidence. No separate SQLite installation is required. + +This operation requires Node.js 24.x >=24.15 or 26+ with the native +`DatabaseSync.limits` API. The worker verifies that capability before querying +the copied database. Other REA features retain their published Node.js 22.x, +24.x or 26+ requirements. + +`rea capabilities` reports `inspect_sqlite_database` availability and the +runtime requirement without opening a database. Unsupported runtimes show +`available: false`, `availability_code: "unsupported_version"`, and an upgrade +note while the inventory command still succeeds. + +```sh +rea capabilities --json +rea inspect-sqlite-database ./application.sqlite --json +rea inspect-sqlite-database ./application.sqlite --table settings --row-limit 20 --json +``` + +```json +{ + "name": "inspect_sqlite_database", + "arguments": { + "path": "/artifacts/application.sqlite", + "table": "settings", + "row_limit": 20 + } +} +``` + +MCP paths must be absolute filesystem paths on the current host. CLI paths +resolve against the operator's working directory. Select a quiescent snapshot: +stop the writer or obtain a consistent database/WAL pair using the application's +backup workflow before inspection. A file copy of a live pair does not establish +an atomic SQLite snapshot, even if no change is detected during inspection. + +## Returned evidence + +The complete schema inventory includes tables and their columns, indexes, views +and triggers. SQL declarations retain the original schema text. Virtual and +shadow tables are identified as such; their supported metadata coverage is +reported. If a virtual table cannot be resolved, possible backing tables have +`kind: "unknown"` and an explicit limitation; their rows are refused. Unrelated +ordinary tables remain selectable. Schema inventory does not execute views, +triggers or arbitrary SQL. + +Rows are optional. `table` selects one exact ordinary-table name. `row_limit` +requires `table`, defaults to 100 and accepts positive safe integers. Samples +remain subject to the byte and worker resource limits below, regardless of row count. The result reports +the selected limit, returned count and whether more rows were observed. Row +order is unspecified; a truncated sample does not prove anything about omitted +records. View, virtual-table and shadow-table row reads are unsupported. + +SQLite integers are decimal strings so every signed 64-bit value survives JSON +serialization. Real values retain their numbers; `"Infinity"`, `"-Infinity"` +and `"-0"` explicitly preserve infinities and negative zero through JSON. +Text returns the decoded string together with its bytes as hex and the database +encoding, preserving malformed text that the SQLite binding may replace for +display. Blobs are hex, and SQL NULL is a distinct tagged value. + +Database and existing sibling `-wal` files have separate original SHA-256 and +byte counts. The Evidence subject identifies the original database file; +the WAL identity is also necessary to interpret pending committed data. The +SQLite engine version is reported with the observed result. Neither digest is +a claim that the database is healthy, complete or trusted. + +## Source files and failures + +REA copies the selected database and existing sibling WAL to an owned private +temporary directory before SQLite opens them. Source files are not opened by +SQLite, so its locks or shared-memory sidecars cannot modify the original +snapshot. Temporary copies and the owned worker are cleaned up on success, +failure and cancellation. Extension loading and schema trust are disabled. +Empty rollback journals and fully zeroed journal headers left by successful +`TRUNCATE` and `PERSIST` commits are accepted after stable identity checks. +Other retained rollback journals are refused because recovery is outside this +profile. Journal files are never modified or copied into the worker snapshot. + +The database and WAL together are limited to 256 MiB. SQLite values/records, +SQL statements and the complete worker JSON reply each have a 16 MiB limit. +The owned worker has a 256 MiB JavaScript heap limit and a 30-second inspection +deadline. SQLite native allocations are separate from the JavaScript heap; +overall native heap and total resident memory are not capped. Row limits do not +override these byte and resource limits. Resource-limit failures do not return +an incomplete schema as successful complete evidence. + +MCP also applies its configured response budget to the complete Evidence +envelope. If delivery exceeds that budget, it reports a transport constraint +with guidance for exporting retained Evidence, requesting a matching larger +client/server response budget or using complete CLI JSON output. + +Changed input identities, special files, malformed databases, unsupported +formats and resource failures are reported as errors with the selected path and +meaningful reason. Encrypted databases, database repair, arbitrary SQL, +cross-database queries and live capture are outside this inspection's coverage. + +## Compatibility verification + +Run `npm run test:sqlite` after changing the SQLite adapter or its shared +process, filesystem, CLI or MCP boundaries. It prepares the compiled runtime +and runs SQLite service, snapshot, inspection, cleanup and public CLI/MCP +tests. Directory-based selection includes new SQLite service and boundary +tests automatically. `npm run test:sqlite:run` runs the same cases when the +compiled runtime and generated test artifacts are already current. + +The reusable SQLite CI workflow consumes the shared build artifact and tests +the latest Node 22.x unavailable-runtime diagnostics, the minimum supported +24.15.0 release, and the latest 24.x and 26.x releases. The repository-pinned +runtime also runs the full suite. CI's required gate waits for every selected +SQLite matrix job; a failed, cancelled, skipped or missing selected lane blocks +the gate. Workflow changes themselves select the build and compatibility lane. + +PR checks run against GitHub's merge with the current base branch. A later +change to `main` can introduce failures outside SQLite; inspect the failing +job and recheck the updated merge before treating an earlier green run as +current verification. Compatibility checks detect regressions, but cannot +guarantee that an arbitrary future upstream change remains compatible. diff --git a/package.json b/package.json index f045fe8de64..438335c92d4 100644 --- a/package.json +++ b/package.json @@ -222,6 +222,8 @@ "verify:native-calls": "npm run build:cached && node scripts/verify-native-calls.mjs", "verify:native-ui:permissions": "npm run build:cached && node scripts/verify-native-ui.mjs --allow-permission-denial", "test:focused": "node scripts/run-development-tests.mjs focused", + "test:sqlite": "npm run test:prepare && npm run test:sqlite:run", + "test:sqlite:run": "vitest run src/application/sqlite/ tests/boundary/sqlite/ tests/acceptance/sqliteDatabaseCli.test.ts tests/boundary/mcp/serverLifecycleMcp.test.ts", "build:windows-native": "node scripts/build-windows-native.mjs", "verify:windows-native:artifact": "node scripts/verify-windows-native-artifact.mjs", "prepublishOnly": "npm run verify:windows-native:artifact", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 40ec03a6b35..d83f8dc7674 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -122,6 +122,8 @@ if ( file === ".github/actionlint.yaml" ) { selected.workflows = true; + // SQLite reuses this run's compiled artifact, so workflow edits need build. + if (file === ".github/workflows/sqlite-runtime.yml") selected.code = true; if (file !== ".github/workflows/ci.yml") continue; } if (owners.length === 1 && owners[0] === "website") continue; diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index fb5115b8ee8..4f65867372a 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,7 +2,7 @@ const jobs = JSON.parse(process.env.CI_JOBS); const outputs = jobs.changes.outputs; const required = ["changes", "static", "test"]; const lanes = { - code: ["build", "test-shard"], + code: ["build", "test-shard", "sqlite-runtime"], runtime: ["windows-native-build", "package-e2e"], package: ["windows-curated"], apple: ["apple-artifacts"], diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 362c2acaaa6..465cb4a145a 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -27,6 +27,7 @@ export const toolFamilyCatalog = (sources) => { contracts: [ ...sources.artifactContracts.ARTIFACT_TOOL_CONTRACTS, ...sources.evmContracts.EVM_TOOL_CONTRACTS, + ...sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, ], }, { @@ -183,6 +184,10 @@ export const providerCatalog = (sources) => { identity: sources.evmProvider.EVMOLE_PROVIDER_IDENTITY, contracts: sources.evmContracts.EVM_TOOL_CONTRACTS, }, + { + identity: sources.sqliteProvider.SQLITE_PROVIDER_IDENTITY, + contracts: sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, + }, { identity: sources.artifactProviders.ARTIFACT_GRAPH_PROVIDER, contracts: sources.artifactContracts.ARTIFACT_TOOL_CONTRACTS, diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 06b237d5129..6e5e7d86992 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -46,6 +46,8 @@ const SOURCE_PATHS = { pwntoolsProvider: "dist/native/pwntools/PwntoolsRelease.js", nativeContracts: "dist/contracts/native/nativeToolContracts.js", evmContracts: "dist/contracts/evm/evmToolContracts.js", + sqliteContracts: "dist/contracts/sqlite/sqliteToolContracts.js", + sqliteProvider: "dist/sqlite/SqliteDatabaseLimits.js", evmProvider: "dist/evm/EvmoleRelease.js", artifactContracts: "dist/contracts/artifactToolContracts.js", managedContracts: "dist/contracts/managed/managedToolContracts.js", diff --git a/scripts/lib/module-boundaries.mjs b/scripts/lib/module-boundaries.mjs index 2948cae3035..d47f5477fae 100644 --- a/scripts/lib/module-boundaries.mjs +++ b/scripts/lib/module-boundaries.mjs @@ -5,6 +5,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const PURE_LAYERS = new Set(["domain", "contracts"]); const MIGRATED_PROVIDER_ROOTS = new Set([ + "sqlite", "evm", "android", "firmware", diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 7d3231d90d9..3ddd1a02a5f 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js"; import type { OptionalProviderLoadFailures, OptionalProviderLoadFailure, @@ -270,6 +271,15 @@ const workflowAvailabilityFor = ({ kind, policy, }: AvailabilityContext): Availability | null => { + if (name === "inspect_sqlite_database") { + const capability = sqliteDatabaseCapability(process.versions.node); + return capability.available + ? { reason: "available", remediation: null } + : { + reason: "provider_unavailable", + remediation: capability.reason, + }; + } if (name === "trace_web_module_imports") return { reason: "available", diff --git a/src/application/DirectAnalysisStatus.ts b/src/application/DirectAnalysisStatus.ts index cc6df790115..db0a220926e 100644 --- a/src/application/DirectAnalysisStatus.ts +++ b/src/application/DirectAnalysisStatus.ts @@ -4,6 +4,8 @@ import { createServerIdentity } from "../serverIdentity.js"; import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js"; +import { providerCapability } from "../contracts/toolOutputSchemaPrimitives.js"; +import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js"; /** Read target-free provider status and operation availability. */ export const runSessionStatus = async ( @@ -17,8 +19,13 @@ export const runSessionStatus = async ( if (!config.ok) return { error: projectAnalysisError(config.error) }; const session = dependencies.createBinarySession(config.value, logger); try { + const status = jsonObjectSchema.parse(session.status()); return { - ...jsonObjectSchema.parse(session.status()), + ...status, + capabilities: [ + ...providerCapability.array().parse(status.capabilities), + sqliteDatabaseCapability(process.versions.node), + ].sort((left, right) => left.operation.localeCompare(right.operation)), server_identity: createServerIdentity({ startedAt: new Date().toISOString(), }), diff --git a/src/application/sqlite/SqliteDatabaseCapability.test.ts b/src/application/sqlite/SqliteDatabaseCapability.test.ts new file mode 100644 index 00000000000..1946cf70407 --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseCapability.test.ts @@ -0,0 +1,30 @@ +import { expect, it } from "vitest"; +import { providerCapability } from "../../contracts/toolOutputSchemaPrimitives.js"; +import { sqliteDatabaseCapability } from "./SqliteDatabaseCapability.js"; + +it.each([ + ["22.23.1", false], + ["24.11.0", false], + ["24.14.9", false], + ["24.15.0", true], + ["25.1.0", false], + ["26.0.0", true], + ["24.15.0-rc.1", false], + ["invalid", false], +])( + "reports SQLite inspection availability on Node %s as %s", + (version, available) => { + const capability = providerCapability.parse( + sqliteDatabaseCapability(version), + ); + expect(capability).toMatchObject({ + operation: "inspect_sqlite_database", + available, + reason: available + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + availability_code: available ? null : "unsupported_version", + limitations: [expect.stringContaining("DatabaseSync.limits")], + }); + }, +); diff --git a/src/application/sqlite/SqliteDatabaseCapability.ts b/src/application/sqlite/SqliteDatabaseCapability.ts new file mode 100644 index 00000000000..62e6af7eeab --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseCapability.ts @@ -0,0 +1,34 @@ +import { satisfies } from "semver"; +import { toolContract } from "../../contracts/toolContracts.js"; +import type { ProviderCapability } from "../../contracts/toolOutputSchemaPrimitives.js"; + +const SQLITE_RUNTIME_REQUIREMENT = + "SQLite inspection requires Node.js 24.x >=24.15 or 26+ with the native DatabaseSync.limits API. Other REA features retain their published runtime requirements. The inspection worker verifies the API before querying the snapshot."; + +/** Discover inspection readiness without opening a database or starting a worker. */ +export const sqliteDatabaseCapability = ( + nodeVersion: string, +): ProviderCapability => { + const contract = toolContract("inspect_sqlite_database"); + const facts = { + operation: contract.name, + effects: { + mutates_artifact: contract.effects.mutatesTarget, + launches_process: contract.effects.launchesProcess, + may_show_ui: contract.effects.changesUiState, + may_access_network: contract.effects.accessesNetwork, + may_write_filesystem: contract.effects.writesFilesystem, + changes_permissions: false, + requires_root: false, + }, + limitations: [SQLITE_RUNTIME_REQUIREMENT], + }; + return satisfies(nodeVersion, "^24.15.0 || >=26.0.0") + ? { ...facts, available: true, reason: null, availability_code: null } + : { + ...facts, + available: false, + reason: SQLITE_RUNTIME_REQUIREMENT, + availability_code: "unsupported_version", + }; +}; diff --git a/src/application/sqlite/SqliteDatabasePort.ts b/src/application/sqlite/SqliteDatabasePort.ts new file mode 100644 index 00000000000..ed5322b6787 --- /dev/null +++ b/src/application/sqlite/SqliteDatabasePort.ts @@ -0,0 +1,17 @@ +import type { + AnalysisExecution, + ExecutionOptions, +} from "../AnalysisProvider.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { InspectSqliteDatabaseInput } from "../../domain/sqlite/sqliteDatabase.js"; +import type { Result } from "../../domain/result.js"; + +/** Offline database inspection boundary shared by CLI and MCP. */ +export interface SqliteDatabasePort { + /** Await and retry provider-owned snapshot cleanup. */ + close?(): Promise; + inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts new file mode 100644 index 00000000000..e01c5fd96ca --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -0,0 +1,146 @@ +import { expect, it } from "vitest"; +import { SqliteDatabaseService } from "./SqliteDatabaseService.js"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { err } from "../../domain/result.js"; +import { ok } from "../../domain/result.js"; +import { createAnalysisExecution } from "../AnalysisProvider.js"; + +const fixture = () => ({ + artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 }, + wal: null, + engine: { name: "SQLite", version: "3.50.2" }, + database_encoding: "UTF-8", + schema: { + completeness: "complete", + tables: [], + indexes: [], + views: [], + triggers: [], + }, + rows: null, + limitations: [], +}); + +const execution = (value: unknown) => + createAnalysisExecution( + value, + { id: "sqlite", name: "SQLite", version: "3.50.2" }, + { + subject: { path: "/selected.db", sha256: "a".repeat(64), format: "file" }, + locations: [{ kind: "artifact-path", path: "/selected.db" }], + }, + ); + +it.each([ + { path: "relative.db" }, + { path: "/selected.db", approval: true }, + { path: "/selected.db", row_limit: 1 }, + { path: "/selected.db", table: 1 }, + { path: "/selected.db", table: "data", row_limit: 0 }, + { + path: "/selected.db", + table: "data", + row_limit: Number.MAX_SAFE_INTEGER + 1, + }, +])( + "rejects invalid SQLite selection before provider effects: %j", + async (input) => { + const service = new SqliteDatabaseService({ + inspect: () => { + throw new Error("Invalid selection must not read a database"); + }, + }); + expect(await service.inspect(input)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +it("preserves producer failures and rejects pre-cancelled requests before effects", async () => { + const failure = new AnalysisCapabilityUnavailableError( + "rea-sqlite", + "inspect_sqlite_database", + "Selected worker could not start", + ); + const service = new SqliteDatabaseService({ + inspect: () => Promise.resolve(err(failure)), + }); + expect(await service.inspect({ path: "/selected.db" })).toEqual(err(failure)); + const signal = AbortSignal.abort(); + const untouched = new SqliteDatabaseService({ + inspect: () => { + throw new Error("Cancelled selection must not read a database"); + }, + }); + expect( + await untouched.inspect({ path: "/selected.db" }, { signal }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); +}); + +it.each(["path", "digest", "subject", "wal", "table"])( + "rejects producer results with changed caller binding: %s", + async (problem) => { + const value = fixture(); + if (problem === "path") value.artifact.path = "/other.db"; + if (problem === "digest") value.artifact.sha256 = "b".repeat(64); + const observed = execution({ + ...value, + ...(problem === "wal" + ? { wal: { path: "/other.db-wal", sha256: "b".repeat(64), bytes: 32 } } + : {}), + }); + const service = new SqliteDatabaseService({ + inspect: () => + Promise.resolve( + ok({ + ...observed, + ...(problem === "subject" ? { subject: null } : {}), + }), + ), + }); + expect( + await service.inspect({ + path: "/selected.db", + ...(problem === "table" ? { table: "selected" } : {}), + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, +); + +it("returns observed inline evidence only while the request remains active", async () => { + const value = fixture(); + const service = new SqliteDatabaseService({ + inspect: () => Promise.resolve(ok(execution(value))), + }); + expect(await service.inspect({ path: "/selected.db" })).toMatchObject({ + ok: true, + value: { + confidence: "observed", + subject: { + local_path: "/selected.db", + digest: { sha256: "a".repeat(64) }, + }, + normalized_result: value, + }, + }); + const controller = new AbortController(); + const cancelled = new SqliteDatabaseService({ + inspect: () => { + controller.abort(); + return Promise.resolve(ok(execution(value))); + }, + }); + expect( + await cancelled.inspect( + { path: "/selected.db" }, + { signal: controller.signal }, + ), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); +}); diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts new file mode 100644 index 00000000000..ed78f81ffb7 --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -0,0 +1,114 @@ +import { isAbsolute } from "node:path"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { SqliteDatabasePort } from "./SqliteDatabasePort.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { + createEvidence, + type Evidence, + type EvidenceSubjectTarget, +} from "../../domain/evidence.js"; +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, + type SqliteDatabase, + SQLITE_ROW_LIMIT_DEFAULT, +} from "../../domain/sqlite/sqliteDatabase.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +const OPERATION = "inspect_sqlite_database"; + +const matchesSelectedDatabase = ( + report: SqliteDatabase, + input: InspectSqliteDatabaseInput, + subject: EvidenceSubjectTarget, +): boolean => + report.artifact.path === input.path && + subject.path === input.path && + subject.sha256 === report.artifact.sha256 && + subject.format === "file" && + (report.wal === null || report.wal.path === `${input.path}-wal`) && + (input.table === undefined + ? report.rows === null + : report.rows?.table === input.table && + report.rows.row_limit === (input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT)); + +/** Shared snapshot inspection and caller-selection validation for CLI and MCP. */ +export class SqliteDatabaseService { + constructor(readonly provider: SqliteDatabasePort) {} + + /** Await provider shutdown, retaining failed cleanup owners for retry. */ + close(): Promise { + return this.provider.close?.() ?? Promise.resolve(); + } + + /** Inspect an explicit database snapshot and preserve its original database/WAL identity. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const input = inspectSqliteDatabaseInputSchema.safeParse(rawInput); + if (!input.success) + return err( + analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, { + cause: input.error, + }), + ); + if (!isAbsolute(input.data.path)) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + const inspected = await this.provider.inspect(input.data, options); + if (!inspected.ok) return inspected; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const execution = inspected.value; + const report = sqliteDatabaseSchema.safeParse(execution.result); + if ( + !report.success || + execution.subject === null || + !matchesSelectedDatabase(report.data, input.data, execution.subject) + ) + return err( + new AnalysisOutputError( + OPERATION, + "SQLite adapter returned malformed data or changed the selected database identity/table/row limit.", + ), + ); + return ok( + createEvidence(execution.subject, execution.provider, { + operation: OPERATION, + parameters: { + path: input.data.path, + ...(input.data.table === undefined + ? {} + : { table: input.data.table }), + ...(input.data.row_limit === undefined + ? {} + : { row_limit: input.data.row_limit }), + }, + result: report.data, + ...(execution.rawResult === null + ? {} + : { rawResult: execution.rawResult }), + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }), + ); + } +} diff --git a/src/artifacts/readStableArtifact.ts b/src/artifacts/readStableArtifact.ts index 560d00c959e..e35a20783f5 100644 --- a/src/artifacts/readStableArtifact.ts +++ b/src/artifacts/readStableArtifact.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { constants, type BigIntStats } from "node:fs"; import { lstat, open, type FileHandle } from "node:fs/promises"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import type { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; /** Filesystem boundary for stable reads, including deterministic replacement probes. */ export interface StableArtifactFileSystem { @@ -42,6 +44,7 @@ export const readStableArtifact = async ( maximumBytes: number, signal?: AbortSignal, fileSystem: StableArtifactFileSystem = FILE_SYSTEM, + resourceScope?: ArtifactResourceScope, ): Promise<{ readonly bytes: Buffer; readonly sha256: string }> => { signal?.throwIfAborted(); const before = await fileSystem.lstat(path); @@ -61,6 +64,9 @@ export const readStableArtifact = async ( constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, ), ); + const owner = + resourceScope === undefined ? undefined : new OwnedFileHandle(file); + let failure: unknown; try { const initial = await file.stat({ bigint: true }); if ( @@ -116,7 +122,22 @@ export const readStableArtifact = async ( ); const bytes = Buffer.concat(chunks, count); return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") }; + } catch (cause: unknown) { + failure = cause; + throw cause; } finally { - await file.close(); + if (owner === undefined || resourceScope === undefined) await file.close(); + else { + const cleanup = await resourceScope.release({ + kind: "file-handle", + resource: path, + handle: owner, + }); + if (cleanup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + failure ?? cleanup.cause, + ArtifactReaderFailure.cleanupObservation(cleanup.cause, path), + ); + } } }; diff --git a/src/cli.ts b/src/cli.ts index a9ac2fa54a3..d3f5343846b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,6 +1,7 @@ import { snapshotEnvironment } from "./process/snapshotEnvironment.js"; import { createDirectAnalysis } from "./composition/directAnalysis.js"; import { registerEvmCommands } from "./cli/evmCommands.js"; +import { registerSqliteCommands } from "./cli/sqliteCommands.js"; import { registerAnalysisViewCommands } from "./cli/analysisViewCommands.js"; import { registerBinaryDiagnosticsCommands } from "./cli/binaryDiagnosticsCommands.js"; import { Cli } from "incur"; @@ -73,6 +74,7 @@ export const createCli = ( registerCoreAnalysisCommands(cli, logger, analysis.runDirectAnalysis); registerUtilityCommands(cli, logger, environment, analysis); registerEvmCommands(cli, logger, environment); + registerSqliteCommands(cli, logger, environment); registerArtifactCommands(cli, logger, analysis.runProviderAnalysis); registerManagedCommands(cli, logger, analysis.runProviderAnalysis); registerAndroidCommands(cli, logger, environment); diff --git a/src/cli/sqliteCommands.ts b/src/cli/sqliteCommands.ts new file mode 100644 index 00000000000..c53a5a9cdc9 --- /dev/null +++ b/src/cli/sqliteCommands.ts @@ -0,0 +1,93 @@ +import { z } from "incur"; +import { resolve } from "node:path"; +import { createSqliteDatabaseService } from "../composition/sqlite.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { logCliCommand } from "../cliLogging.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Logger } from "pino"; +import type { CliInstance } from "./types.js"; +import { withCommandCancellation } from "./commandCancellation.js"; + +/** Resolve an operator-selected SQLite snapshot through the shared inspection workflow. */ +export const registerSqliteCommands = ( + cli: CliInstance, + logger: Logger, + environment: Readonly, +): void => { + const service = createSqliteDatabaseService(environment); + cli.command(CLI_COMMANDS.inspectSqliteDatabase, { + description: + "Inspect SQLite schema and optional ordinary-table rows from a local snapshot", + args: z.object({ + path: z.string().describe("Local SQLite database snapshot"), + }), + options: z.object({ + table: z + .string() + .optional() + .describe("Exact ordinary table name to read"), + rowLimit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Positive safe integer row count; default 100; requires table", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectSqliteDatabase, async () => { + const inspection = await service + .inspect( + { + path: resolve(args.path), + ...(options.table === undefined + ? {} + : { table: options.table }), + ...(options.rowLimit === undefined + ? {} + : { row_limit: options.rowLimit }), + }, + { signal }, + ) + .then( + (result) => ({ result }), + (cause: unknown) => ({ cause }), + ); + try { + await service.close(); + } catch (cause: unknown) { + const primary = + "cause" in inspection + ? inspection.cause + : inspection.result.ok + ? undefined + : inspection.result.error; + if ( + primary instanceof AnalysisError && + cause instanceof AnalysisError + ) + throw analysisErrorWithCleanupFailure( + primary, + cause, + "inspect_sqlite_database", + ); + if ("cause" in inspection) + throw new AggregateError( + [inspection.cause, cause], + "SQLite inspection and provider cleanup both failed", + { cause: inspection.cause }, + ); + throw cause; + } + if ("cause" in inspection) throw inspection.cause; + return inspection.result.ok + ? inspection.result.value + : projectAnalysisError(inspection.result.error); + }), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 4312cbd5567..d9ed7aead10 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -1,5 +1,6 @@ /** Canonical names used by every one-shot CLI command registration. */ export const CLI_COMMANDS = Object.freeze({ + inspectSqliteDatabase: "inspect-sqlite-database", inspectEvmInterface: "inspect-evm-interface", inspectWebNetworkCapture: "inspect-web-network-capture", observeWebExecution: "observe-web-execution", diff --git a/src/composition/sqlite.ts b/src/composition/sqlite.ts new file mode 100644 index 00000000000..786684a2a0f --- /dev/null +++ b/src/composition/sqlite.ts @@ -0,0 +1,8 @@ +import { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import { SqliteDatabaseProvider } from "../sqlite/SqliteDatabaseProvider.js"; + +/** Compose local snapshot inspection without opening a native analysis provider. */ +export const createSqliteDatabaseService = ( + environment: Readonly = process.env, +): SqliteDatabaseService => + new SqliteDatabaseService(new SqliteDatabaseProvider(environment)); diff --git a/src/contracts/sqlite/sqliteToolContracts.ts b/src/contracts/sqlite/sqliteToolContracts.ts new file mode 100644 index 00000000000..e256c98e123 --- /dev/null +++ b/src/contracts/sqlite/sqliteToolContracts.ts @@ -0,0 +1,34 @@ +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, +} from "../../domain/sqlite/sqliteDatabase.js"; +import type { ToolContract } from "../toolContractTypes.js"; +import { toolContractMetadata } from "../toolEffects.js"; +import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; + +/** Snapshot inspection is independent of any active native-analysis target. */ +export const SQLITE_TOOL_CONTRACTS = [ + { + name: "inspect_sqlite_database", + ...toolContractMetadata("inspect_sqlite_database"), + kind: "artifact-provider", + description: + "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns database/WAL SHA-256, tables and columns, indexes, views and triggers inline as observed Evidence. Optionally select an exact ordinary table and a row limit (default 100, subject to byte and worker resource limits); returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", + inputSchema: inspectSqliteDatabaseInputSchema, + outputSchema: evidenceResultOf(sqliteDatabaseSchema), + examples: [ + { + title: "Inspect the complete schema of a local SQLite snapshot", + input: { path: "/artifacts/application.sqlite" }, + }, + { + title: "Read selected rows from an ordinary table", + input: { + path: "/artifacts/application.sqlite", + table: "settings", + row_limit: 20, + }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index df055b2f5aa..11ba391b7c4 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -1,4 +1,5 @@ import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; +import { SQLITE_TOOL_CONTRACTS } from "./sqlite/sqliteToolContracts.js"; import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "./native/nativeToolContracts.js"; @@ -31,6 +32,7 @@ export const TOOL_CONTRACTS = [ ...ANALYSIS_VIEW_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...EVM_TOOL_CONTRACTS, + ...SQLITE_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...ANDROID_TOOL_CONTRACTS, ...APKTOOL_TOOL_CONTRACTS, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 4ef01c9acfa..d3112edd862 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -41,6 +41,11 @@ const sessionEvidence = effects({ mutatesSession: true }); /** Explicit effect audit for every public tool. */ export const TOOL_EFFECTS: Readonly> = { + inspect_sqlite_database: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + }), inspect_evm_interface: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts index d7c8cea4948..c6df251800d 100644 --- a/src/domain/referenceSourceRequireBindings.ts +++ b/src/domain/referenceSourceRequireBindings.ts @@ -53,7 +53,8 @@ export const unshadowedReferenceSourceRequireCalls = ( if (start === undefined) return; if ( readAncestors().some( - (ancestor) => t.isWithStatement(ancestor) && contains(ancestor.body, node), + (ancestor) => + t.isWithStatement(ancestor) && contains(ancestor.body, node), ) ) return; @@ -91,9 +92,7 @@ const isExplicitModuleSource = (path: string): boolean => { return false; }; -const hasUninitializedTopLevelVarRequire = ( - program: t.Program, -): boolean => { +const hasUninitializedTopLevelVarRequire = (program: t.Program): boolean => { let found = false; let initialized = false; traverseJavaScriptAst(program, { diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts new file mode 100644 index 00000000000..f6dc8a90794 --- /dev/null +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -0,0 +1,162 @@ +import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; + +/** Default ordinary-table sample size when the caller selects a table. */ +export const SQLITE_ROW_LIMIT_DEFAULT = 100; + +const nonnegative = z.number().int().nonnegative(); +const hex = z.string().regex(/^(?:[0-9a-f]{2})*$/); +const encoding = z.enum(["UTF-8", "UTF-16le", "UTF-16be"]); +const artifactSchema = z.strictObject({ + path: z.string().min(1), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + bytes: nonnegative, +}); + +/** Inspect one frozen database and its sibling WAL, with optional ordinary-table rows. */ +export const inspectSqliteDatabaseInputSchema = z + .strictObject({ + path: localPathStringSchema.describe( + "Absolute path to an offline SQLite database snapshot", + ), + table: z + .string() + .optional() + .describe( + "Exact ordinary table name; views and virtual/shadow tables are not executed", + ), + row_limit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Selected-table sample size, default 100; subject to byte and worker resource limits", + ), + }) + .superRefine((input, context) => { + if (input.row_limit !== undefined && input.table === undefined) + context.addIssue({ + code: "custom", + path: ["row_limit"], + message: "row_limit requires an explicit table selection", + }); + }) + .meta({ dependentRequired: { row_limit: ["table"] } }); + +/** JSON-safe SQLite cells retain integers, binary content and special REAL values. */ +export const sqliteValueSchema = z.discriminatedUnion("type", [ + z.strictObject({ type: z.literal("null") }), + z.strictObject({ + type: z.literal("integer"), + value: z.string().regex(/^-?(?:0|[1-9][0-9]*)$/), + }), + z.strictObject({ + type: z.literal("real"), + value: z.union([z.number(), z.enum(["Infinity", "-Infinity", "-0"])]), + }), + z.strictObject({ + type: z.literal("text"), + value: z.string(), + bytes_hex: hex, + encoding, + }), + z.strictObject({ type: z.literal("blob"), hex }), +]); + +/** Frozen schema facts and explicitly bounded selected records, without arbitrary SQL. */ +export const sqliteDatabaseSchema = z + .strictObject({ + artifact: artifactSchema, + wal: artifactSchema.nullable(), + engine: z.strictObject({ + name: z.literal("SQLite"), + version: z.string().min(1), + }), + database_encoding: encoding, + schema: z.strictObject({ + completeness: z.literal("complete"), + tables: z.array( + z.strictObject({ + name: z.string(), + sql: z.string().nullable(), + root_page: nonnegative, + kind: z + .enum(["table", "virtual", "shadow", "unknown"]) + .describe( + "Unknown means a possible shadow table of an unresolved virtual table; its rows cannot be inspected", + ), + without_rowid: z.boolean(), + strict: z.boolean(), + columns_completeness: z.enum(["complete", "unsupported"]), + columns: z.array( + z.strictObject({ + cid: nonnegative, + name: z.string(), + declared_type: z.string(), + not_null: z.boolean(), + default_sql: z.string().nullable(), + primary_key_position: nonnegative, + hidden: z.union([ + z.literal(0), + z.literal(1), + z.literal(2), + z.literal(3), + ]), + }), + ), + }), + ), + indexes: z.array( + z.strictObject({ + name: z.string(), + table: z.string(), + sql: z.string().nullable(), + root_page: nonnegative, + }), + ), + views: z.array( + z.strictObject({ name: z.string(), sql: z.string().nullable() }), + ), + triggers: z.array( + z.strictObject({ + name: z.string(), + table: z.string(), + sql: z.string().nullable(), + }), + ), + }), + rows: z + .strictObject({ + table: z.string(), + columns: z.array(z.string()), + values: z.array(z.array(sqliteValueSchema)), + row_limit: z.number().int().min(1), + returned_rows: nonnegative, + truncated: z.boolean(), + order: z.literal("unspecified"), + }) + .nullable(), + limitations: z.array(z.string()), + }) + .superRefine((report, context) => { + const rows = report.rows; + if ( + rows !== null && + (rows.returned_rows !== rows.values.length || + rows.returned_rows > rows.row_limit || + rows.values.some((row) => row.length !== rows.columns.length)) + ) + context.addIssue({ + code: "custom", + path: ["rows"], + message: + "Selected row counts and column dimensions must match the retained values", + }); + }); + +export type InspectSqliteDatabaseInput = z.infer< + typeof inspectSqliteDatabaseInputSchema +>; +export type SqliteDatabase = z.infer; +export type SqliteValue = z.infer; diff --git a/src/main/transport.ts b/src/main/transport.ts index f8d6014f3cb..93754c90de5 100644 --- a/src/main/transport.ts +++ b/src/main/transport.ts @@ -55,6 +55,17 @@ export const startMcpTransport = async ( ); } const androidProviders: AndroidAnalysisPort[] = []; + const servers = new Set>(); + const closeServers = async (): Promise => { + const results = await Promise.allSettled( + [...servers].map((server) => server.close()), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "REA server cleanup failed"); + }; const closeAndroid = async (): Promise => { const results = await Promise.allSettled( androidProviders.map((provider) => provider.close()), @@ -71,7 +82,7 @@ export const startMcpTransport = async ( serverContext.environment, ); androidProviders.push(android); - return (dependencies.createServer ?? createServer)( + const server = (dependencies.createServer ?? createServer)( { kind: "session", session }, { logger: serverContext.logger, @@ -81,6 +92,14 @@ export const startMcpTransport = async ( androidAnalysis: android, }, ); + // The SDK can discard a server even when its provider cleanup failed. + servers.add(server); + const closeServer = server.close.bind(server); + server.close = async () => { + await closeServer(); + servers.delete(server); + }; + return server; }, { onerror: () => { @@ -90,10 +109,12 @@ export const startMcpTransport = async ( }, ); } catch (cause: unknown) { - const [binaryCleanup, androidCleanup] = await Promise.allSettled([ - session.close(), - closeAndroid(), - ]); + const [binaryCleanup, androidCleanup, serverCleanup] = + await Promise.allSettled([ + session.close(), + closeAndroid(), + closeServers(), + ]); serverLogger.error( { error: cause instanceof Error ? cause.message : String(cause), @@ -109,7 +130,7 @@ export const startMcpTransport = async ( ); dependencies.writeStderr(`${cleanupError.message}\n`); } - for (const cleanup of [binaryCleanup, androidCleanup]) { + for (const cleanup of [binaryCleanup, androidCleanup, serverCleanup]) { if (cleanup.status === "rejected") { const reason = cleanup.reason instanceof Error @@ -124,5 +145,22 @@ export const startMcpTransport = async ( } return { ok: false }; } - return { ok: true, handle, closeAndroid }; + return { + ok: true, + handle: { + close: async () => { + const results = await Promise.allSettled([ + handle.close(), + closeServers(), + ]); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) + throw new AggregateError(failures, "REA transport cleanup failed"); + }, + }, + closeAndroid, + }; }; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index ff1a3383513..4faef26aa84 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -4,6 +4,9 @@ import { registerPeResourcesTool } from "./registerPeResourcesTool.js"; import type { EvmInterfaceService } from "../application/evm/EvmInterfaceService.js"; import { createEvmInterfaceService } from "../composition/evm.js"; import { registerEvmTools } from "./registerEvmTools.js"; +import type { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import { createSqliteDatabaseService } from "../composition/sqlite.js"; +import { registerSqliteTools } from "./registerSqliteTools.js"; import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; import { createRecordedCrashService } from "../composition/binaryDiagnostics.js"; import type { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; @@ -104,6 +107,7 @@ export interface CreateServerOptions { readonly environment?: Readonly; readonly delivery?: ToolResultDelivery; readonly evmInterface?: EvmInterfaceService; + readonly sqliteDatabase?: SqliteDatabaseService; readonly logger?: Logger; readonly binaryLayout?: BinaryLayoutService; readonly recordedCrash?: RecordedCrashService; @@ -207,6 +211,8 @@ export const createServer = ( const delivery = selectToolResultDelivery(environment, options.delivery); const evmInterface = options.evmInterface ?? createEvmInterfaceService(environment); + const sqliteDatabase = + options.sqliteDatabase ?? createSqliteDatabaseService(environment); const webSourceLocation = options.webSourceLocation ?? createWebSourceLocationService(environment); const javascriptRecovery = @@ -220,6 +226,7 @@ export const createServer = ( environment, delivery, evmInterface, + sqliteDatabase, webSourceLocation, javascriptRecovery, }; @@ -246,6 +253,7 @@ export const createServer = ( jeb.close(), adbDevice.close(), evmInterface.close(), + sqliteDatabase.close(), webSourceLocation.close(), javascriptRecovery.close?.() ?? Promise.resolve(), artifactResources.close(), @@ -420,6 +428,12 @@ const registerConfiguredAnalysisTools = ( recordEvidence, withAdmittedAnalysis, ); + registerSqliteTools( + server, + options.sqliteDatabase ?? createSqliteDatabaseService(environment), + toolLogger, + recordEvidence, + ); registerRecordedCrashTools( server, options.recordedCrash ?? createRecordedCrashService(environment), diff --git a/src/server/registerSqliteTools.ts b/src/server/registerSqliteTools.ts new file mode 100644 index 00000000000..0f8c72745c0 --- /dev/null +++ b/src/server/registerSqliteTools.ts @@ -0,0 +1,33 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "pino"; +import { logToolExecution } from "./toolLogging.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; + +/** Bind snapshot inspection to its named contract and additive session Evidence. */ +export const registerSqliteTools = ( + server: EvidenceMcpServer, + service: SqliteDatabaseService, + logger: Logger, + recordEvidence?: EvidenceWriter["recordEvidence"], +): void => { + const contract = toolContract("inspect_sqlite_database"); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input, context) => { + const result = await logToolExecution(logger, contract.name, () => + service.inspect(input, { signal: context.mcpReq.signal }), + ); + if (!result.ok) return server.delivery.toCallToolResult(result, contract); + const recorded = recordEvidence?.(result.value); + return server.delivery.toEvidenceToolResult( + result.value, + contract, + recorded, + ); + }, + ); +}; diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts new file mode 100644 index 00000000000..4dc4145ed9c --- /dev/null +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -0,0 +1,167 @@ +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + AnalysisError, + type AnalysisCapturedOutput, +} from "../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; +import { + SQLITE_DATABASE_LIMITS, + SQLITE_PROVIDER_IDENTITY, +} from "./SqliteDatabaseLimits.js"; + +/** Preserve diagnostics independently of worker exit and private-root cleanup. */ +export const capturedSqliteOutput = ( + snapshot: ProviderProcessSnapshot, +): AnalysisCapturedOutput => ({ + stdout: snapshot.stdout.text, + stderr: snapshot.stderr.text, + truncated: snapshot.diagnosticTruncated === true, +}); + +/** Preserve artifact, caller, process, resource and output failure meanings. */ +export const sqliteDatabaseFailure = ( + cause: unknown, + path: string, + phase: string, + signal?: AbortSignal, +): AnalysisError => { + const operation = "inspect_sqlite_database"; + if (cause instanceof AnalysisError) return cause; + if (cause instanceof ArtifactReaderFailure) { + if (cause.cleanup !== undefined) + return new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cause.cleanup.resources, + { + reason: cause.cleanup.reason, + previous_error: + cause.cause === undefined + ? null + : projectAnalysisError( + signal?.aborted + ? new AnalysisCancelledError(operation, { + cause: cause.cause, + }) + : sqliteDatabaseFailure(cause.cause, path, phase), + ), + }, + { operation, cause }, + ); + if (cause.reason === "io" && cause.cause !== undefined) + return sqliteDatabaseFailure(cause.cause, path, phase, signal); + if (cause.reason === "integrity") + return new AnalysisArtifactChangedError(operation, path, cause.message, { + cause, + }); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(operation, { cause }); + return new AnalysisInputError(operation, { cause }, [ + { + path: ["path"], + reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + message: cause.message, + }, + ]); + } + if (cause instanceof Error && cause.name === "AbortError") + return new AnalysisCancelledError(operation, { cause }); + if (cause instanceof OwnedCommandFailure) { + const projected = ownedCommandFailure(cause); + if (projected !== undefined) return projected; + } + if (phase === "artifact-read" && cause instanceof Error && "code" in cause) { + if (cause.code === "EACCES" || cause.code === "EPERM") + return new AnalysisAccessDeniedError(operation, path, cause.code, { + cause, + }); + if (cause.code === "ENOENT" || cause.code === "ENOTDIR") + return new AnalysisInputError(operation, { cause }, [ + { + path: ["path"], + reason: "invalid_value", + message: `Selected database file set could not be read (${String(cause.code)}): ${path}`, + }, + ]); + } + return new ProviderAdapterError(SQLITE_PROVIDER_IDENTITY.id, operation, { + cause, + diagnostics: { + phase, + path, + reason: cause instanceof Error ? cause.message : String(cause), + ...(cause instanceof OwnedCommandFailure && cause.snapshot !== null + ? { + failure_kind: cause.reason, + exit_code: cause.snapshot.exitCode ?? null, + signal: cause.snapshot.signal ?? null, + captured_output: { ...capturedSqliteOutput(cause.snapshot) }, + } + : {}), + }, + }); +}; + +const ownedCommandFailure = ( + cause: OwnedCommandFailure, +): AnalysisError | undefined => { + const operation = "inspect_sqlite_database"; + const capturedOutput = + cause.snapshot === null ? undefined : capturedSqliteOutput(cause.snapshot); + const options = { + cause, + ...(capturedOutput === undefined ? {} : { capturedOutput }), + }; + if (cause.cleanupFailure !== null) + return new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cause.resources, + { + reason: cause.cleanupFailure, + previous_error: { + failure_kind: cause.reason, + message: cause.message, + }, + ...(capturedOutput === undefined + ? {} + : { captured_output: { ...capturedOutput } }), + }, + { operation, cause }, + ); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(operation, options); + if (cause.reason === "timeout") + return new AnalysisTimeoutError( + operation, + SQLITE_DATABASE_LIMITS.timeoutMs, + options, + ); + if (cause.reason === "output-limit") + return new AnalysisOutputError(operation, cause.message, options); + if ( + cause.snapshot?.stderr.text.includes("JavaScript heap out of memory") === + true + ) + return new AnalysisResourceConstraintError( + operation, + "memory", + "Owned SQLite worker reported JavaScript heap exhaustion.", + { + javascript_heap_bytes: 256 * 1024 * 1024, + }, + options, + ); + return undefined; +}; diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts new file mode 100644 index 00000000000..5dad30b3f86 --- /dev/null +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -0,0 +1,448 @@ +import { DatabaseSync } from "node:sqlite"; +import { z } from "zod"; +import { + SQLITE_ROW_LIMIT_DEFAULT, + type InspectSqliteDatabaseInput, + type SqliteDatabase, + type SqliteValue, +} from "../domain/sqlite/sqliteDatabase.js"; +import { + SQLITE_DATABASE_LIMITS, + SqliteInspectionFailure, +} from "./SqliteDatabaseLimits.js"; + +const natural = z + .bigint() + .nonnegative() + .max(BigInt(Number.MAX_SAFE_INTEGER)) + .transform(Number); +const schemaRow = z.object({ + type: z.enum(["table", "index", "view", "trigger"]), + name: z.instanceof(Uint8Array), + tbl_name: z.instanceof(Uint8Array), + rootpage: natural, + sql: z.instanceof(Uint8Array).nullable(), +}); +const tableRow = z.object({ + schema: z.string(), + name: z.string(), + type: z.enum(["table", "view", "virtual", "shadow"]), + ncol: natural, + wr: z.bigint(), + strict: z.bigint(), +}); +const columnRow = z.object({ + cid: natural, + name: z.string(), + type: z.string(), + notnull: z.bigint(), + dflt_value: z.string().nullable(), + pk: natural, + hidden: z + .bigint() + .transform(Number) + .pipe(z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3)])), +}); +const arrayRow = z.array( + z.union([ + z.null(), + z.bigint(), + z.custom( + (value) => typeof value === "number" && !Number.isNaN(value), + ), + z.string(), + ]), +); +const encodingSchema = z.enum(["UTF-8", "UTF-16le", "UTF-16be"]); +type DatabaseEncoding = z.output; +type Inspection = Pick< + SqliteDatabase, + "engine" | "database_encoding" | "schema" | "rows" | "limitations" +>; + +class OutputBudget { + #bytes = 1024; + retain(value: unknown): void { + this.#bytes += Buffer.byteLength(JSON.stringify(value)) + 1; + if (this.#bytes > SQLITE_DATABASE_LIMITS.outputBytes) + throw new SqliteInspectionFailure( + "output-limit", + "Complete SQLite evidence exceeds the 16 MiB JSON reply budget; no partial success was returned.", + ); + } + beforeDecode(row: z.output): void { + let minimumBytes = this.#bytes; + for (const value of row) + if (typeof value === "string") minimumBytes += Buffer.byteLength(value); + if (minimumBytes > SQLITE_DATABASE_LIMITS.outputBytes) + throw new SqliteInspectionFailure( + "output-limit", + "Selected cells exceed the 16 MiB reply budget before text/hex expansion.", + ); + } + beforeProjection(lengths: readonly bigint[]): void { + const encodedBytes = lengths.reduce( + (total, length) => total + length * 2n, + BigInt(lengths.length * 64), + ); + if ( + encodedBytes + BigInt(this.#bytes) > + BigInt(SQLITE_DATABASE_LIMITS.outputBytes) + ) + throw new SqliteInspectionFailure( + "output-limit", + "Selected row byte lengths exceed the 16 MiB reply budget before hex projection.", + ); + } +} +const quoteIdentifier = (name: string): string => + `"${name.replaceAll('"', '""')}"`; +// SQLite folds only ASCII letters when matching virtual/shadow table names. +const sqliteIdentifierKey = (name: string): string => + name.replace(/[A-Z]/g, (letter) => letter.toLowerCase()); +const textEncoding = (encoding: DatabaseEncoding): string => + encoding === "UTF-8" + ? "utf-8" + : encoding === "UTF-16le" + ? "utf-16le" + : "utf-16be"; + +const decodeSchemaText = ( + value: Uint8Array, + encoding: DatabaseEncoding, + label: string, +): string => { + try { + return new TextDecoder(textEncoding(encoding), { + fatal: true, + ignoreBOM: true, + }).decode(value); + } catch (cause: unknown) { + throw new SqliteInspectionFailure( + "format", + `SQLite schema ${label} contains malformed ${encoding} text: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } +}; + +const cell = ( + value: z.output[number], + encoding: DatabaseEncoding, +): SqliteValue => { + if (value === null) return { type: "null" }; + if (typeof value === "bigint") + return { type: "integer", value: String(value) }; + if (typeof value === "number") + return { + type: "real", + value: + value === Infinity + ? "Infinity" + : value === -Infinity + ? "-Infinity" + : Object.is(value, -0) + ? "-0" + : value, + }; + const hex = value.slice(1).toLowerCase(); + if (!/^(?:[0-9a-f]{2})*$/.test(hex)) + throw new SqliteInspectionFailure( + "format", + "SQLite returned an invalid encoded cell.", + ); + if (value[0] === "b") return { type: "blob", hex }; + if (value[0] !== "t") + throw new SqliteInspectionFailure( + "format", + "SQLite returned an unknown encoded cell type.", + ); + return { + type: "text", + value: new TextDecoder(textEncoding(encoding), { ignoreBOM: true }).decode( + Buffer.from(hex, "hex"), + ), + bytes_hex: hex, + encoding, + }; +}; + +const readColumns = ( + db: DatabaseSync, + name: string, + budget: OutputBudget, +): SqliteDatabase["schema"]["tables"][number]["columns"] => { + const columns: SqliteDatabase["schema"]["tables"][number]["columns"] = []; + // Direct PRAGMA syntax cannot be shadowed by a stored table or view. + for (const raw of db + .prepare(`PRAGMA main.table_xinfo(${quoteIdentifier(name)})`) + .iterate()) { + const value = columnRow.parse(raw); + const column = { + cid: value.cid, + name: value.name, + declared_type: value.type, + not_null: value.notnull !== 0n, + default_sql: value.dflt_value, + primary_key_position: value.pk, + hidden: value.hidden, + }; + budget.retain(column); + columns.push(column); + } + return columns; +}; + +const readSchema = ( + db: DatabaseSync, + context: { + readonly budget: OutputBudget; + readonly limitations: string[]; + readonly encoding: DatabaseEncoding; + }, +): SqliteDatabase["schema"] => { + const { budget, limitations, encoding } = context; + const metadata = new Map>(); + // Names bound the minimum eventual schema output while metadata is collected. + // The full schema objects below account for them in the actual reply budget. + const metadataBudget = new OutputBudget(); + for (const raw of db.prepare("PRAGMA main.table_list").iterate()) { + const value = tableRow.parse(raw); + if (value.schema === "main") { + metadataBudget.retain(value.name); + metadata.set(value.name, value); + } + } + const unresolvedVirtualTables = [...metadata.values()] + .filter((value) => value.type === "virtual" && value.ncol === 0) + .map((value) => ({ + name: value.name, + prefix: sqliteIdentifierKey(`${value.name}_`), + })); + const schema: SqliteDatabase["schema"] = { + completeness: "complete", + tables: [], + indexes: [], + views: [], + triggers: [], + }; + // Raw bytes preserve text without hex expansion; avoid sorter records that + // duplicate large names or definitions beyond SQLite's native record limit. + for (const raw of db + .prepare( + "SELECT type, CAST(name AS BLOB) AS name, CAST(tbl_name AS BLOB) AS tbl_name, rootpage, CAST(sql AS BLOB) AS sql FROM main.sqlite_schema", + ) + .iterate()) { + const parsed = schemaRow.parse(raw); + const value = { + ...parsed, + name: decodeSchemaText(parsed.name, encoding, "name"), + tbl_name: decodeSchemaText(parsed.tbl_name, encoding, "table"), + sql: + parsed.sql === null + ? null + : decodeSchemaText(parsed.sql, encoding, "sql"), + }; + if (value.type === "table") { + const listed = metadata.get(value.name); + if (listed === undefined || listed.type === "view") + throw new SqliteInspectionFailure( + "format", + `SQLite schema/table inventory mismatch: ${value.name}`, + ); + const identifier = sqliteIdentifierKey(value.name); + const possibleShadowParent = + listed.type === "table" + ? unresolvedVirtualTables.find((parent) => + identifier.startsWith(parent.prefix), + ) + : undefined; + const table: SqliteDatabase["schema"]["tables"][number] = { + name: value.name, + sql: value.sql, + root_page: value.rootpage, + kind: possibleShadowParent === undefined ? listed.type : "unknown", + without_rowid: listed.wr !== 0n, + strict: listed.strict !== 0n, + columns_completeness: "complete", + columns: [], + }; + if (possibleShadowParent !== undefined) + limitations.push( + `SQLite could not resolve virtual table ${possibleShadowParent.name}. Table ${value.name} may be its shadow table; its kind is unknown and row inspection is unavailable.`, + ); + budget.retain(table); + if (listed.type === "virtual") { + table.columns_completeness = "unsupported"; + limitations.push( + `Virtual table column expansion is not performed: ${value.name}. Its complete stored CREATE statement is retained.`, + ); + } else table.columns = readColumns(db, value.name, budget); + schema.tables.push(table); + } else if (value.type === "index") { + const index = { + name: value.name, + table: value.tbl_name, + sql: value.sql, + root_page: value.rootpage, + }; + budget.retain(index); + schema.indexes.push(index); + } else if (value.type === "view") { + const view = { name: value.name, sql: value.sql }; + budget.retain(view); + schema.views.push(view); + } else { + const trigger = { + name: value.name, + table: value.tbl_name, + sql: value.sql, + }; + budget.retain(trigger); + schema.triggers.push(trigger); + } + } + return schema; +}; + +const readRows = ( + db: DatabaseSync, + input: InspectSqliteDatabaseInput, + schema: SqliteDatabase["schema"], + context: { + readonly encoding: DatabaseEncoding; + readonly budget: OutputBudget; + }, +): SqliteDatabase["rows"] => { + const { encoding, budget } = context; + if (input.table === undefined) return null; + const table = schema.tables.find((value) => value.name === input.table); + if (table === undefined) + throw new SqliteInspectionFailure( + "selection", + `Selected ordinary table does not exist with that exact name: ${input.table}`, + ); + if (table.kind === "unknown") + throw new SqliteInspectionFailure( + "selection", + `SQLite cannot determine whether selected table ${input.table} is a shadow table of an unresolved virtual table. Row inspection requires a confirmed ordinary table.`, + ); + if (table.kind !== "table") + throw new SqliteInspectionFailure( + "selection", + `Rows from ${table.kind} table ${input.table} are outside the ordinary-table inspection profile.`, + ); + const limit = input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT; + const columns = table.columns + .filter((column) => column.hidden !== 1) + .map((column) => column.name); + const projection = columns + .map((name) => { + const column = quoteIdentifier(name); + return `CASE typeof(${column}) WHEN 'text' THEN 't' || hex(CAST(${column} AS BLOB)) WHEN 'blob' THEN 'b' || hex(${column}) ELSE ${column} END`; + }) + .join(", "); + const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`; + const lengths = db.prepare( + `SELECT ${columns + .map((name) => { + const column = quoteIdentifier(name); + return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`; + }) + .join(", ")} FROM ${source} LIMIT ?`, + ); + lengths.setReturnArrays(true); + for (const raw of lengths.iterate(limit)) + budget.beforeProjection(z.array(z.bigint().nonnegative()).parse(raw)); + const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`); + statement.setReturnArrays(true); + const rows: NonNullable = { + table: input.table, + columns, + values: [], + row_limit: limit, + returned_rows: 0, + truncated: false, + order: "unspecified", + }; + for (const raw of statement.iterate(limit)) { + const rawRow = arrayRow.parse(raw); + // Hex expansion already belongs to the retained-output budget, before decoding/copying cells. + budget.beforeDecode(rawRow); + const values = rawRow.map((value) => cell(value, encoding)); + budget.retain(values); + rows.values.push(values); + } + rows.returned_rows = rows.values.length; + rows.truncated = + db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !== + undefined; + return rows; +}; + +/** Query only a provider-owned snapshot; no caller SQL, extensions, views or virtual rows. */ +export const inspectSqliteDatabaseSnapshot = ( + snapshotPath: string, + input: InspectSqliteDatabaseInput, +): Inspection => { + const db = new DatabaseSync(snapshotPath, { + readOnly: true, + allowExtension: false, + readBigInts: true, + enableDoubleQuotedStringLiterals: false, + timeout: 0, + limits: { + length: SQLITE_DATABASE_LIMITS.outputBytes, + sqlLength: SQLITE_DATABASE_LIMITS.outputBytes, + }, + }); + try { + const limits: unknown = db.limits; + const supported = z + .object({ + length: z + .number() + .int() + .positive() + .max(SQLITE_DATABASE_LIMITS.outputBytes), + sqlLength: z + .number() + .int() + .positive() + .max(SQLITE_DATABASE_LIMITS.outputBytes), + }) + .safeParse(limits); + if (!supported.success) + throw new SqliteInspectionFailure( + "unavailable", + "SQLite inspection requires the native DatabaseSync.limits API to bound untrusted values and SQL. Use Node.js 24.15 or newer in the supported 24.x line, or a compatible Node.js 26+ runtime.", + ); + db.exec( + "PRAGMA trusted_schema=OFF; PRAGMA query_only=ON; PRAGMA mmap_size=0; PRAGMA cell_size_check=ON; BEGIN;", + ); + const version = z + .object({ version: z.string() }) + .parse(db.prepare("SELECT sqlite_version() AS version").get()).version; + const encoding = z + .object({ encoding: encodingSchema }) + .parse(db.prepare("PRAGMA encoding").get()).encoding; + const limitations = [ + "Inspects an offline stable file set. Filesystem identity checks detect observed acquisition changes; they do not establish an atomic snapshot of an actively changing database. Supply an application-created backup or quiescent database/WAL pair.", + "The source database and WAL are never opened by SQLite or modified. A private copy may acquire its own WAL-index sidecar and is removed after inspection.", + "SQLite reports current schema and selected committed rows from its valid database/WAL state. Deleted records, uncommitted or invalid WAL tails, encrypted databases, forensic recovery, and full integrity verification are outside this profile.", + "Stored CREATE statements preserve index/view/trigger definitions. Views, triggers, and virtual/shadow table rows are not executed; virtual-table column expansion is unsupported.", + "Selected rows have unspecified order and are a bounded sample. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.", + "Resource boundaries: combined input 256 MiB, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB and supervised deadline 30 seconds. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.", + ]; + const budget = new OutputBudget(); + const schema = readSchema(db, { budget, limitations, encoding }); + return { + engine: { name: "SQLite", version }, + database_encoding: encoding, + schema, + rows: readRows(db, input, schema, { encoding, budget }), + limitations, + }; + } finally { + db.close(); + } +}; diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts new file mode 100644 index 00000000000..db5ef88623d --- /dev/null +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -0,0 +1,28 @@ +/** The engine version is observed inside each successful owned worker. */ +export const SQLITE_PROVIDER_IDENTITY = { + id: "sqlite", + name: "REA SQLite database inspector", + version: null, +} as const; +/** Bound frozen input, retained JSON and worker allocation before parsing untrusted files. */ +export const SQLITE_DATABASE_LIMITS = { + inputBytes: 256 * 1024 * 1024, + outputBytes: 16 * 1024 * 1024, + timeoutMs: 30_000, + diagnosticBytes: 1024 * 1024, +} as const; +/** Internal rejection retains the failed boundary without returning partial success. */ +export class SqliteInspectionFailure extends Error { + constructor( + readonly reason: + | "format" + | "selection" + | "unsupported" + | "unavailable" + | "output-limit" + | "memory", + message: string, + ) { + super(message); + } +} diff --git a/src/sqlite/SqliteDatabaseProvider.ts b/src/sqlite/SqliteDatabaseProvider.ts new file mode 100644 index 00000000000..aa523024dfb --- /dev/null +++ b/src/sqlite/SqliteDatabaseProvider.ts @@ -0,0 +1,370 @@ +import { + createAnalysisExecution, + type AnalysisExecution, + type ExecutionOptions, +} from "../application/AnalysisProvider.js"; +import type { SqliteDatabasePort } from "../application/sqlite/SqliteDatabasePort.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { + AnalysisCapturedOutput, + AnalysisError, +} from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, +} from "../domain/sqlite/sqliteDatabase.js"; +import { isAbsolute } from "node:path"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js"; +import { SQLITE_PROVIDER_IDENTITY } from "./SqliteDatabaseLimits.js"; +import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js"; + +import { + executeSqliteDatabaseWorker, + type SqliteWorkerLauncher, +} from "./SqliteDatabaseWorkerExecution.js"; + +const operation = "inspect_sqlite_database"; +type SqliteOutcome = Result; +type SqliteRuntimeRoot = Pick; +interface PendingSnapshotCleanup { + readonly result: SqliteOutcome; + readonly output: AnalysisCapturedOutput | undefined; + process: ProviderProcessSupervisor | undefined; +} +/** Inspect a provider-owned copy through a cancellable SQLite child, without source writes. */ +export class SqliteDatabaseProvider implements SqliteDatabasePort { + readonly #active = new Set>(); + readonly #artifactResources = new ArtifactResourceScope(); + readonly #pendingCleanup = new Map< + SqliteRuntimeRoot, + PendingSnapshotCleanup + >(); + #closed = false; + #closePromise: Promise | undefined; + #retryPromise: Promise> | undefined; + constructor( + readonly environment: Readonly = process.env, + readonly launcher?: SqliteWorkerLauncher, + readonly createRuntime: () => Promise< + Pick + > = () => PrivateRuntimeRoot.create({ prefix: "rea-sqlite-database-" }), + ) {} + + /** Return exact original DB/WAL identities and current schema/selected committed rows. */ + async inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise { + if (this.#closed) return err(new AnalysisCancelledError(operation)); + const inspection = this.#artifactResources + .run(() => this.#inspect(input, options)) + .catch((cause: unknown) => + err( + selectedFailure( + sqliteDatabaseFailure( + cause, + input.path, + "artifact-read", + options?.signal, + ), + options?.signal, + ), + ), + ); + this.#active.add(inspection); + try { + return await inspection; + } finally { + this.#active.delete(inspection); + } + } + + /** Await inspections and retry retained descriptors, workers, and snapshot roots. */ + close(): Promise { + this.#closed = true; + this.#closePromise ??= Promise.allSettled(this.#active) + .then(async () => { + let descriptorFailure: AnalysisError | undefined; + try { + await this.#artifactResources.close(); + } catch (cause: unknown) { + const cleanup = ArtifactReaderFailure.cleanupObservation( + cause, + "SQLite artifact descriptors", + ); + descriptorFailure = new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cleanup.resources, + { reason: cleanup.reason }, + { operation, cause }, + ); + } + const cleaned = await this.#retryCleanup(); + if (descriptorFailure !== undefined && !cleaned.ok) + throw new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [ + ...descriptorFailure.cleanupResources, + ...cleaned.error.cleanupResources, + ], + { + failures: [ + projectAnalysisError(descriptorFailure), + projectAnalysisError(cleaned.error), + ], + }, + { operation }, + ); + if (descriptorFailure !== undefined) throw descriptorFailure; + if (!cleaned.ok) throw cleaned.error; + }) + .catch((cause: unknown) => { + this.#closePromise = undefined; + throw cause; + }); + return this.#closePromise; + } + + #retryCleanup(): Promise> { + this.#retryPromise ??= this.#cleanupPending().finally(() => { + this.#retryPromise = undefined; + }); + return this.#retryPromise; + } + + async #cleanupPending(): Promise> { + const failures: AnalysisError[] = []; + for (const [root, pending] of [...this.#pendingCleanup]) { + if (pending.process !== undefined) { + const stopped = await pending.process.stop(); + if (stopped.status === "incomplete") { + failures.push( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [ + pending.process.launch.ownership?.runId ?? + "owned-sqlite-worker", + root.path, + ], + { + reason: stopped.reason, + previous_error: pending.result.ok + ? null + : projectAnalysisError(pending.result.error), + ...(pending.output === undefined + ? {} + : { captured_output: { ...pending.output } }), + }, + { operation }, + ), + ); + continue; + } + pending.process = undefined; + } + const cleaned = await cleanupSqliteRoot( + root, + pending.result, + pending.output, + ); + if (cleaned.ok) this.#pendingCleanup.delete(root); + else failures.push(cleaned.error); + } + if (failures.length === 1 && failures[0] !== undefined) + return err(failures[0]); + return failures.length === 0 + ? ok(null) + : err( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + failures.flatMap((failure) => failure.cleanupResources), + { failures: failures.map(projectAnalysisError) }, + { operation }, + ), + ); + } + + async #inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise { + let root: Pick | undefined; + let processOwner: ProviderProcessSupervisor | undefined; + let retainedOutput: AnalysisCapturedOutput | undefined; + let phase = "configuration"; + let result: Result; + try { + if (options?.signal?.aborted) throw new AnalysisCancelledError(operation); + validateSqliteInput(input); + const retired = await this.#retryCleanup(); + if (!retired.ok) return retired; + if (this.#closed || options?.signal?.aborted) + throw new AnalysisCancelledError(operation); + root = await this.createRuntime(); + phase = "artifact-read"; + const snapshot = await captureSqliteDatabaseSnapshot( + input.path, + root.path, + this.#artifactResources, + options?.signal, + ); + phase = "worker"; + const inspection = await executeSqliteDatabaseWorker({ + input, + root: root.path, + snapshotPath: snapshot.snapshotPath, + environment: this.environment, + resourceScope: this.#artifactResources, + ...(this.launcher === undefined ? {} : { launcher: this.launcher }), + ...(options?.signal === undefined ? {} : { signal: options.signal }), + captured: (output) => { + retainedOutput = output; + }, + }); + const validated = sqliteDatabaseSchema.safeParse({ + artifact: snapshot.artifact, + wal: snapshot.wal, + ...inspection, + }); + if (!validated.success) + throw new AnalysisOutputError( + operation, + "SQLite worker report dimensions or identity fields are invalid", + retainedOutput === undefined + ? undefined + : { capturedOutput: retainedOutput }, + ); + result = ok(createSqliteObservation(validated.data, input.path)); + } catch (cause: unknown) { + if (cause instanceof OwnedCommandFailure) + processOwner = cause.cleanupOwner; + const failure = sqliteDatabaseFailure( + cause, + input.path, + phase, + options?.signal, + ); + retainedOutput ??= failure.capturedOutput; + result = err(selectedFailure(failure, options?.signal, retainedOutput)); + } + if (root !== undefined) { + if (processOwner !== undefined) { + this.#pendingCleanup.set(root, { + result, + output: retainedOutput, + process: processOwner, + }); + return result; + } + const cleaned = await cleanupSqliteRoot(root, result, retainedOutput); + if (!cleaned.ok) { + this.#pendingCleanup.set(root, { + result, + output: retainedOutput, + process: undefined, + }); + return cleaned; + } + } + return completedResult(result, options?.signal, retainedOutput); + } +} + +const selectedFailure = ( + failure: AnalysisError, + signal?: AbortSignal, + output?: AnalysisCapturedOutput, +): AnalysisError => + signal?.aborted && + failure._tag !== "AnalysisCancelledError" && + !failure.cleanupIncomplete + ? new AnalysisCancelledError( + operation, + output === undefined ? undefined : { capturedOutput: output }, + ) + : failure; +const completedResult = ( + result: Result, + signal?: AbortSignal, + output?: AnalysisCapturedOutput, +): Result => + result.ok && signal?.aborted + ? err( + new AnalysisCancelledError( + operation, + output === undefined ? undefined : { capturedOutput: output }, + ), + ) + : result; + +const createSqliteObservation = ( + report: ReturnType, + path: string, +): AnalysisExecution => { + const locations: AnalysisExecution["locations"][number][] = [ + { kind: "artifact-path", path }, + ]; + if (report.wal !== null) + locations.push({ kind: "artifact-path", path: report.wal.path }); + return createAnalysisExecution( + report, + { ...SQLITE_PROVIDER_IDENTITY, version: `SQLite ${report.engine.version}` }, + { + rawResult: null, + subject: { path, format: "file", sha256: report.artifact.sha256 }, + locations, + limitations: report.limitations, + }, + ); +}; +const cleanupSqliteRoot = async ( + root: Pick, + result: Result, + output?: AnalysisCapturedOutput, +): Promise> => { + try { + await root.close(); + return ok(null); + } catch (cause: unknown) { + return err( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [root.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + previous_error: result.ok ? null : projectAnalysisError(result.error), + previous_result: result.ok ? result.value.result : null, + ...(output === undefined ? {} : { captured_output: { ...output } }), + }, + { operation, cause }, + ), + ); + } +}; + +const validateSqliteInput = (input: InspectSqliteDatabaseInput): void => { + const parsed = inspectSqliteDatabaseInputSchema.safeParse(input); + if (!parsed.success || !isAbsolute(input.path)) + throw new AnalysisInputError(operation, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: parsed.success + ? "path must be an absolute local filesystem path" + : parsed.error.message, + }, + ]); +}; diff --git a/src/sqlite/SqliteDatabaseSnapshot.ts b/src/sqlite/SqliteDatabaseSnapshot.ts new file mode 100644 index 00000000000..62549c5ed04 --- /dev/null +++ b/src/sqlite/SqliteDatabaseSnapshot.ts @@ -0,0 +1,255 @@ +import { lstat, open, writeFile } from "node:fs/promises"; +import { constants, type BigIntStats } from "node:fs"; +import { join } from "node:path"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { + readStableArtifact, + type StableArtifactFileSystem, +} from "../artifacts/readStableArtifact.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open, +}; +const optionalStat = async ( + path: string, + selectedFileSystem: StableArtifactFileSystem, +): Promise => { + try { + return await selectedFileSystem.lstat(path); + } catch (cause: unknown) { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") + return null; + throw cause; + } +}; +const sameState = ( + before: BigIntStats | null, + after: BigIntStats | null, +): boolean => + before === null || after === null + ? before === after + : before.dev === after.dev && + before.ino === after.ino && + before.mode === after.mode && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs; + +const duringAcquisition = async ( + path: string, + read: () => Promise, +): Promise => { + try { + return await read(); + } catch (cause: unknown) { + if ( + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code)) + ) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${path}`, + { cause }, + ); + throw cause; + } +}; + +const readObservedArtifact = ( + path: string, + observed: BigIntStats, + maximumBytes: number, + selectedFileSystem: StableArtifactFileSystem, + resourceScope: ArtifactResourceScope, + signal?: AbortSignal, +) => + duringAcquisition(path, () => + readStableArtifact( + path, + maximumBytes, + signal, + { + open: (selectedPath, flags) => + selectedFileSystem.open(selectedPath, flags), + lstat: async (selectedPath) => { + const current = await selectedFileSystem.lstat(selectedPath); + if (!sameState(observed, current)) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${selectedPath}`, + ); + return current; + }, + }, + resourceScope, + ), + ); + +const verifyInactiveJournal = async ( + path: string, + before: BigIntStats, + selectedFileSystem: StableArtifactFileSystem, + resourceScope: ArtifactResourceScope, + signal?: AbortSignal, +): Promise => { + if (!before.isFile() || before.isSymbolicLink()) + throw new ArtifactReaderFailure( + "path", + `Expected a regular journal without a symlink: ${path}`, + ); + const file = await duringAcquisition(path, () => + selectedFileSystem.open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ), + ); + const owner = new OwnedFileHandle(file); + let failure: unknown; + try { + if (!sameState(before, await file.stat({ bigint: true }))) + throw new ArtifactReaderFailure( + "integrity", + `SQLite journal changed before open: ${path}`, + ); + // TRUNCATE commits leave an empty file; PERSIST commits zero the complete + // 28-byte fixed header. A new active journal may zero only its magic bytes. + const header = Buffer.alloc(28); + let bytesRead = 0; + if (before.size > 0n) { + while (bytesRead < header.length) { + signal?.throwIfAborted(); + const read = await file.read( + header, + bytesRead, + header.length - bytesRead, + bytesRead, + ); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + } + } + if (!sameState(before, await file.stat({ bigint: true }))) + throw new ArtifactReaderFailure( + "integrity", + `SQLite journal changed during inspection: ${path}`, + ); + if ( + before.size !== 0n && + (bytesRead !== header.length || header.some((byte) => byte !== 0)) + ) + throw new ArtifactReaderFailure( + "format", + `Rollback journal may require recovery: ${path}. Supply a clean offline snapshot; only empty journals or a fully zeroed header are supported.`, + ); + } catch (cause: unknown) { + failure = cause; + throw cause; + } finally { + const cleanup = await resourceScope.release({ + kind: "file-handle", + resource: path, + handle: owner, + }); + if (cleanup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + failure ?? cleanup.cause, + ArtifactReaderFailure.cleanupObservation(cleanup.cause, path), + ); + } +}; + +/** Freeze a stable offline file set without ever opening the source through SQLite. */ +export const captureSqliteDatabaseSnapshot = async ( + path: string, + root: string, + resourceScope: ArtifactResourceScope, + signal?: AbortSignal, + selectedFileSystem: StableArtifactFileSystem = fileSystem, +) => { + signal?.throwIfAborted(); + const paths = [path, `${path}-wal`, `${path}-journal`]; + const before = await Promise.all( + paths.map((path) => optionalStat(path, selectedFileSystem)), + ); + const database = before[0]; + const wal = before[1]; + if (database === undefined || wal === undefined || before[2] === undefined) + throw new Error("Incomplete database file-set observation"); + if (database === null) + throw new ArtifactReaderFailure( + "path", + `Selected SQLite database does not exist: ${path}`, + ); + if (before[2] !== null) + await verifyInactiveJournal( + `${path}-journal`, + before[2], + selectedFileSystem, + resourceScope, + signal, + ); + const selectedSize = database.size + (wal?.size ?? 0n); + if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) + throw new ArtifactReaderFailure( + "limit", + `Selected SQLite database and WAL exceed the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte snapshot budget: ${path}`, + ); + const snapshotPath = join(root, "database.snapshot"); + const captured = await readObservedArtifact( + path, + database, + SQLITE_DATABASE_LIMITS.inputBytes, + selectedFileSystem, + resourceScope, + signal, + ); + await writeFile(snapshotPath, captured.bytes, { + flag: "wx", + mode: 0o600, + ...(signal === undefined ? {} : { signal }), + }); + const artifact = { + path, + sha256: captured.sha256, + bytes: captured.bytes.length, + }; + let walArtifact: typeof artifact | null = null; + if (wal !== null) { + const capturedWal = await readObservedArtifact( + `${path}-wal`, + wal, + SQLITE_DATABASE_LIMITS.inputBytes - artifact.bytes, + selectedFileSystem, + resourceScope, + signal, + ); + await writeFile(`${snapshotPath}-wal`, capturedWal.bytes, { + flag: "wx", + mode: 0o600, + ...(signal === undefined ? {} : { signal }), + }); + walArtifact = { + path: `${path}-wal`, + sha256: capturedWal.sha256, + bytes: capturedWal.bytes.length, + }; + } + const after = await Promise.all( + paths.map((path) => + duringAcquisition(path, () => optionalStat(path, selectedFileSystem)), + ), + ); + for (const [index, expected] of before.entries()) + if (!sameState(expected, after[index] ?? null)) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${paths[index] ?? path}`, + ); + signal?.throwIfAborted(); + return { snapshotPath, artifact, wal: walArtifact }; +}; diff --git a/src/sqlite/SqliteDatabaseWorker.ts b/src/sqlite/SqliteDatabaseWorker.ts new file mode 100644 index 00000000000..ebd2d5bac12 --- /dev/null +++ b/src/sqlite/SqliteDatabaseWorker.ts @@ -0,0 +1,57 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { z } from "zod"; +import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabase.js"; +import { inspectSqliteDatabaseSnapshot } from "./SqliteDatabaseInspection.js"; +import { + SQLITE_DATABASE_LIMITS, + SqliteInspectionFailure, +} from "./SqliteDatabaseLimits.js"; + +const requestSchema = z.strictObject({ + snapshot_path: z.string(), + reply_path: z.string(), + input: inspectSqliteDatabaseInputSchema, +}); +const main = async (): Promise => { + const requestPath = process.argv[2]; + if (requestPath === undefined) + throw new Error("Owned SQLite worker request path missing."); + const request = requestSchema.parse( + JSON.parse(await readFile(requestPath, "utf8")), + ); + let reply: unknown; + try { + reply = { + ok: true, + inspection: inspectSqliteDatabaseSnapshot( + request.snapshot_path, + request.input, + ), + }; + } catch (cause: unknown) { + const code = + cause instanceof Error && "errcode" in cause ? cause.errcode : undefined; + reply = { + ok: false, + reason: + cause instanceof SqliteInspectionFailure + ? cause.reason + : code === 7 + ? "memory" + : code === 18 + ? "output-limit" + : "format", + message: cause instanceof Error ? cause.message : String(cause), + }; + } + let encoded = JSON.stringify(reply); + if (Buffer.byteLength(encoded) > SQLITE_DATABASE_LIMITS.outputBytes) + encoded = JSON.stringify({ + ok: false, + reason: "output-limit", + message: + "Complete SQLite evidence exceeds the 16 MiB JSON reply budget; no partial success was returned.", + }); + await writeFile(request.reply_path, encoded, { flag: "wx", mode: 0o600 }); +}; +await main(); diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts new file mode 100644 index 00000000000..9474a6d27cb --- /dev/null +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -0,0 +1,169 @@ +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisCapabilityUnavailableError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, +} from "../domain/sqlite/sqliteDatabase.js"; +import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { runOwnedCommand } from "../process/OwnedCommand.js"; +import { capturedSqliteOutput } from "./SqliteDatabaseFailures.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +const operation = "inspect_sqlite_database"; +const { + artifact: _artifactSchema, + wal: _walSchema, + ...inspectionShape +} = sqliteDatabaseSchema.shape; +const inspectionSchema = z.strictObject(inspectionShape); +const replySchema = z.discriminatedUnion("ok", [ + z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }), + z.strictObject({ + ok: z.literal(false), + reason: z.enum([ + "format", + "selection", + "unsupported", + "unavailable", + "output-limit", + "memory", + ]), + message: z.string(), + }), +]); +/** Replaceable launcher retains the shared owned-process supervision boundary. */ +export type SqliteWorkerLauncher = NonNullable< + Parameters[2] +>["launcher"]; + +const rejectedReply = ( + reply: z.output & { ok: false }, + capturedOutput: AnalysisCapturedOutput, +) => { + if (reply.reason === "unavailable") + return new AnalysisCapabilityUnavailableError( + "sqlite", + operation, + reply.message, + { capturedOutput, userMessage: reply.message }, + ); + if (reply.reason === "memory") + return new AnalysisResourceConstraintError( + operation, + "memory", + reply.message, + { + javascript_heap_bytes: 256 * 1024 * 1024, + }, + { capturedOutput }, + ); + if (reply.reason === "output-limit") + return new AnalysisResourceConstraintError( + operation, + "transport", + reply.message, + { + sqlite_value_or_record_bytes: SQLITE_DATABASE_LIMITS.outputBytes, + worker_reply_bytes: SQLITE_DATABASE_LIMITS.outputBytes, + }, + { capturedOutput }, + ); + return new AnalysisInputError(operation, { capturedOutput }, [ + { + path: [reply.reason === "selection" ? "table" : "path"], + reason: "invalid_format", + message: reply.message, + }, + ]); +}; + +/** Execute only the private snapshot and validate the bounded worker protocol. */ +export const executeSqliteDatabaseWorker = async (request: { + readonly input: InspectSqliteDatabaseInput; + readonly root: string; + readonly snapshotPath: string; + readonly environment: Readonly; + readonly resourceScope: ArtifactResourceScope; + readonly launcher?: SqliteWorkerLauncher; + readonly signal?: AbortSignal; + readonly captured: (output: AnalysisCapturedOutput) => void; +}): Promise> => { + const requestPath = join(request.root, "request.json"); + const replyPath = join(request.root, "reply.json"); + await writeFile( + requestPath, + JSON.stringify({ + snapshot_path: request.snapshotPath, + reply_path: replyPath, + input: request.input, + }), + { flag: "wx", mode: 0o600 }, + ); + const { NODE_OPTIONS: _ambientNodeOptions, ...environment } = + request.environment; + const execution = await runOwnedCommand( + { + command: process.execPath, + arguments: [ + "--max-old-space-size=256", + fileURLToPath(new URL("./SqliteDatabaseWorker.js", import.meta.url)), + requestPath, + ], + cwd: request.root, + expectedCommand: null, + runId: `rea-sqlite-database-${randomUUID()}`, + hostEnvironment: environment, + }, + { + timeoutMs: SQLITE_DATABASE_LIMITS.timeoutMs, + diagnosticBytes: SQLITE_DATABASE_LIMITS.diagnosticBytes, + }, + { + ...(request.signal === undefined ? {} : { signal: request.signal }), + ...(request.launcher === undefined ? {} : { launcher: request.launcher }), + }, + ); + const capturedOutput = capturedSqliteOutput(execution); + request.captured(capturedOutput); + let reply: z.output; + try { + const file = await readStableArtifact( + replyPath, + SQLITE_DATABASE_LIMITS.outputBytes, + request.signal, + undefined, + request.resourceScope, + ); + reply = replySchema.parse(JSON.parse(file.bytes.toString("utf8"))); + } catch (cause: unknown) { + const options = { + cause, + capturedOutput, + ...(cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined + ? { cleanup: cause.cleanup } + : {}), + }; + if (request.signal?.aborted) + throw new AnalysisCancelledError(operation, options); + throw new AnalysisOutputError( + operation, + `Owned SQLite worker reply is missing or malformed: ${cause instanceof Error ? cause.message : String(cause)}`, + options, + ); + } + if (!reply.ok) throw rejectedReply(reply, capturedOutput); + return reply.inspection; +}; diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts new file mode 100644 index 00000000000..2f0bd59c16d --- /dev/null +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -0,0 +1,463 @@ +import { createHash } from "node:crypto"; +import { readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, onTestFinished } from "vitest"; + +import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../fixtures/sqlite/database.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +const digest = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +cliTest( + "discovers SQLite runtime availability before inspecting a database through CLI and MCP", + async ({ cli }) => { + const result = await cli.run({ + arguments: ["capabilities", "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + open: false, + capabilities: expect.arrayContaining([ + expect.objectContaining({ + operation: "inspect_sqlite_database", + available: SQLITE_NATIVE_LIMITS_AVAILABLE, + reason: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + availability_code: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : "unsupported_version", + limitations: [expect.stringContaining("DatabaseSync.limits")], + effects: { + mutates_artifact: false, + launches_process: true, + may_show_ui: false, + may_access_network: false, + may_write_filesystem: true, + changes_permissions: false, + requires_root: false, + }, + }), + ]), + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("binary_session", {}); + expect(response.isError).not.toBe(true); + expect(response.structuredContent).toMatchObject({ + result: { + tool_availability: expect.arrayContaining([ + expect.objectContaining({ + name: "inspect_sqlite_database", + available: SQLITE_NATIVE_LIMITS_AVAILABLE, + remediation: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + }), + ]), + }, + }); + }, +); + +cliTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reports the required native SQLite limits capability through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-unavailable-"); + const path = join(root, "selected.db"); + const database = new DatabaseSync(path); + try { + database.exec("CREATE TABLE items(value INTEGER)"); + } finally { + database.close(); + } + const result = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining("DatabaseSync.limits"), + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { path }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining("DatabaseSync.limits"), + }); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "accepts small samples above 1000 rows within the real resource budgets", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-row-budget-"); + const path = join(root, "rows.db"); + const database = new DatabaseSync(path); + try { + database.exec(`CREATE TABLE items(value INTEGER); + WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002) + INSERT INTO items SELECT value FROM numbers;`); + } finally { + database.close(); + } + const { call } = await connectLocalToolsMcp(); + for (const limit of [1001, Number.MAX_SAFE_INTEGER]) { + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "items", + "--row-limit", + String(limit), + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { + rows: { + row_limit: limit, + returned_rows: Math.min(limit, 1002), + truncated: limit < 1002, + }, + }, + }); + const response = await call("inspect_sqlite_database", { + path, + table: "items", + row_limit: limit, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + } + }, +); + +const sourceState = async (path: string) => { + const bytes = await readFile(path); + const metadata = await stat(path, { bigint: true }); + return { + bytes, + device: metadata.dev, + inode: metadata.ino, + size: metadata.size, + modified: metadata.mtimeNs, + changed: metadata.ctimeNs, + }; +}; + +cliTest + .runIf(SQLITE_NATIVE_LIMITS_AVAILABLE) + .for(["DELETE", "PERSIST", "TRUNCATE"])( + "inspects a real %s SQLite schema and lossless selected records identically through CLI and MCP", + async (journalMode, { cli }) => { + const root = await createTestTempDirectory("rea-sqlite-public-"); + const path = join(root, "selected.db"); + const database = new DatabaseSync(path); + try { + database.exec(`PRAGMA journal_mode=${journalMode}`); + database.exec(` + CREATE TABLE "odd "" table" ( + id INTEGER PRIMARY KEY, + payload BLOB, + amount REAL, + missing TEXT, + text_value TEXT, + negative_zero, + generated TEXT GENERATED ALWAYS AS (hex(payload)) VIRTUAL + ); + CREATE UNIQUE INDEX "quoted "" index" ON "odd "" table"(payload, id); + CREATE VIEW readable AS SELECT id FROM "odd "" table"; + CREATE TRIGGER observed AFTER INSERT ON "odd "" table" BEGIN SELECT 1; END; + CREATE TABLE keyed(k TEXT PRIMARY KEY) WITHOUT ROWID, STRICT; + `); + const insert = database.prepare( + 'INSERT INTO "odd "" table"(id,payload,amount,missing,text_value,negative_zero) VALUES(?,?,?,NULL,CAST(X\'80ff\' AS TEXT),?)', + ); + insert.setReadBigInts(true); + expect( + insert.run( + 9007199254740993n, + Buffer.from("deadbeef", "hex"), + Infinity, + -0, + ).lastInsertRowid, + ).toBe(9007199254740993n); + insert.run( + 9007199254740994n, + Buffer.from("deadbeef", "hex"), + Infinity, + -0, + ); + } finally { + database.close(); + } + const original = await readFile(path); + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + "selected.db", + "--table", + 'odd " table', + "--row-limit", + "1", + "--json", + ], + cwd: root, + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_GHIDRA_HOME: "/unconfigured/ghidra", + }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + operation: "inspect_sqlite_database", + confidence: "observed", + subject: { local_path: path, digest: { sha256: digest(original) } }, + normalized_result: { + artifact: { path, sha256: digest(original), bytes: original.length }, + wal: null, + schema: { + completeness: "complete", + tables: expect.arrayContaining([ + expect.objectContaining({ + name: 'odd " table', + columns: expect.arrayContaining([ + expect.objectContaining({ + name: "id", + primary_key_position: 1, + }), + expect.objectContaining({ name: "generated", hidden: 2 }), + ]), + }), + expect.objectContaining({ + name: "keyed", + without_rowid: true, + strict: true, + }), + ]), + indexes: expect.arrayContaining([ + expect.objectContaining({ + name: 'quoted " index', + table: 'odd " table', + }), + ]), + views: [ + { name: "readable", sql: expect.stringContaining("SELECT id") }, + ], + triggers: [ + { + name: "observed", + table: 'odd " table', + sql: expect.stringContaining("AFTER INSERT"), + }, + ], + }, + rows: { + table: 'odd " table', + columns: [ + "id", + "payload", + "amount", + "missing", + "text_value", + "negative_zero", + "generated", + ], + returned_rows: 1, + row_limit: 1, + truncated: true, + values: [ + [ + { + type: "integer", + value: expect.stringMatching(/^900719925474099[34]$/), + }, + { type: "blob", hex: "deadbeef" }, + { type: "real", value: "Infinity" }, + { type: "null" }, + { type: "text", value: "\ufffd\ufffd", bytes_hex: "80ff" }, + { type: "real", value: "-0" }, + { + type: "text", + value: "DEADBEEF", + bytes_hex: "4445414442454546", + }, + ], + ], + }, + }, + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { + path, + table: 'odd " table', + row_limit: 1, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + expect(await readFile(path)).toEqual(original); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "includes uncheckpointed committed WAL data without modifying the source database or sidecars", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-wal-public-"); + const path = join(root, "wal.db"); + const database = new DatabaseSync(path); + onTestFinished(() => database.close()); + database.exec( + "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; CREATE TABLE committed(value TEXT); INSERT INTO committed VALUES('only in WAL');", + ); + const files = [path, `${path}-wal`, `${path}-shm`]; + const before = await Promise.all(files.map(sourceState)); + const db = before[0]; + const wal = before[1]; + expect(db).toBeDefined(); + expect(wal).toBeDefined(); + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "committed", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { + artifact: { sha256: db === undefined ? "missing" : digest(db.bytes) }, + wal: { + path: `${path}-wal`, + sha256: wal === undefined ? "missing" : digest(wal.bytes), + }, + rows: { + values: [ + [ + { + type: "text", + value: "only in WAL", + bytes_hex: Buffer.from("only in WAL").toString("hex"), + }, + ], + ], + returned_rows: 1, + truncated: false, + }, + }, + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { + path, + table: "committed", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + const after = await Promise.all(files.map(sourceState)); + expect(after).toEqual(before); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "returns actionable public errors for corrupt input, unknown or ambiguous tables and invalid record choices", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-errors-public-"); + const path = join(root, "invalid.db"); + await writeFile(path, "not a SQLite database"); + const { call } = await connectLocalToolsMcp(); + const corrupt = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(corrupt.exitCode).toBe(1); + expect(corrupt.json).toMatchObject({ code: "invalid_request" }); + expect((await call("inspect_sqlite_database", { path })).isError).toBe( + true, + ); + const valid = join(root, "valid.db"); + const database = new DatabaseSync(valid); + try { + database.exec( + "CREATE TABLE present(value TEXT); CREATE VIRTUAL TABLE search USING fts5(body);", + ); + } finally { + database.close(); + } + const bytes = await readFile(valid); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) throw new Error("Virtual declaration missing"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(valid, bytes); + const schema = await call("inspect_sqlite_database", { path: valid }); + expect(schema.isError).not.toBe(true); + expect(schema.structuredContent).toMatchObject({ + normalized_result: { + schema: { + tables: expect.arrayContaining([ + expect.objectContaining({ name: "search_data", kind: "unknown" }), + ]), + }, + }, + }); + const ambiguous = await cli.run({ + arguments: [ + "inspect-sqlite-database", + valid, + "--table", + "search_data", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(ambiguous.exitCode).toBe(1); + expect(ambiguous.stdout).toContain("shadow table"); + expect( + ( + await call("inspect_sqlite_database", { + path: valid, + table: "search_data", + }) + ).isError, + ).toBe(true); + const missing = await cli.run({ + arguments: [ + "inspect-sqlite-database", + valid, + "--table", + "absent", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(missing.exitCode).toBe(1); + expect(missing.stdout).toContain("absent"); + expect( + (await call("inspect_sqlite_database", { path: valid, table: "absent" })) + .isError, + ).toBe(true); + for (const input of [ + { path: valid, row_limit: 1 }, + { path: valid, table: "present", row_limit: 0 }, + { path: valid, query: "DELETE FROM present" }, + ]) { + expect( + (await call("inspect_sqlite_database", input)).isError, + JSON.stringify(input), + ).toBe(true); + } + }, +); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index b8924d34c08..2785eabe9da 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -345,6 +345,10 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn }); it.each([ + [ + ".github/workflows/sqlite-runtime.yml", + { workflows: true, code: true, package: false, website: false }, + ], [ ".github/workflows/real-termux.yml", { workflows: true, code: false, package: false, website: false }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts new file mode 100644 index 00000000000..3f5640bef14 --- /dev/null +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -0,0 +1,78 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const workflow = async () => + z + .object({ + jobs: z.record( + z.string(), + z + .object({ + needs: z.union([z.string(), z.array(z.string())]).optional(), + }) + .passthrough(), + ), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ), + ), + ); + +const requiredNeeds = async () => { + const { jobs } = await workflow(); + return { jobs, needs: z.array(z.string()).parse(jobs.required?.needs) }; +}; + +it("makes the required gate wait for every CI lane, including future additions", async () => { + const { jobs, needs } = await requiredNeeds(); + expect(new Set(needs)).toEqual( + new Set(Object.keys(jobs).filter((name) => name !== "required")), + ); +}); + +it.each([ + ["true", "success", true], + ["true", "failure", false], + ["true", "cancelled", false], + ["true", "skipped", false], + ["true", "missing", false], + ["false", "skipped", true], +] as const)( + "enforces the SQLite required gate for code=%s and result=%s", + async (code, result, passes) => { + const workflow = await requiredNeeds(); + // GitHub exposes only direct dependencies in the required job's needs context. + const needs: Record< + string, + { result: string; outputs?: Record } + > = Object.fromEntries( + workflow.needs.map((name) => [name, { result: "success" }]), + ); + needs.changes = { result: "success", outputs: { code } }; + if (result === "missing") delete needs["sqlite-runtime"]; + else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result }; + const execution = executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { env: { ...process.env, CI_JOBS: JSON.stringify(needs) } }, + ); + if (passes) await expect(execution).resolves.toMatchObject({ stderr: "" }); + else + await expect(execution).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("sqlite-runtime"), + }); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceShadowedRequire.test.ts b/tests/boundary/filesystem/referenceSourceShadowedRequire.test.ts index 9ffe7ef95be..8c438e64737 100644 --- a/tests/boundary/filesystem/referenceSourceShadowedRequire.test.ts +++ b/tests/boundary/filesystem/referenceSourceShadowedRequire.test.ts @@ -112,9 +112,7 @@ it("resolves require calls by lexical binding and source mode", async () => { result.value.relationships .filter(({ kind }) => kind === "requires") .map(({ from_path, to }) => [from_path, to]) - .sort((left, right) => - (left[0] ?? "").localeCompare(right[0] ?? ""), - ), + .sort((left, right) => (left[0] ?? "").localeCompare(right[0] ?? "")), ).toEqual([ ["create-require-esm.mjs", "loaded.cjs"], ["nested.cjs", "loaded.cjs"], diff --git a/tests/boundary/mcp/serverLifecycleMcp.test.ts b/tests/boundary/mcp/serverLifecycleMcp.test.ts index 6a140befe76..3decf0361a0 100644 --- a/tests/boundary/mcp/serverLifecycleMcp.test.ts +++ b/tests/boundary/mcp/serverLifecycleMcp.test.ts @@ -8,9 +8,141 @@ import { HopperRemoteError } from "../../../src/domain/hopperErrors.js"; import { err } from "../../../src/domain/result.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createServer } from "../../../src/server/createServer.js"; +import { SqliteDatabaseService } from "../../../src/application/sqlite/SqliteDatabaseService.js"; +import { PassThrough } from "node:stream"; +import { once } from "node:events"; +import { access } from "node:fs/promises"; +import { + serveStdio, + StdioServerTransport, +} from "@modelcontextprotocol/server/stdio"; +import pino from "pino"; +import { createManagedBinarySession } from "../../../src/composition/binary.js"; +import { startMcpTransport } from "../../../src/main/transport.js"; +import { createToolResultDelivery } from "../../../src/server/toolResult.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; const resources: Array<{ close(): Promise }> = []; +it.each(["handle", "stdin"] as const)( + "retains failed SQLite cleanup through production stdio shutdown: %s", + async (shutdown) => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-sqlite-stdio-cleanup-", + }); + let removalAllowed = false; + let attempts = 0; + const sqliteDatabase = new SqliteDatabaseService({ + inspect: async () => { + throw new Error("No inspection requested"); + }, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("SQLite snapshot removal denied"); + await root.close(); + }, + }); + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const logger = pino({ enabled: false }); + const session = createManagedBinarySession(); + const started = await startMcpTransport( + { + env: {}, + serve: (factory, options) => + serveStdio(factory, { + ...options, + transport: new StdioServerTransport(stdin, stdout), + }), + createServer: (source, options) => + createServer(source, { ...options, sqliteDatabase }), + writeStderr: () => undefined, + setExitCode: () => undefined, + registerShutdown: () => () => undefined, + }, + session, + { + environment: {}, + delivery: createToolResultDelivery(undefined), + logger, + serverLogger: logger, + loadOptionalProviders: async () => ({}), + }, + ); + if (!started.ok) throw new Error("Transport startup failed"); + try { + const response = once(stdout, "data"); + stdin.write( + `${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "sqlite-cleanup-test", version: "1.0.0" }, + }, + })}\n`, + ); + const [reply] = await response; + expect(JSON.parse(String(reply))).toMatchObject({ + id: 1, + result: { serverInfo: { name: "rea" } }, + }); + if (shutdown === "stdin") { + stdin.end(); + await new Promise((resolve) => setImmediate(resolve)); + } + await expect(started.handle.close()).rejects.toBeInstanceOf( + AggregateError, + ); + expect(attempts).toBeGreaterThan(0); + await expect(access(root.path)).resolves.toBeUndefined(); + removalAllowed = true; + await started.handle.close(); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + const completedAttempts = attempts; + await started.handle.close(); + expect(attempts).toBe(completedAttempts); + } finally { + removalAllowed = true; + await started.handle.close(); + await started.closeAndroid(); + await session.close(); + await root.close(); + stdin.destroy(); + stdout.destroy(); + } + }, +); + +it("awaits SQLite cleanup and retries its retained owner after failed server shutdown", async () => { + let removalAllowed = false; + let attempts = 0; + const sqliteDatabase = new SqliteDatabaseService({ + inspect: async () => { + throw new Error("No inspection requested"); + }, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("SQLite snapshot removal denied"); + }, + }); + const server = createServer( + { kind: "fixed", analysis: { execute: async () => ok(null) } }, + { sqliteDatabase }, + ); + resources.push(server); + try { + await expect(server.close()).rejects.toBeInstanceOf(AggregateError); + expect(attempts).toBe(1); + } finally { + removalAllowed = true; + } + await server.close(); + expect(attempts).toBe(2); +}); + afterEach(async () => { await Promise.all( resources.splice(0).map(async (resource) => resource.close()), diff --git a/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts new file mode 100644 index 00000000000..cdfb8e2c7c1 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts @@ -0,0 +1,260 @@ +import { access } from "node:fs/promises"; +import { expect, it, onTestFinished } from "vitest"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; +import { ProviderProcessSupervisor } from "../../../src/process/ProviderProcess.js"; +import { + spawnProviderProcessFixture, + stopProviderProcessFixture, + waitForProviderProcessReady, +} from "../../fixtures/providerProcess.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const createUnavailableProvider = ( + createRuntime: SqliteDatabaseProvider["createRuntime"], +) => + new SqliteDatabaseProvider( + {}, + () => { + throw new Error("SQLite worker deliberately unavailable"); + }, + createRuntime, + ); + +const cleanupFixture = async () => { + const workspace = await createTestWorkspace("rea-sqlite-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const roots: PrivateRuntimeRoot[] = []; + let removalAllowed = false; + let attempts = 0; + const provider = createUnavailableProvider(async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + return { + path: root.path, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("snapshot removal denied"); + await root.close(); + }, + }; + }); + onTestFinished(async () => { + await Promise.all(roots.map((root) => root.close())); + await removeTestWorkspace(workspace.root); + }); + return { + provider, + roots, + path: database.path, + allowRemoval: () => { + removalAllowed = true; + }, + attempts: () => attempts, + }; +}; + +it("retains failed snapshot owners and blocks new acquisition until retry succeeds", async () => { + const fixture = await cleanupFixture(); + const first = await fixture.provider.inspect({ path: fixture.path }); + expect(first).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + const root = fixture.roots[0]; + if (root === undefined) throw new Error("Missing owned root"); + await expect(access(root.path)).resolves.toBeUndefined(); + const second = await fixture.provider.inspect({ path: fixture.path }); + expect(second).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(fixture.roots).toHaveLength(1); + expect(fixture.attempts()).toBe(2); + fixture.allowRemoval(); + await fixture.provider.inspect({ path: fixture.path }); + expect(fixture.roots).toHaveLength(2); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + await fixture.provider.close(); +}); + +it("awaits shared shutdown cleanup and retains failures for a later close", async () => { + const fixture = await cleanupFixture(); + await fixture.provider.inspect({ path: fixture.path }); + const closed = await Promise.allSettled([ + fixture.provider.close(), + fixture.provider.close(), + ]); + expect(closed.map((result) => result.status)).toEqual([ + "rejected", + "rejected", + ]); + expect(fixture.attempts()).toBe(2); + expect(await fixture.provider.inspect({ path: fixture.path })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + fixture.allowRemoval(); + await fixture.provider.close(); + await fixture.provider.close(); + expect(fixture.attempts()).toBe(3); + for (const root of fixture.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("waits for concurrent acquisitions and releases only owners whose cleanup succeeds", async () => { + const workspace = await createTestWorkspace("rea-sqlite-concurrent-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const acquired = createDeferred(); + const release = createDeferred(); + const roots: PrivateRuntimeRoot[] = []; + const allowed = new Set(); + const provider = createUnavailableProvider(async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + if (roots.length === 2) acquired.resolve(); + await release.promise; + return { + path: root.path, + close: async () => { + if (!allowed.has(root)) throw new Error("snapshot removal denied"); + await root.close(); + }, + }; + }); + onTestFinished(async () => { + release.resolve(); + roots.forEach((root) => allowed.add(root)); + await provider.close(); + await removeTestWorkspace(workspace.root); + }); + const inspections = [ + provider.inspect({ path: database.path }), + provider.inspect({ path: database.path }), + ]; + await acquired.promise; + let settled = false; + const closing = provider.close().finally(() => { + settled = true; + }); + const rejected = expect(closing).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + release.resolve(); + await Promise.all(inspections); + await rejected; + const [first, second] = roots; + if (first === undefined || second === undefined) + throw new Error("Missing owned roots"); + allowed.add(first); + await expect(provider.close()).rejects.toMatchObject({ + cleanupResources: [second.path], + }); + await expect(access(first.path)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(second.path)).resolves.toBeUndefined(); + allowed.add(second); + await provider.close(); + await expect(access(second.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("retains the snapshot until a failed worker cleanup owner can be stopped", async () => { + const workspace = await createTestWorkspace("rea-sqlite-worker-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const child = spawnProviderProcessFixture("graceful"); + let workerRemovalAllowed = false; + let rootRemovalAllowed = false; + let workerAttempts = 0; + let rootAttempts = 0; + const roots: PrivateRuntimeRoot[] = []; + const supervisor = new ProviderProcessSupervisor({ + process: child, + ownsProcessLifetime: true, + cleanup: async () => { + workerAttempts += 1; + if (!workerRemovalAllowed) + return { cleaned: false, reason: "SQLite worker cleanup denied" }; + await stopProviderProcessFixture(child); + return { cleaned: true, signaled: true }; + }, + }); + const provider = new SqliteDatabaseProvider( + {}, + async (spawn) => { + const stopped = await supervisor.stop(); + if (stopped.status !== "incomplete") + throw new Error("Expected retained worker"); + throw new OwnedCommandFailure( + "process", + "SQLite worker cleanup failed", + supervisor.snapshot(), + stopped.reason, + undefined, + [spawn.runId, spawn.cwd ?? ""], + supervisor, + ); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + return { + path: root.path, + close: async () => { + rootAttempts += 1; + if (!rootRemovalAllowed) + throw new Error("SQLite snapshot removal denied"); + await root.close(); + }, + }; + }, + ); + onTestFinished(async () => { + workerRemovalAllowed = true; + rootRemovalAllowed = true; + await supervisor.stop(); + await provider.close(); + await Promise.all(roots.map((root) => root.close())); + await removeTestWorkspace(workspace.root); + }); + await waitForProviderProcessReady(child); + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + // The snapshot cannot be removed while its worker may still use it. + expect(rootAttempts).toBe(0); + const root = roots[0]; + if (root === undefined) throw new Error("Missing retained snapshot"); + await expect(access(root.path)).resolves.toBeUndefined(); + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(roots).toHaveLength(1); + expect(workerAttempts).toBe(2); + await expect(provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(rootAttempts).toBe(0); + workerRemovalAllowed = true; + await expect(provider.close()).rejects.toMatchObject({ + cleanupResources: [root.path], + }); + expect(child.signalCode).toBe("SIGKILL"); + expect(workerAttempts).toBe(4); + expect(rootAttempts).toBe(1); + rootRemovalAllowed = true; + await provider.close(); + expect(workerAttempts).toBe(4); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts new file mode 100644 index 00000000000..f656c35f480 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts @@ -0,0 +1,378 @@ +import { Cli } from "incur"; +import { access, writeFile } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it, onTestFinished, vi } from "vitest"; + +import { SqliteDatabaseService } from "../../../src/application/sqlite/SqliteDatabaseService.js"; +import { registerSqliteCommands } from "../../../src/cli/sqliteCommands.js"; +import { createSqliteDatabaseService } from "../../../src/composition/sqlite.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { silentLogger } from "../../../src/logger.js"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const descriptorCleanup = vi.hoisted(() => ({ + path: "", + denied: false, + invalidate: false, + attempts: 0, + handles: [] as FileHandle[], + readFailure: undefined as Error | undefined, + onOpen: undefined as (() => void) | undefined, +})); + +vi.mock("../../../src/composition/sqlite.js", () => ({ + createSqliteDatabaseService: vi.fn(), +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const filesystem = await importOriginal(); + return { + ...filesystem, + open: async (...args: Parameters) => { + const handle = await filesystem.open(...args); + if (args[0] === descriptorCleanup.path) { + descriptorCleanup.handles.push(handle); + const close = handle.close.bind(handle); + handle.close = async () => { + // FileHandle.close also asks attached streams to close the invalidated handle. + if (handle.fd < 0) { + await close(); + return; + } + descriptorCleanup.attempts += 1; + if (descriptorCleanup.denied) { + if (descriptorCleanup.invalidate) await close(); + throw new Error("injected SQLite descriptor close failure"); + } + await close(); + }; + if (descriptorCleanup.readFailure !== undefined) + vi.spyOn(handle, "stat").mockRejectedValue( + descriptorCleanup.readFailure, + ); + descriptorCleanup.onOpen?.(); + } + return handle; + }, + }; +}); + +const fixture = async ( + member: "database" | "WAL" | "journal" | "reply", + invalidate = false, +) => { + const workspace = await createTestWorkspace("rea-sqlite-descriptor-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root, true); + const path = + member === "database" + ? database.path + : member === "WAL" + ? database.walPath + : member === "journal" + ? `${database.path}-journal` + : ""; + if (member === "journal") await writeFile(path, Buffer.alloc(28)); + const roots: PrivateRuntimeRoot[] = []; + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => { + if (member !== "reply") + throw new Error("SQLite worker deliberately unavailable"); + return spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 + ? fileURLToPath( + new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, + ), + ) + : argument, + ), + }); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + if (member === "reply") + descriptorCleanup.path = join(root.path, "reply.json"); + roots.push(root); + return root; + }, + ); + descriptorCleanup.path = path; + descriptorCleanup.denied = true; + descriptorCleanup.invalidate = invalidate; + descriptorCleanup.attempts = 0; + descriptorCleanup.handles = []; + descriptorCleanup.readFailure = undefined; + descriptorCleanup.onOpen = undefined; + onTestFinished(async () => { + descriptorCleanup.denied = false; + try { + await provider.close(); + } catch (cause) { + if (!invalidate) throw cause; + } + for (const handle of descriptorCleanup.handles) await handle.close(); + descriptorCleanup.path = ""; + database.close(); + await removeTestWorkspace(workspace.root); + }); + return { provider, roots, path, database }; +}; + +const serveInspection = async ( + provider: SqliteDatabaseProvider, + path: string, +) => { + const previousExitCode = process.exitCode; + onTestFinished(() => { + process.exitCode = previousExitCode; + }); + vi.mocked(createSqliteDatabaseService).mockReturnValue( + new SqliteDatabaseService(provider), + ); + const cli = Cli.create("rea-sqlite-cleanup-test", { sync: false }); + registerSqliteCommands(cli, silentLogger, {}); + let stdout = ""; + await cli.serve(["inspect-sqlite-database", path, "--json"], { + env: {}, + stdout: (text) => { + stdout += text; + }, + exit: () => {}, + }); + return { output: JSON.parse(stdout) as unknown, exitCode: process.exitCode }; +}; + +it.each(["database", "WAL", "journal"] as const)( + "retains a failed %s descriptor and retries it at shutdown", + async (member) => { + const selected = await fixture(member); + const first = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(first).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [expect.stringContaining(selected.path)], + }, + }); + const handle = descriptorCleanup.handles[0]; + if (handle === undefined) + throw new Error("Expected owned source descriptor"); + expect(handle.fd).toBeGreaterThanOrEqual(0); + const blocked = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(blocked).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(selected.roots).toHaveLength(1); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(descriptorCleanup.attempts).toBe(3); + descriptorCleanup.denied = false; + await selected.provider.close(); + expect(descriptorCleanup.attempts).toBe(4); + expect(handle.fd).toBe(-1); + for (const root of selected.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it.each(["EACCES", "EPERM"])( + "retains the original %s inspection diagnostics through CLI shutdown", + async (code) => { + const selected = await fixture("database"); + descriptorCleanup.readFailure = Object.assign( + new Error("injected source read denial"), + { code }, + ); + const result = await serveInspection( + selected.provider, + selected.database.path, + ); + expect(result).toMatchObject({ + exitCode: 1, + output: { + code: "cleanup_incomplete", + details: { + resources: [expect.stringContaining(selected.path)], + diagnostics: { + primary_error: { + details: { + diagnostics: { + previous_error: { + code: "access_denied", + details: { path: selected.path, system_code: code }, + }, + }, + }, + }, + cleanup_error: { code: "cleanup_incomplete" }, + }, + }, + }, + }); + expect(descriptorCleanup.attempts).toBe(2); + }, +); + +it("retains cancellation diagnostics and the interrupt exit code through CLI shutdown", async () => { + const selected = await fixture("database"); + descriptorCleanup.onOpen = () => { + const interrupt = process.listeners("SIGINT").at(-1); + if (interrupt === undefined) + throw new Error("Missing CLI interrupt handler"); + interrupt("SIGINT"); + }; + const result = await serveInspection( + selected.provider, + selected.database.path, + ); + expect(result).toMatchObject({ + exitCode: 130, + output: { + code: "cleanup_incomplete", + details: { + resources: [expect.stringContaining(selected.path)], + diagnostics: { + primary_error: { + details: { + diagnostics: { previous_error: { code: "cancelled" } }, + }, + }, + cleanup_error: { code: "cleanup_incomplete" }, + }, + }, + }, + }); + expect(descriptorCleanup.attempts).toBeGreaterThanOrEqual(2); +}); + +it("retains reply-read cancellation and retries the failed reply descriptor close", async () => { + const selected = await fixture("reply"); + const controller = new AbortController(); + descriptorCleanup.onOpen = () => + controller.abort(new Error("requested cancellation during reply read")); + const result = await selected.provider.inspect( + { path: selected.database.path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancelled reply read"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + execution_failure: "cancelled", + resources: [expect.stringContaining(descriptorCleanup.path)], + captured_output: { + stdout: "", + stderr: expect.any(String), + truncated: false, + }, + }, + }); + const handle = descriptorCleanup.handles[0]; + if (handle === undefined) throw new Error("Expected owned reply descriptor"); + expect(handle.fd).toBeGreaterThanOrEqual(0); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + descriptorCleanup.denied = false; + await selected.provider.close(); + expect(handle.fd).toBe(-1); + for (const root of selected.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("retries source cleanup before admitting another snapshot", async () => { + const selected = await fixture("database"); + await selected.provider.inspect({ path: selected.database.path }); + const first = descriptorCleanup.handles[0]; + if (first === undefined) throw new Error("Expected source descriptor"); + descriptorCleanup.denied = false; + const resumed = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(selected.roots).toHaveLength(2); + expect(first.fd).toBe(-1); + if (resumed.ok) throw new Error("The worker is deliberately unavailable"); + expect(resumed.error.cleanupIncomplete).toBe(false); + await selected.provider.close(); +}); + +it("keeps an invalidated descriptor close failure unconfirmed across retries", async () => { + const selected = await fixture("journal", true); + const first = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(first).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(descriptorCleanup.handles[0]?.fd).toBe(-1); + descriptorCleanup.denied = false; + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(descriptorCleanup.attempts).toBe(1); +}); + +it.each(["EACCES", "EPERM"])( + "preserves %s read diagnostics alongside a failed descriptor close", + async (code) => { + const selected = await fixture("database"); + descriptorCleanup.readFailure = Object.assign( + new Error("injected source read denial"), + { code }, + ); + const result = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + diagnostics: { previous_error: { code: "access_denied" } }, + }, + }); + }, +); + +it.each(["database", "WAL", "journal"] as const)( + "preserves caller cancellation alongside a failed %s descriptor close", + async (member) => { + const selected = await fixture(member); + const controller = new AbortController(); + descriptorCleanup.onOpen = () => + controller.abort(new Error("requested caller cancellation")); + const result = await selected.provider.inspect( + { path: selected.database.path }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + diagnostics: { previous_error: { code: "cancelled" } }, + }, + }); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts new file mode 100644 index 00000000000..7695571ecd1 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -0,0 +1,301 @@ +import { DatabaseSync } from "node:sqlite"; +import { readFile, writeFile } from "node:fs/promises"; +import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../../fixtures/sqlite/database.js"; +import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; +import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const it = nativeTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE); + +const createDatabase = async (sql: string): Promise => { + const workspace = await createTestWorkspace("rea-sqlite-expansion-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("private.db"); + const producer = new DatabaseSync(path); + try { + producer.exec(sql); + } finally { + producer.close(); + } + return path; +}; + +it("preserves negative-zero REAL values through JSON separately from positive and integer zero", async () => { + const path = await createDatabase(` + CREATE TABLE numbers(value); + INSERT INTO numbers VALUES (-0.0), (0.0), (0), (5e-324), (-5e-324); + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "numbers", + }); + expect(JSON.parse(JSON.stringify(result.rows))).toMatchObject({ + values: [ + [{ type: "real", value: "-0" }], + [{ type: "real", value: 0 }], + [{ type: "integer", value: "0" }], + [{ type: "real", value: Number.MIN_VALUE }], + [{ type: "real", value: -Number.MIN_VALUE }], + ], + }); +}); + +it.each(["table", "view"] as const)( + "inspects columns when a %s shadows the table-valued PRAGMA name", + async (kind) => { + const path = await createDatabase(` + CREATE TABLE "selected "" table" ("0" TEXT, "" INTEGER); + INSERT INTO "selected "" table" VALUES ('retained', 7); + ${kind === "table" ? "CREATE TABLE pragma_table_xinfo(value TEXT)" : "CREATE VIEW pragma_table_xinfo AS SELECT 1"}; + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: 'selected " table', + }); + expect(result.schema.tables).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'selected " table', + columns: [ + expect.objectContaining({ name: "0", declared_type: "TEXT" }), + expect.objectContaining({ name: "", declared_type: "INTEGER" }), + ], + }), + ]), + ); + expect(result.rows).toMatchObject({ + columns: ["0", ""], + values: [ + [ + { type: "text", value: "retained" }, + { type: "integer", value: "7" }, + ], + ], + }); + }, +); + +it.each([6, 9])( + "preserves a %i MiB schema statement without expanding its native record", + async (mebibytes) => { + const trigger = `CREATE TRIGGER large_definition AFTER INSERT ON records BEGIN /*${"x".repeat(mebibytes * 1024 * 1024)}*/ SELECT 1; END`; + const path = await createDatabase(` + CREATE TABLE records(value TEXT); + ${trigger}; + `); + const result = inspectSqliteDatabaseSnapshot(path, { path }); + expect(result.schema.triggers).toEqual([ + { name: "large_definition", table: "records", sql: trigger }, + ]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); + }, +); + +it("inspects a large schema identifier without expanding a native sort record", async () => { + const name = "x".repeat(4 * 1024 * 1024); + const sql = `CREATE TABLE "${name}" (value TEXT)`; + const path = await createDatabase(sql); + const result = inspectSqliteDatabaseSnapshot(path, { path }); + expect(result.schema.tables).toMatchObject([ + { name, sql, columns: [{ name: "value", declared_type: "TEXT" }] }, + ]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); +}); + +it.each(["UTF-8", "UTF-16le", "UTF-16be"])( + "preserves exact %s schema text when decoding native bytes", + async (encoding) => { + const sql = 'CREATE TABLE "\uFEFFé𝄞" ("列" TEXT DEFAULT \'\uFEFFvalue\')'; + const path = await createDatabase(`PRAGMA encoding='${encoding}'; ${sql}`); + const result = inspectSqliteDatabaseSnapshot(path, { path }); + expect(result.schema.tables).toMatchObject([ + { + name: "\uFEFFé𝄞", + sql, + columns: [{ name: "列", default_sql: "'\uFEFFvalue'" }], + }, + ]); + }, +); + +it("counts schema names once when enforcing the complete reply budget", async () => { + const names = Array.from( + { length: 48 }, + (_, index) => `${index}_${"x".repeat(128 * 1024)}`, + ); + const path = await createDatabase( + names.map((name) => `CREATE TABLE "${name}" (value TEXT);`).join("\n"), + ); + const result = inspectSqliteDatabaseSnapshot(path, { path }); + expect(result.schema.tables).toHaveLength(names.length); + expect(result.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); +}); + +it("rejects malformed schema bytes instead of returning replacement text", async () => { + const path = await createDatabase( + "CREATE TABLE records(value TEXT /*schema-marker*/)", + ); + const bytes = await readFile(path); + const marker = bytes.indexOf(Buffer.from("schema-marker")); + if (marker < 0) throw new Error("Schema marker missing"); + bytes[marker] = 0xff; + await writeFile(path, bytes); + expect(() => inspectSqliteDatabaseSnapshot(path, { path })).toThrow( + /schema sql contains malformed UTF-8 text/, + ); +}); + +it("refuses possible shadow tables when their virtual module is unavailable", async () => { + const path = await createDatabase(` + CREATE VIRTUAL TABLE "SeArCh_É" USING fts5(body); + INSERT INTO "SeArCh_É" VALUES ('retained'); + CREATE TABLE "search_É_manual" (value TEXT); + CREATE TABLE "search_é_manual" (value TEXT); + INSERT INTO "search_é_manual" VALUES ('distinct Unicode name'); + CREATE TABLE "SeArCh_Élite" (value TEXT); + `); + const bytes = await readFile(path); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) throw new Error("Virtual declaration missing"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(path, bytes); + + for (const table of ["SeArCh_É_data", "search_É_manual"]) + expect(() => inspectSqliteDatabaseSnapshot(path, { path, table })).toThrow( + /cannot determine.*shadow table/i, + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "search_é_manual", + }); + expect(result.schema.tables).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "SeArCh_É", kind: "virtual" }), + expect.objectContaining({ name: "SeArCh_É_data", kind: "unknown" }), + expect.objectContaining({ name: "search_É_manual", kind: "unknown" }), + expect.objectContaining({ name: "search_é_manual", kind: "table" }), + expect.objectContaining({ name: "SeArCh_Élite", kind: "table" }), + ]), + ); + expect(result.rows).toMatchObject({ + values: [[{ type: "text", value: "distinct Unicode name" }]], + }); + expect(result.limitations.join("\n")).toMatch(/shadow table/i); +}); + +it("rejects an oversized generated cell through SQLite's native value limit", async () => { + const path = await createDatabase(` + CREATE TABLE records ( + id INTEGER, + payload BLOB GENERATED ALWAYS AS (zeroblob(72 * 1024 * 1024)) VIRTUAL + ); + INSERT INTO records(id) VALUES (1); + `); + try { + inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + throw new Error("An oversized generated cell must not be projected"); + } catch (cause: unknown) { + expect(cause).toMatchObject({ + code: "ERR_SQLITE_ERROR", + errcode: 18, + }); + } +}); + +it("reports additional rows without expanding an unselected generated cell", async () => { + const path = await createDatabase(` + CREATE TABLE records ( + id INTEGER, + payload BLOB GENERATED ALWAYS AS ( + CASE WHEN id = 2 THEN zeroblob(72 * 1024 * 1024) ELSE X'01' END + ) VIRTUAL + ); + INSERT INTO records(id) VALUES (1), (2); + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + expect(result.rows).toMatchObject({ + columns: ["id", "payload"], + values: [ + [ + { type: "integer", value: "1" }, + { type: "blob", hex: "01" }, + ], + ], + returned_rows: 1, + truncated: true, + }); +}); + +it("rejects the combined representation of individually small generated cells", async () => { + const columns = Array.from( + { length: 6 }, + (_, index) => + `payload_${String(index)} BLOB GENERATED ALWAYS AS (zeroblob(2 * 1024 * 1024)) VIRTUAL`, + ); + const path = await createDatabase(` + CREATE TABLE records (id INTEGER, ${columns.join(", ")}); + INSERT INTO records(id) VALUES (1); + `); + try { + inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + throw new Error( + "Aggregate row expansion must be rejected before projection", + ); + } catch (cause: unknown) { + expect(cause).toBeInstanceOf(SqliteInspectionFailure); + expect(cause).toMatchObject({ reason: "output-limit" }); + } +}); + +it("samples above 1000 rows and safe integer limits within the byte budget", async () => { + const path = await createDatabase(`CREATE TABLE items(value INTEGER); + WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002) + INSERT INTO items SELECT value FROM numbers;`); + for (const row_limit of [1001, Number.MAX_SAFE_INTEGER]) { + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + row_limit, + }); + expect(result.rows).toMatchObject({ + row_limit, + returned_rows: Math.min(row_limit, 1002), + truncated: row_limit < 1002, + }); + } +}); + +nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "rejects a runtime without native limits before querying the snapshot", + async () => { + const path = await createDatabase("CREATE TABLE items(value INTEGER)"); + expect(() => inspectSqliteDatabaseSnapshot(path, { path })).toThrow( + "DatabaseSync.limits", + ); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts new file mode 100644 index 00000000000..7eca2d32ba2 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -0,0 +1,513 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { access, lstat, readFile, symlink, writeFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; +import { + createSqliteDatabaseFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const it = nativeTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE); + +const workerUrl = new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, +); +const createProvider = () => + new SqliteDatabaseProvider(process.env, (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 ? fileURLToPath(workerUrl) : argument, + ), + }), + ); + +const createFixture = async (wal = false) => { + const workspace = await createTestWorkspace("rea-sqlite-boundary-"); + const database = createSqliteDatabaseFixture(workspace.root, wal); + if (!wal) database.close(); + onTestFinished(async () => { + if (wal) database.close(); + await removeTestWorkspace(workspace.root); + }); + return { workspace, database }; +}; + +const expectPrivateRootRemoved = (path: string) => + expect(access(path)).rejects.toMatchObject({ code: "ENOENT" }); + +it("inspects committed WAL schema and rows while preserving every source artifact", async () => { + const { database } = await createFixture(true); + const paths = [database.path, database.walPath, `${database.path}-shm`]; + const original = await Promise.all( + paths.map(async (path) => ({ + path, + bytes: await readFile(path), + stat: await lstat(path), + })), + ); + const result = await createProvider().inspect({ + path: database.path, + table: "records", + row_limit: 1, + }); + if (!result.ok) throw result.error; + const main = original[0]; + const wal = original[1]; + if (main === undefined || wal === undefined) + throw new Error("Missing fixture identity"); + expect(result.value.result).toMatchObject({ + artifact: { + path: database.path, + bytes: main.bytes.length, + sha256: createHash("sha256").update(main.bytes).digest("hex"), + }, + wal: { + path: database.walPath, + bytes: wal.bytes.length, + sha256: createHash("sha256").update(wal.bytes).digest("hex"), + }, + schema: { + completeness: "complete", + tables: expect.arrayContaining([ + { + name: "records", + sql: expect.stringContaining("CREATE TABLE"), + root_page: expect.any(Number), + kind: "table", + without_rowid: false, + strict: false, + columns_completeness: "complete", + columns: expect.arrayContaining([ + expect.objectContaining({ name: "generated_value", hidden: 2 }), + ]), + }, + ]), + indexes: [ + expect.objectContaining({ name: "records_text", table: "records" }), + ], + views: [ + { + name: "record_names", + sql: expect.stringContaining("SELECT text_value"), + }, + ], + triggers: [ + { + name: "records_insert", + table: "records", + sql: expect.stringContaining("AFTER INSERT"), + }, + ], + }, + rows: { + table: "records", + returned_rows: 1, + truncated: true, + row_limit: 1, + order: "unspecified", + }, + }); + for (const artifact of original) { + expect(await readFile(artifact.path)).toEqual(artifact.bytes); + const after = await lstat(artifact.path); + expect({ + ino: after.ino, + size: after.size, + mtimeMs: after.mtimeMs, + ctimeMs: after.ctimeMs, + }).toEqual({ + ino: artifact.stat.ino, + size: artifact.stat.size, + mtimeMs: artifact.stat.mtimeMs, + ctimeMs: artifact.stat.ctimeMs, + }); + } +}); + +it("preserves SQLite integer, binary, nonfinite, null, and text byte values", async () => { + const { database } = await createFixture(); + const result = await createProvider().inspect({ + path: database.path, + table: "records", + row_limit: 10, + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + wal: null, + rows: { + columns: [ + "id", + "integer_value", + "blob_value", + "real_value", + "nullable_value", + "text_value", + "generated_value", + ], + returned_rows: 3, + truncated: false, + values: [ + [ + { type: "integer", value: "1" }, + { type: "integer", value: "9223372036854775807" }, + { type: "blob", hex: "001fefff" }, + { type: "real", value: "Infinity" }, + { type: "null" }, + { + type: "text", + value: "first é\0value", + bytes_hex: Buffer.from("first é\0value").toString("hex"), + }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + [ + { type: "integer", value: "2" }, + { type: "integer", value: "-9223372036854775808" }, + { type: "blob", hex: "" }, + { type: "real", value: "-Infinity" }, + { + type: "text", + value: "present", + bytes_hex: Buffer.from("present").toString("hex"), + }, + { + type: "text", + value: "second", + bytes_hex: Buffer.from("second").toString("hex"), + }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + [ + { type: "integer", value: "3" }, + { type: "integer", value: "0" }, + { type: "blob", hex: "ff" }, + { type: "real", value: 1.25 }, + { type: "null" }, + { type: "text", bytes_hex: "ff00c0af" }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + ], + }, + }); + expect(() => JSON.stringify(result.value.result)).not.toThrow(); +}); + +it("selects an exact quoted table and refuses to execute table input as SQL", async () => { + const { database } = await createFixture(); + const provider = createProvider(); + const selected = await provider.inspect({ + path: database.path, + table: database.quotedTable, + row_limit: 1, + }); + if (!selected.ok) throw selected.error; + expect(selected.value.result).toMatchObject({ + rows: { + table: database.quotedTable, + columns: ["__proto__"], + values: [ + [ + { + type: "text", + value: "exact selected table", + bytes_hex: Buffer.from("exact selected table").toString("hex"), + }, + ], + ], + returned_rows: 1, + truncated: false, + }, + }); + const unknown = await provider.inspect({ + path: database.path, + table: 'records"; DROP TABLE records;--', + row_limit: 1, + }); + expect(unknown).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + const after = await provider.inspect({ + path: database.path, + table: "records", + row_limit: 10, + }); + if (!after.ok) throw after.error; + expect(after.value.result).toMatchObject({ rows: { returned_rows: 3 } }); +}); + +it("does not execute views when selecting rows", async () => { + const { database } = await createFixture(); + const result = await createProvider().inspect({ + path: database.path, + table: "record_names", + row_limit: 1, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); +}); + +it("rejects malformed databases without altering the selected file", async () => { + const workspace = await createTestWorkspace("rea-sqlite-invalid-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = await workspace.write( + "not-a-database.db", + "selected non-SQLite bytes", + ); + const bytes = await readFile(path); + const result = await createProvider().inspect({ path }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(await readFile(path)).toEqual(bytes); +}); + +nativeTest.runIf(process.platform !== "win32")( + "rejects a symlink database", + async () => { + const { database, workspace } = await createFixture(); + const alias = workspace.path("alias.db"); + await symlink(database.path, alias); + const provider = createProvider(); + expect(await provider.inspect({ path: alias })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +nativeTest( + "rejects an actual open rollback transaction without changing its journal", + async () => { + const { database } = await createFixture(); + const writer = new DatabaseSync(database.path); + writer.exec( + "BEGIN IMMEDIATE; INSERT INTO records(id, text_value) VALUES (4, 'uncommitted')", + ); + try { + const main = await readFile(database.path); + const journalPath = `${database.path}-journal`; + const journal = await readFile(journalPath); + expect(journal.length).toBeGreaterThan(0); + expect( + await createProvider().inspect({ path: database.path }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(await readFile(database.path)).toEqual(main); + expect(await readFile(journalPath)).toEqual(journal); + } finally { + writer.exec("ROLLBACK"); + writer.close(); + } + }, +); + +nativeTest.runIf(process.platform !== "win32")( + "rejects FIFO input without waiting for a writer", + async () => { + const workspace = await createTestWorkspace("rea-sqlite-fifo-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("database.fifo"); + await promisify(execFile)("mkfifo", [path]); + const result = await createProvider().inspect({ path }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +nativeTest( + "honors a cancelled caller before opening the database", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + controller.abort(); + expect( + await createProvider().inspect( + { path: database.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + }, +); + +nativeTest( + "removes an acquired private root when cancelled before snapshot creation", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + let ownedRoot = ""; + const provider = new SqliteDatabaseProvider( + {}, + () => { + throw new Error("Cancelled request must not launch SQLite"); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-sqlite-cancel-root-", + }); + ownedRoot = root.path; + controller.abort(); + return root; + }, + ); + expect( + await provider.inspect( + { path: database.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + await expectPrivateRootRemoved(ownedRoot); + }, +); + +it.each(["UTF-8", "UTF-16le", "UTF-16be"] as const)( + "preserves %s text, a leading BOM and empty table selection", + async (encoding) => { + const workspace = await createTestWorkspace("rea-sqlite-encoding-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("encoding.db"); + const writer = new DatabaseSync(path); + writer.exec( + `PRAGMA encoding='${encoding}'; CREATE TABLE "" ("1" TEXT, "0" TEXT)`, + ); + const value = "\uFEFFé𝄞\0selected"; + writer + .prepare('INSERT INTO "" VALUES (?, ?)') + .run(value, "second numeric column"); + writer.close(); + const bytes = Buffer.from(value, encoding === "UTF-8" ? "utf8" : "utf16le"); + if (encoding === "UTF-16be") bytes.swap16(); + const result = await createProvider().inspect({ + path, + table: "", + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + database_encoding: encoding, + rows: { + table: "", + columns: ["1", "0"], + returned_rows: 1, + values: [ + [ + { type: "text", value, bytes_hex: bytes.toString("hex"), encoding }, + { type: "text", value: "second numeric column" }, + ], + ], + }, + }); + }, +); + +it("retains an unavailable virtual module definition without executing it", async () => { + const workspace = await createTestWorkspace("rea-sqlite-virtual-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("virtual.db"); + const writer = new DatabaseSync(path); + writer.exec( + "CREATE VIRTUAL TABLE optional_search USING fts5(body); INSERT INTO optional_search VALUES ('selected source')", + ); + writer.close(); + const bytes = await readFile(path); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) + throw new Error("SQLite virtual module declaration not found"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(path, bytes); + const provider = createProvider(); + const result = await provider.inspect({ path }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + rows: null, + schema: { + tables: expect.arrayContaining([ + expect.objectContaining({ + name: "optional_search", + kind: "virtual", + sql: expect.stringContaining("USING zzzz"), + columns: [], + columns_completeness: "unsupported", + }), + ]), + }, + }); + expect( + await provider.inspect({ path, table: "optional_search" }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + expect(await readFile(path)).toEqual(bytes); +}); + +nativeTest( + "cancels an acquired SQLite worker and removes its private snapshot", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + let ownedRoot = ""; + let pid: number | undefined; + const provider = new SqliteDatabaseProvider({}, async (spawn) => { + ownedRoot = spawn.cwd ?? ""; + const requestPath = spawn.arguments[2]; + if (requestPath === undefined) + throw new Error("SQLite worker arguments missing"); + const launched = await spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + "--input-type=module", + "-e", + `await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`, + "sqlite-worker-test", + requestPath, + ], + }); + pid = launched.process.pid; + await waitForProviderProcessReady(launched.process); + await access(`${ownedRoot}/reply.json`); + setImmediate(() => controller.abort()); + return launched; + }); + const result = await provider.inspect( + { path: database.path, table: "records" }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + if (pid === undefined) throw new Error("SQLite worker not acquired"); + const acquiredPid = pid; + expect(() => process.kill(acquiredPid, 0)).toThrow(); + await expectPrivateRootRemoved(ownedRoot); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts new file mode 100644 index 00000000000..0bf8b77414d --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -0,0 +1,240 @@ +import { + lstat, + open, + readFile, + rename, + symlink, + unlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import { expect, it, onTestFinished } from "vitest"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import type { StableArtifactFileSystem } from "../../../src/artifacts/readStableArtifact.js"; +import { captureSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseSnapshot.js"; +import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; +import { sqliteDatabaseFailure } from "../../../src/sqlite/SqliteDatabaseFailures.js"; +import { + createSqliteDatabaseFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const fixture = async (wal = false) => { + const workspace = await createTestWorkspace("rea-sqlite-snapshot-race-"); + const database = createSqliteDatabaseFixture(workspace.root, wal); + if (!wal) database.close(); + const resourceScope = new ArtifactResourceScope(); + onTestFinished(async () => { + await resourceScope.close(); + if (wal) database.close(); + await removeTestWorkspace(workspace.root); + }); + const root = await workspace.mkdir("private"); + const capture = (fileSystem?: StableArtifactFileSystem) => + captureSqliteDatabaseSnapshot( + database.path, + root, + resourceScope, + undefined, + fileSystem, + ); + return { workspace, database, root, capture }; +}; + +it.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE).each(["PERSIST", "TRUNCATE"])( + "inspects a committed %s journal without modifying the source file set", + async (mode) => { + const { database, capture } = await fixture(); + const writer = new DatabaseSync(database.path); + try { + writer.exec( + `PRAGMA journal_mode=${mode}; INSERT INTO records(id, text_value) VALUES (4, 'committed');`, + ); + } finally { + writer.close(); + } + const journalPath = `${database.path}-journal`; + const before = await readFile(journalPath); + const beforeStat = await lstat(journalPath, { bigint: true }); + const snapshot = await capture(); + const result = inspectSqliteDatabaseSnapshot(snapshot.snapshotPath, { + path: database.path, + table: "records", + }); + expect(result.rows).toMatchObject({ returned_rows: 4, truncated: false }); + expect(await readFile(journalPath)).toEqual(before); + const afterStat = await lstat(journalPath, { bigint: true }); + expect([ + afterStat.ino, + afterStat.size, + afterStat.mtimeNs, + afterStat.ctimeNs, + ]).toEqual([ + beforeStat.ino, + beforeStat.size, + beforeStat.mtimeNs, + beforeStat.ctimeNs, + ]); + }, +); + +it.runIf(process.platform !== "win32").each([ + ["database", "remove"], + ["WAL", "remove"], + ["database", "replace with symlink"], + ["WAL", "replace with symlink"], +])( + "reports an observed %s change during acquisition as changed evidence: %s", + async (member, change) => { + const { database, workspace, capture } = await fixture(true); + const replacement = await workspace.write("replacement", ""); + const selectedPath = + member === "database" ? database.path : database.walPath; + let observations = 0; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === selectedPath && ++observations === 2) { + await unlink(path); + if (change === "replace with symlink") + await symlink(replacement, path); + } + return lstat(path, { bigint: true }); + }, + open, + }; + try { + await capture(fileSystem); + throw new Error("A disappearing source must be rejected"); + } catch (cause) { + expect( + sqliteDatabaseFailure(cause, database.path, "artifact-read"), + ).toMatchObject({ _tag: "AnalysisArtifactChangedError" }); + expect(cause).toMatchObject({ + message: expect.stringContaining(selectedPath), + }); + } + }, +); + +it("refuses an active journal even before SQLite writes its magic header", async () => { + const { database, capture } = await fixture(); + const writer = new DatabaseSync(database.path); + try { + writer.exec( + "PRAGMA journal_mode=PERSIST; BEGIN IMMEDIATE; INSERT INTO records(id, text_value) VALUES (4, 'uncommitted');", + ); + const journalPath = `${database.path}-journal`; + const journal = await readFile(journalPath); + expect(journal.subarray(0, 8).every((byte) => byte === 0)).toBe(true); + expect(journal.subarray(8, 28).some((byte) => byte !== 0)).toBe(true); + const original = await readFile(database.path); + await expect(capture()).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("recovery"), + }); + expect(await readFile(database.path)).toEqual(original); + expect(await readFile(journalPath)).toEqual(journal); + } finally { + writer.exec("ROLLBACK"); + writer.close(); + } +}); + +it("refuses an incomplete zeroed journal header", async () => { + const { database, capture } = await fixture(); + await writeFile(`${database.path}-journal`, Buffer.alloc(27)); + await expect(capture()).rejects.toMatchObject({ reason: "format" }); +}); + +it.runIf(process.platform !== "win32")( + "refuses a journal symlink even when its target is empty", + async () => { + const { database, workspace, capture } = await fixture(); + const empty = await workspace.write("empty-journal", ""); + await symlink(empty, `${database.path}-journal`); + await expect(capture()).rejects.toMatchObject({ reason: "path" }); + }, +); + +it.each(["activate", "replace", "remove"])( + "rejects a cold journal changed during database acquisition: %s", + async (change) => { + const { database, capture } = await fixture(); + const journalPath = `${database.path}-journal`; + await writeFile(journalPath, Buffer.alloc(28)); + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) { + if (change === "activate") + await writeFile(journalPath, Buffer.alloc(28, 1)); + else if (change === "remove") await unlink(journalPath); + else { + await rename(journalPath, `${journalPath}.previous`); + await writeFile(journalPath, Buffer.alloc(28)); + } + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + }, +); + +it("rejects a database changed after its copy but before WAL acquisition", async () => { + const { database, root, capture } = await fixture(true); + const initial = await readFile(database.path); + const initialStat = await lstat(database.path); + let changed = false; + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.walPath) { + const bytes = Buffer.from(initial); + const offset = bytes.length - 1; + bytes[offset] = (bytes[offset] ?? 0) ^ 1; + await writeFile(database.path, bytes); + await utimes(database.path, initialStat.atime, initialStat.mtime); + changed = true; + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + expect(changed).toBe(true); + expect(await readFile(`${root}/database.snapshot`)).toEqual(initial); +}); + +it.each(["-wal", "-journal"] as const)( + "rejects a previously absent %s created during database acquisition", + async (suffix) => { + const { database, capture } = await fixture(); + let created = false; + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) { + await writeFile( + `${database.path}${suffix}`, + "newly selected sidecar", + ); + created = true; + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + expect(created).toBe(true); + }, +); diff --git a/tests/fixtures/android/jadx-mcp.mjs b/tests/fixtures/android/jadx-mcp.mjs index e95029d531a..d67e6a68c3a 100644 --- a/tests/fixtures/android/jadx-mcp.mjs +++ b/tests/fixtures/android/jadx-mcp.mjs @@ -148,3 +148,6 @@ for (const name of ["get_xrefs_to_class", "get_xrefs_to_method"]) }), ); await server.connect(new StdioServerTransport()); +// A refused cleanup must leave a live process for verified fixture teardown. +// Otherwise stdin EOF races the ownership scan with the server's natural exit. +if (mode === "cleanup-failure") setInterval(() => {}, 1000); diff --git a/tests/fixtures/sqlite/database.ts b/tests/fixtures/sqlite/database.ts new file mode 100644 index 00000000000..ae174c44d7e --- /dev/null +++ b/tests/fixtures/sqlite/database.ts @@ -0,0 +1,74 @@ +import { DatabaseSync } from "node:sqlite"; +import { join } from "node:path"; + +/** Engine-dependent cases require the same native limit capability as the worker. */ +export const SQLITE_NATIVE_LIMITS_AVAILABLE = (() => { + const database = new DatabaseSync(":memory:"); + try { + return "limits" in database; + } finally { + database.close(); + } +})(); + +/** Exact selected SQLite fixture with committed rows and optional live WAL. */ +export interface SqliteDatabaseFixture { + readonly path: string; + readonly walPath: string; + readonly quotedTable: string; + close(): void; +} + +/** Produce schema and lossless SQLite values through the actual SQLite engine. */ +export const createSqliteDatabaseFixture = ( + root: string, + wal = false, +): SqliteDatabaseFixture => { + const path = join(root, "selected.db"); + const db = new DatabaseSync(path); + if (wal) db.exec("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0"); + db.exec(` + CREATE TABLE records ( + id INTEGER PRIMARY KEY, + integer_value INTEGER, + blob_value BLOB, + real_value REAL, + nullable_value TEXT, + text_value TEXT, + generated_value TEXT GENERATED ALWAYS AS (typeof(integer_value)) VIRTUAL + ); + CREATE INDEX records_text ON records(text_value); + CREATE VIEW record_names AS SELECT text_value FROM records; + CREATE TRIGGER records_insert AFTER INSERT ON records BEGIN SELECT 1; END; + CREATE TABLE "strange""'; DROP TABLE records;--" ("__proto__" TEXT); + INSERT INTO "strange""'; DROP TABLE records;--" VALUES ('exact selected table'); + `); + const insert = db.prepare( + "INSERT INTO records(id, integer_value, blob_value, real_value, nullable_value, text_value) VALUES (?, ?, ?, ?, ?, ?)", + ); + insert.run( + 1, + 9223372036854775807n, + Buffer.from([0, 31, 239, 255]), + Infinity, + null, + "first é\0value", + ); + insert.run( + 2, + -9223372036854775808n, + Buffer.alloc(0), + -Infinity, + "present", + "second", + ); + db.exec( + "INSERT INTO records(id, integer_value, blob_value, real_value, nullable_value, text_value) VALUES (3, 0, X'FF', 1.25, NULL, CAST(X'FF00C0AF' AS TEXT))", + ); + return { + path, + walPath: `${path}-wal`, + quotedTable: "strange\"'; DROP TABLE records;--", + close: () => db.close(), + }; +};