fix(javascript): keep static facts after semantic results use the parent budget

docs/javascript-artifact-reconstruction.md

@@ -876,8 +876,22 @@ and Evidence. If that expansion does not fit, the file node retains
 `static_analysis` and any admitted `semantic_module_analysis` observations,
 including their exact values and source locations. Use the public modules item
 view to read them. The graph marks those unexpanded relationships as unknown.
-If cumulative result capacity is exhausted, analysis stops; inventoried but
-unexamined files remain present with explicit unknown scopes.
+
+The parent charges everything it retains against the heap that was free when
+analysis started, less a transfer reserve. Semantic facts and expanded
+relationships stop at half of that capacity, and a source refused there keeps
+its static facts with the refusal reported. Static facts can continue to three
+quarters of it while the live heap also has room for another copy of what it
+holds, so results charged for earlier sources do not cost later sources their
+static facts when the heap can still hold them. When that share runs short,
+relationships that were reserved but not yet expanded are given up before any
+static facts, and each affected file reports the omission.
+
+If static facts themselves exhaust that share, or the parent's live heap cannot
+hold the next checkpoint, analysis stops; inventoried but unexamined files remain
+present with explicit unknown scopes. The failure reports the limit that applied
+in bytes, what was still available, the live heap in use and what the checkpoint
+needed.
 
 ### Large local JSON inputs
 

src/javascript/analysis/JavaScriptAnalysisWorker.ts

@@ -83,6 +83,12 @@ const OPERATION = "analyze_javascript_application";
 const PROVIDER = "javascript-static-analysis";
 const MIB = 1024 * 1024;
 const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6;
+/**
+ * Share of the parent's free heap that static checkpoints may fill once the
+ * result budget is used. Unlike the live heap, the charges also cover application
+ * relationships that are reserved but not yet materialized.
+ */
+const STATIC_CAPACITY_SHARE = 0.75;
 /** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */
 const WORKER_HEAP_FLAGS_ENFORCED =
   javaScriptRuntime(process.versions).id === "node";
@@ -130,10 +136,11 @@ export class JavaScriptAnalysisWorker {
   #revokedStaticProjections: JavaScriptRevokedStaticProjection[] = [];
   #nextSourceIndex = 0;
   #applicationProjectionFailure: AnalysisResourceConstraintError | undefined;
-  #parentCapacityExhausted = false;
+  #parentCapacityExhaustion: Readonly<Record<string, number>> | undefined;
   #sourcesRemaining: number | undefined;
   #deadline: ProviderStartupDeadline | undefined;
   #resultBudgetBytes = 0;
+  #staticCapacityBytes = 0;
   /** Null when the runtime enforces the V8 heap flags instead. */
   readonly #processMemoryLimitBytes: number | null;
   readonly #onStdout = (chunk: Buffer | string): void => {
@@ -248,7 +255,7 @@ export class JavaScriptAnalysisWorker {
     this.#staticProjectionReservations = [];
     this.#revokedStaticProjections = [];
     this.#nextSourceIndex = 0;
-    this.#parentCapacityExhausted = false;
+    this.#parentCapacityExhaustion = undefined;
     this.#applicationProjectionFailure = undefined;
     this.#refreshResultBudget();
     return true;
@@ -262,17 +269,20 @@ export class JavaScriptAnalysisWorker {
 
   /**
    * Reserve host-side control, schema/Evidence work, and a maximum record's
-   * transient copies. Admission also accounts for retained earlier files.
+   * transient copies. Everything retained is charged against what is left.
+   *
+   * Semantic and application results stop at half of it. Static checkpoints
+   * may go on to a larger share while the live heap has room for them, so a
+   * charge for earlier results cannot cost a later source its static facts.
    */
   #refreshResultBudget(): void {
-    this.#resultBudgetBytes = Math.max(
+    const capacity = Math.max(
       0,
-      Math.floor(
-        (this.#remainingParentHeapBytes() -
-          24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES) /
-          2,
-      ),
+      this.#remainingParentHeapBytes() -
+        24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES,
     );
+    this.#resultBudgetBytes = Math.floor(capacity / 2);
+    this.#staticCapacityBytes = Math.floor(capacity * STATIC_CAPACITY_SHARE);
   }
 
   /** Analyze inert source; hard worker failure does not terminate this process. */
@@ -290,7 +300,7 @@ export class JavaScriptAnalysisWorker {
     const id = randomUUID();
     const sourceIndex = this.#nextSourceIndex++;
     this.#applicationProjectionFailure = undefined;
-    this.#parentCapacityExhausted = false;
+    this.#parentCapacityExhaustion = undefined;
     this.#revokedStaticProjections = [];
     if (this.#sourcesRemaining !== undefined)
       this.#sourcesRemaining = Math.max(0, this.#sourcesRemaining - 1);
@@ -388,20 +398,8 @@ export class JavaScriptAnalysisWorker {
             sourceIndex,
             response,
           );
-          if (this.#parentCapacityExhausted)
-            throw new AnalysisResourceConstraintError(
-              OPERATION,
-              "memory",
-              `The parent's result capacity was exhausted after retaining static facts for ${file.path}; remaining sources were not analyzed`,
-              {
-                ...this.#limits(),
-                parent_result_budget_bytes: this.#resultBudgetBytes,
-                parent_result_budget_exhausted: true,
-              },
-              {
-                remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION,
-              },
-            );
+          const exhausted = this.#parentCapacityFailure(file.path);
+          if (exhausted !== undefined) throw exhausted;
           offeredResultBudget =
             this.#applicationProjectionFailure !== undefined &&
             this.#sourcesRemaining !== undefined &&
@@ -799,11 +797,21 @@ export class JavaScriptAnalysisWorker {
     // Application relationships are materialized only after all source files
     // have been examined. Reclaim their pending reservations to keep room for
     // later static observations, without relaxing the actual retained bound.
-    const needed =
-      retainedBytes - (this.#resultBudgetBytes - this.#estimatedRetainedBytes);
+    const { heap_size_limit, used_heap_size } = getHeapStatistics();
     const liveAvailable =
-      this.#remainingParentHeapBytes() -
+      heap_size_limit -
+      used_heap_size -
       24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES;
+    // Past the result budget, the live heap must also keep room for another
+    // copy of what it holds, which final graph and Evidence work needs. The
+    // limit read here counts young-generation capacity that retained objects
+    // cannot use, so being merely below it does not protect the parent.
+    const staticCeiling =
+      retainedBytes + used_heap_size <= liveAvailable
+        ? this.#staticCapacityBytes
+        : this.#resultBudgetBytes;
+    const needed =
+      retainedBytes - (staticCeiling - this.#estimatedRetainedBytes);
     if (needed > 0 && retainedBytes <= liveAvailable) {
       // Plan all revocations first so an insufficient set leaves the earlier
       // relationships intact rather than sacrificing them for a failed read.
@@ -836,12 +844,17 @@ export class JavaScriptAnalysisWorker {
     // admission so a second read cannot reject after revocations commit.
     const remaining = Math.max(
       0,
-      Math.min(
-        this.#resultBudgetBytes - this.#estimatedRetainedBytes,
-        liveAvailable,
-      ),
+      Math.min(staticCeiling - this.#estimatedRetainedBytes, liveAvailable),
     );
-    this.#parentCapacityExhausted = retainedBytes > remaining;
+    this.#parentCapacityExhaustion =
+      retainedBytes > remaining
+        ? {
+            parent_static_capacity_bytes: staticCeiling,
+            parent_available_static_capacity_bytes: remaining,
+            parent_used_heap_bytes: used_heap_size,
+            static_retention_estimate_bytes: retainedBytes,
+          }
+        : undefined;
     this.#estimatedRetainedBytes += retainedBytes;
     const expandedBytes = estimateJavaScriptStaticApplicationProjection(
       facts,
@@ -873,6 +886,26 @@ export class JavaScriptAnalysisWorker {
     return facts;
   }
 
+  #parentCapacityFailure(
+    path: string,
+  ): AnalysisResourceConstraintError | undefined {
+    if (this.#parentCapacityExhaustion === undefined) return undefined;
+    return new AnalysisResourceConstraintError(
+      OPERATION,
+      "memory",
+      `The parent's result capacity was exhausted after retaining static facts for ${path}; remaining sources were not analyzed`,
+      {
+        ...this.#limits(),
+        parent_result_budget_bytes: this.#resultBudgetBytes,
+        ...this.#parentCapacityExhaustion,
+        parent_result_budget_exhausted: true,
+      },
+      {
+        remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION,
+      },
+    );
+  }
+
   #limits() {
     return {
       ...(WORKER_HEAP_FLAGS_ENFORCED

tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts

@@ -170,3 +170,74 @@ cliTest.skipIf(process.versions.bun !== undefined).for([
     expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu);
   },
 );
+
+cliTest.skipIf(process.versions.bun !== undefined)(
+  "keeps every source's static facts after semantic results use the parent result budget",
+  { timeout: 90_000 },
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-js-static-capacity-");
+    // Plain functions produce semantic graphs large enough to use the result
+    // budget of this heap. The later sources' storage findings then produce
+    // static checkpoints that exceed its reserve for the next checkpoint.
+    const vendors = Array.from({ length: 12 }, (_, vendor) => [
+      `a-vendor-${String(vendor)}.mjs`,
+      Array.from(
+        { length: 100 },
+        (_, index) =>
+          `export function v${String(vendor)}_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`,
+      ).join("\n"),
+    ]);
+    const callsPerSource = 12_000;
+    const sources = Array.from({ length: 5 }, (_, source) => [
+      `z-source-${String(source)}.mjs`,
+      Array.from(
+        { length: callsPerSource },
+        (_, index) =>
+          `localStorage.getItem("k${String(source)}_${String(index)}");`,
+      ).join("\n"),
+    ]);
+    for (const [name, text] of [...vendors, ...sources])
+      await writeFile(join(root, name!), text!);
+    const response = await cli.run({
+      arguments: ["analyze", root, "--format", "json"],
+      environment: {
+        NODE_OPTIONS: "--max-old-space-size=640 --max-semi-space-size=8",
+      },
+      timeoutMs: 75_000,
+    });
+    expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu);
+    expect(
+      response.exitCode,
+      JSON.stringify(response.json)?.slice(0, 2000),
+    ).toBe(0);
+    const result = javascriptApplicationAnalysisResultSchema.parse(
+      parseEvidence(response.json).normalized_result,
+    );
+    expect(result.statistics).toMatchObject({
+      parsed_javascript_files: vendors.length + sources.length,
+      skipped_javascript_files: 0,
+      parse_failures: 0,
+    });
+    // The omission is reported where a reader finds the retained facts.
+    for (const [name, text] of sources) {
+      const digest = createHash("sha256").update(text!).digest("hex");
+      const node = result.graph.nodes.find(
+        ({ kind, identity }) =>
+          kind === "javascript-asset" &&
+          identity.strategy === "content-digest" &&
+          identity.sha256 === digest,
+      );
+      const observation = node?.observations.find(
+        ({ properties }) =>
+          properties.application_projection_failure !== undefined,
+      );
+      expect(observation?.label, name).toBe(name);
+      expect(
+        z
+          .object({ storage: z.array(z.unknown()) })
+          .parse(observation?.properties.static_analysis).storage,
+      ).toHaveLength(callsPerSource);
+    }
+    expect(result.graph.coverage.status).toBe("partial");
+  },
+);