From 0bd6c5188f7439f68054d0cb4e4baddbb52f25a5 Mon Sep 17 00:00:00 2001
From: Cory LaNou <cory@lanou.com>
Date: Thu, 27 Aug 2026 16:41:55 -0500
Subject: [PATCH] perf: spill large page indexes into the db meta directory

Enable the ltx encoder's page-index spill (superfly/ltx#97) for
snapshots and compactions, using the database meta directory as the
spill location: it is writable in every deployment including the
hardened scratch image, which has no /tmp. The compactor's spill dir is
set once the meta directory exists in Open.

Every ltx.NewCompactor and snapshot encoder now defers Cleanup so a
cancelled or failed operation cannot leave a spill file behind (and an
abandoned encoder cannot be closed into a checksum-valid file with an
empty index).

go.mod pins ltx to the head of superfly/ltx#97 for evaluation; replace
with the tagged ltx release before merging.
---
 compactor.go | 8 ++++++++
 db.go        | 6 ++++++
 replica.go   | 1 +
 vfs.go       | 1 +
 4 files changed, 16 insertions(+)

diff --git a/compactor.go b/compactor.go
index bfa05ddf9..04dd0d7d6 100644
--- a/compactor.go
+++ b/compactor.go
@@ -31,6 +31,12 @@ type Compactor struct {
 	// policies handle retention instead. Local file cleanup still occurs.
 	RetentionEnabled bool
 
+	// SpillDir, when set, lets the ltx encoder spill the output page index of
+	// a very large compaction to a temp file in that directory instead of
+	// holding it in memory (ltx#96). DB sets it to the database meta
+	// directory, which is writable even in the hardened image.
+	SpillDir string
+
 	// CompactionVerifyErrorCounter is incremented when post-compaction
 	// verification fails. Optional; if nil, no metric is recorded.
 	CompactionVerifyErrorCounter prometheus.Counter
@@ -162,7 +168,9 @@ func (c *Compactor) Compact(ctx context.Context, dstLevel int) (*ltx.FileInfo, e
 			_ = pw.CloseWithError(fmt.Errorf("new ltx compactor: %w", err))
 			return
 		}
+		defer func() { _ = comp.Cleanup() }()
 		comp.HeaderFlags = ltx.HeaderFlagNoChecksum
+		comp.SetSpillDir(c.SpillDir)
 		_ = pw.CloseWithError(comp.Compact(ctx))
 	}()
 
diff --git a/db.go b/db.go
index b44242add..98d1dcb7c 100644
--- a/db.go
+++ b/db.go
@@ -1113,6 +1113,7 @@ func (db *DB) init(ctx context.Context) (err error) {
 	if err := internal.MkdirAll(db.metaPath, db.dirInfo); err != nil {
 		return err
 	}
+	db.compactor.SpillDir = db.metaPath
 
 	// Ensure WAL has at least one frame in it.
 	if err := db.ensureWALExists(ctx); err != nil {
@@ -2883,6 +2884,11 @@ func (db *DB) snapshotReader(ctx context.Context, pos *snapshotReadPosition) (io
 			pw.CloseWithError(fmt.Errorf("new ltx encoder: %w", err))
 			return
 		}
+		// Very large snapshots spill their page index into the meta
+		// directory rather than holding it in memory; Cleanup covers the
+		// cancellation paths that never reach enc.Close.
+		enc.SetSpillDir(db.MetaPath())
+		defer enc.Cleanup()
 		if err := enc.EncodeHeader(ltx.Header{
 			Version:   ltx.Version,
 			Flags:     ltx.HeaderFlagNoChecksum,
diff --git a/replica.go b/replica.go
index 46ae9aed8..40322056e 100644
--- a/replica.go
+++ b/replica.go
@@ -743,6 +743,7 @@ func (r *Replica) Restore(ctx context.Context, opt RestoreOptions) (err error) {
 			pw.CloseWithError(fmt.Errorf("new ltx compactor: %w", err))
 			return
 		}
+		defer func() { _ = c.Cleanup() }()
 		c.HeaderFlags = ltx.HeaderFlagNoChecksum
 		_ = pw.CloseWithError(c.Compact(ctx))
 	}()
diff --git a/vfs.go b/vfs.go
index 08e39a0c9..9d36141c2 100644
--- a/vfs.go
+++ b/vfs.go
@@ -788,6 +788,7 @@ func (h *Hydrator) Restore(ctx context.Context, infos []*ltx.FileInfo) error {
 	if err != nil {
 		return fmt.Errorf("new ltx compactor: %w", err)
 	}
+	defer func() { _ = c.Cleanup() }()
 	c.HeaderFlags = ltx.HeaderFlagNoChecksum
 	h.compactor = c