feat(crypto): add per-page HPKE encryption for LTX v4

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.24'
+          go-version: '1.26'
 
       - id: release
         uses: bruceadams/get-release@v1.3.2

.github/workflows/test.yml

@@ -10,11 +10,28 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.24'
+          go-version: '1.26'
 
       - name: Run unit tests
         run: go test -v ./...
 
       - name: Build binary
         run: go install ./cmd/ltx
 
+  e2e:
+    name: E2E Encryption
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.26'
+
+      # The suite drives the real CLI against real SQLite databases, so it
+      # needs sqlite3 present. python3, dd and docker are already on the runner.
+      - name: Install sqlite3
+        run: sudo apt-get update && sudo apt-get install -y sqlite3
+
+      - name: Run E2E encryption tests
+        run: ./e2e_encryption_test.sh

.gitignore

@@ -1,2 +1,3 @@
 .vscode
-
+dist/
+test_hpke

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.24 AS builder
+FROM golang:1.26 AS builder
 
 WORKDIR /src/ltx
 COPY . .

README.md

@@ -6,21 +6,33 @@ a way that can be encrypted and compacted and is optimized for performance.
 
 ## File Format
 
-This document describes format version 3. LTX files carry no on-disk version
-field, and versions 2 and 3 both use the `LTX1` magic number. Version 2 page
-frames used a four-byte page header with no flags or compressed-size prefix.
-Version 3 uses a six-byte page header and, in the current encoding, a four-byte
-compressed-size prefix. A reader cannot determine the format version from the
-file alone and must know it out of band.
+This document describes format versions 3 and 4.
 
-An LTX file is composed of four sections:
+Versions 2 and 3 share the `LTX1` magic number and carry no on-disk version
+field, so a reader cannot tell them apart from the file alone and must know
+which it has out of band. They differ in page frame layout: version 2 used a
+four-byte page header with no flags and no compressed-size prefix, while
+version 3 uses a six-byte page header and, in the current encoding, a four-byte
+compressed-size prefix.
+
+Version 4 adds per-page encryption and is the one version that *is* self
+identifying, because it changed the magic to `LTX4`. Encryption is the only
+feature version 4 adds, so the encoder writes an unencrypted file as version 3
+with the `LTX1` magic: an unencrypted version 4 header would be byte-identical
+to a version 3 header anyway, and emitting `LTX4` would stop existing readers
+accepting a file that had not otherwise changed. In practice `LTX4` therefore
+means encrypted.
+
+An LTX file is composed of four sections, or five when encrypted:
 
 1. Header
-2. Page block
-3. Page index
-4. Trailer
+2. Recipient block (encrypted files only)
+3. Page block
+4. Page index
+5. Trailer
 
-The header contains metadata about the file, the page block contains page
+The header contains metadata about the file, the recipient block carries the
+content encryption key wrapped to each recipient, the page block contains page
 frames, the page index enables random access to frames, and the trailer contains
 checksums for the file and the database end state. Unless otherwise specified,
 all fixed-width integer fields use big-endian byte order.
@@ -33,7 +45,7 @@ range represented by the file.
 
 | Offset | Size | Field             | Description                                      |
 | ------ | ---- | ----------------- | ------------------------------------------------ |
-| 0      | 4    | Magic             | Always `LTX1`.                                   |
+| 0      | 4    | Magic             | `LTX1` for version 3, `LTX4` for version 4.      |
 | 4      | 4    | Flags             | Header flags.                                    |
 | 8      | 4    | PageSize          | Database page size, in bytes.                    |
 | 12     | 4    | Commit            | Database size after applying the file, in pages. |
@@ -46,18 +58,47 @@ range represented by the file.
 | 64     | 4    | WALSalt1          | First WAL salt; zero for a journal or compaction. |
 | 68     | 4    | WALSalt2          | Second WAL salt; zero for journal or compaction. |
 | 72     | 8    | NodeID            | Creator node ID; zero if unset.                  |
-| 80     | 20   | Reserved          | Written as zero by the current encoder.          |
+| 80     | 2    | RecipientCount    | Encrypted recipients; zero if unencrypted.       |
+| 82     | 2    | KEMID             | HPKE KEM identifier; zero if unencrypted.        |
+| 84     | 2    | KDFID             | HPKE KDF identifier; zero if unencrypted.        |
+| 86     | 2    | AEADID            | HPKE AEAD identifier; zero if unencrypted.       |
+| 88     | 12   | Reserved          | Written as zero by the current encoder.          |
+
+Bytes 80 through 87 are the version 4 encryption parameters. They fall inside
+the region version 3 reserved and wrote as zero, which is why an unencrypted
+version 4 header is byte-identical to a version 3 one.
 
 ##### Header flags
 
-| Flag         | Name                 | Description                         |
-| ------------ | -------------------- | ----------------------------------- |
-| `0x00000002` | HeaderFlagNoChecksum | Disable database checksum tracking. |
+| Flag         | Name                    | Description                          |
+| ------------ | ----------------------- | ------------------------------------ |
+| `0x00000002` | HeaderFlagNoChecksum    | Disable database checksum tracking.  |
+| `0x00000004` | HeaderFlagEncryptedHPKE | Pages are encrypted (version 4 only). |
 
 `HeaderFlagNoChecksum` is bit 1 (`1 << 1`). When set, the pre-apply and
-post-apply database checksums are zero. All other header flag bits are currently
-invalid. The file checksum is still required when database checksum tracking is
-disabled.
+post-apply database checksums are zero. The file checksum is still required when
+database checksum tracking is disabled.
+
+`HeaderFlagEncryptedHPKE` is bit 2 (`1 << 2`). When set, the file must be
+version 4, `RecipientCount` must be non-zero, and a recipient block follows the
+header. All other header flag bits are currently invalid.
+
+
+#### Recipient block
+
+Present only when `HeaderFlagEncryptedHPKE` is set, immediately after the
+header, and repeated immediately before the trailer. It holds
+`RecipientCount` entries of 80 bytes each.
+
+Each entry is an HPKE (RFC 9180) single-shot sealing of the 32-byte content
+encryption key (CEK) to one recipient public key, and is laid out as the
+32-byte encapsulated key, the 32-byte wrapped CEK, and a 16-byte authentication
+tag. A recipient recovers the CEK by trying to open each entry with its private
+key.
+
+The CEK is generated fresh for every file. Two keys are derived from it with
+HKDF-SHA256: a page key using the info string `ltx-page-key`, and an index key
+using `ltx-index-key`.
 
 
 #### Page block
@@ -88,6 +129,26 @@ bits are invalid.
 A six-byte zero page header terminates the page block and has no size prefix or
 page data.
 
+##### Encrypted page frames
+
+When `HeaderFlagEncryptedHPKE` is set, the payload is the LZ4-compressed page
+data sealed with ChaCha20-Poly1305 under the page key, and `CompressedSize`
+counts the sealed payload rather than the compressed one:
+
+| Offset | Size | Field      | Description                                    |
+| ------ | ---- | ---------- | ---------------------------------------------- |
+| 0      | 12   | Nonce      | Random per-page nonce.                         |
+| 12     | M    | Ciphertext | Sealed LZ4-compressed page data.               |
+| 12+M   | 16   | Tag        | Poly1305 authentication tag.                   |
+
+The additional authenticated data is the 32-byte SHA-256 hash of the header,
+the four-byte page number, and the six-byte page header, concatenated in that
+order. Binding those in means a frame cannot be moved to a different page
+number, or into a different file, without detection.
+
+Encrypted files must use the size-prefixed frame layout; a frame without
+`PageHeaderFlagSize` is rejected rather than treated as a legacy frame.
+
 
 #### Page index
 
@@ -103,6 +164,13 @@ A zero page-number varint terminates the entries. An eight-byte big-endian
 unsigned integer follows and contains the total byte size of the varint entries,
 including the zero terminator but excluding the size field itself.
 
+In an encrypted file the index itself is not encrypted, but a 16-byte
+Poly1305 tag follows it, computed over the index bytes as additional data under
+the index key with an empty plaintext. The duplicate recipient block follows the
+tag, and the trailer follows that. Verifying the tag requires the index key, so
+a reader without a decryption key skips it; the file checksum still covers those
+bytes.
+
 
 #### Trailer
 
@@ -135,3 +203,14 @@ structural metadata; it is not a byte-for-byte checksum of the file on disk. In
 particular, different valid LZ4 payload bytes produce the same checksum when
 they decompress to the same page data and do not change the hashed size or index
 values.
+
+Encrypted files differ on step 2: the sealed payload is hashed exactly as
+written, rather than the plaintext it protects, and the recipient blocks are
+hashed as well. That is deliberate. It means whoever holds the file can verify
+its integrity without holding any decryption key, which is what lets `ltx
+verify` check an archived encrypted file. Such a check covers structure and
+bytes only; it says nothing about page contents, and the tool reports the
+narrower guarantee rather than a bare `ok`.
+
+The post-apply database checksum is unaffected by encryption. It is always a
+rolling checksum over plaintext pages, so verifying it does require a key.

cmd/ltx/apply.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -22,6 +23,7 @@ func NewApplyCommand() *ApplyCommand {
 func (c *ApplyCommand) Run(ctx context.Context, args []string) (ret error) {
 	fs := flag.NewFlagSet("ltx-apply", flag.ContinueOnError)
 	dbPath := fs.String("db", "", "database path")
+	keyHex := fs.String("key", "", "hex-encoded private key for decryption")
 	fs.Usage = func() {
 		fmt.Println(`
 The apply command applies one or more LTX files to a database file.
@@ -45,6 +47,15 @@ Arguments:
 		return fmt.Errorf("required: -db PATH")
 	}
 
+	var decryptionKey []byte
+	if *keyHex != "" {
+		var err error
+		decryptionKey, err = hex.DecodeString(*keyHex)
+		if err != nil {
+			return fmt.Errorf("invalid -key: %w", err)
+		}
+	}
+
 	// Open database file. Create if it doesn't exist.
 	dbFile, err := os.OpenFile(*dbPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
@@ -54,7 +65,7 @@ Arguments:
 
 	// Apply LTX files in order.
 	for _, filename := range fs.Args() {
-		if err := c.applyLTXFile(ctx, dbFile, filename); err != nil {
+		if err := c.applyLTXFile(ctx, dbFile, filename, decryptionKey); err != nil {
 			return fmt.Errorf("%s: %s", filename, err)
 		}
 	}
@@ -66,7 +77,7 @@ Arguments:
 	return dbFile.Close()
 }
 
-func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename string) error {
+func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename string, decryptionKey []byte) error {
 	ltxFile, err := os.Open(filename)
 	if err != nil {
 		return err
@@ -75,6 +86,9 @@ func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename
 
 	// Read LTX header and verify initial checksum matches.
 	dec := ltx.NewDecoder(ltxFile)
+	if decryptionKey != nil {
+		dec.SetDecryptionKey(decryptionKey)
+	}
 	if err := dec.DecodeHeader(); err != nil {
 		return fmt.Errorf("decode ltx header: %w", err)
 	}

cmd/ltx/dump.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -23,6 +24,7 @@ func NewDumpCommand() *DumpCommand {
 // Run executes the command.
 func (c *DumpCommand) Run(ctx context.Context, args []string) (ret error) {
 	fs := flag.NewFlagSet("ltx-dump", flag.ContinueOnError)
+	keyHex := fs.String("key", "", "hex-encoded private key for decryption")
 	fs.Usage = func() {
 		fmt.Println(`
 The dump command writes out all data for a single LTX file.
@@ -44,13 +46,25 @@ Arguments:
 		return fmt.Errorf("too many arguments")
 	}
 
+	var decryptionKey []byte
+	if *keyHex != "" {
+		var err error
+		decryptionKey, err = hex.DecodeString(*keyHex)
+		if err != nil {
+			return fmt.Errorf("invalid -key: %w", err)
+		}
+	}
+
 	f, err := os.Open(fs.Arg(0))
 	if err != nil {
 		return err
 	}
 	defer func() { _ = f.Close() }()
 
 	dec := ltx.NewDecoder(f)
+	if decryptionKey != nil {
+		dec.SetDecryptionKey(decryptionKey)
+	}
 
 	// Read & print header information.
 	err = dec.DecodeHeader()
@@ -67,6 +81,12 @@ Arguments:
 	fmt.Printf("WAL offset: %d\n", hdr.WALOffset)
 	fmt.Printf("WAL size:   %d\n", hdr.WALSize)
 	fmt.Printf("WAL salt:   %08x %08x\n", hdr.WALSalt1, hdr.WALSalt2)
+	if hdr.Encrypted() {
+		fmt.Printf("Encrypted: yes (recipients=%d, KEM=0x%04x, KDF=0x%04x, AEAD=0x%04x)\n",
+			hdr.RecipientCount, hdr.KEMID, hdr.KDFID, hdr.AEADID)
+	} else {
+		fmt.Printf("Encrypted: no\n")
+	}
 	fmt.Printf("\n")
 	if err != nil {
 		return err

cmd/ltx/encode_db.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/superfly/ltx"
@@ -32,6 +34,7 @@ func NewEncodeDBCommand() *EncodeDBCommand {
 func (c *EncodeDBCommand) Run(ctx context.Context, args []string) (ret error) {
 	fs := flag.NewFlagSet("ltx-encode-db", flag.ContinueOnError)
 	outPath := fs.String("o", "", "output path")
+	encryptTo := fs.String("encrypt-to", "", "comma-separated hex-encoded public keys for encryption")
 	fs.Usage = func() {
 		fmt.Println(`
 The encode-db command encodes an SQLite database into an LTX file.
@@ -61,6 +64,17 @@ Arguments:
 	}
 	defer func() { _ = db.Close() }()
 
+	var recipientKeys [][]byte
+	if *encryptTo != "" {
+		for _, s := range strings.Split(*encryptTo, ",") {
+			key, err := hex.DecodeString(strings.TrimSpace(s))
+			if err != nil {
+				return fmt.Errorf("invalid -encrypt-to key: %w", err)
+			}
+			recipientKeys = append(recipientKeys, key)
+		}
+	}
+
 	dbInfo, err := db.Stat()
 	if err != nil {
 		return fmt.Errorf("stat DB file: %w", err)
@@ -106,6 +120,11 @@ Arguments:
 	if err != nil {
 		return fmt.Errorf("create ltx encoder: %w", err)
 	}
+	if len(recipientKeys) > 0 {
+		if err := enc.SetEncryption(recipientKeys); err != nil {
+			return fmt.Errorf("set encryption: %w", err)
+		}
+	}
 	if err := enc.EncodeHeader(ltx.Header{
 		Version:   ltx.Version,
 		PageSize:  hdr.pageSize,

cmd/ltx/encode_db_test.go

@@ -36,7 +36,9 @@ func TestEncodeDBCommand(t *testing.T) {
 		dec := ltx.NewDecoder(f)
 		if err := dec.Verify(); err != nil {
 			t.Fatal(err)
-		} else if got, want := dec.Header().Version, ltx.Version; got != want {
+		} else if got, want := dec.Header().Version, ltx.Version3; got != want {
+			// Unencrypted output stays at v3 so existing readers keep working;
+			// encryption is the only thing that requires v4.
 			t.Fatalf("version=%d, want %d", got, want)
 		}