diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a099c91..22b47da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.26' - id: release uses: bruceadams/get-release@v1.3.2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c28d4ee..637dea5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/setup-go@v2 with: - go-version: '1.24' + go-version: '1.26' - name: Run unit tests run: go test -v ./... @@ -18,3 +18,20 @@ jobs: - name: Build binary run: go install ./cmd/ltx + e2e: + name: E2E Encryption + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: '1.26' + + # The suite drives the real CLI against real SQLite databases, so it + # needs sqlite3 present. python3, dd and docker are already on the runner. + - name: Install sqlite3 + run: sudo apt-get update && sudo apt-get install -y sqlite3 + + - name: Run E2E encryption tests + run: ./e2e_encryption_test.sh diff --git a/.gitignore b/.gitignore index faa1838..4c544de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vscode - +dist/ +test_hpke diff --git a/Dockerfile b/Dockerfile index c2a2578..bdeef2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24 AS builder +FROM golang:1.26 AS builder WORKDIR /src/ltx COPY . . diff --git a/README.md b/README.md index cde54c4..ca39605 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,33 @@ a way that can be encrypted and compacted and is optimized for performance. ## File Format -This document describes format version 3. LTX files carry no on-disk version -field, and versions 2 and 3 both use the `LTX1` magic number. Version 2 page -frames used a four-byte page header with no flags or compressed-size prefix. -Version 3 uses a six-byte page header and, in the current encoding, a four-byte -compressed-size prefix. A reader cannot determine the format version from the -file alone and must know it out of band. +This document describes format versions 3 and 4. -An LTX file is composed of four sections: +Versions 2 and 3 share the `LTX1` magic number and carry no on-disk version +field, so a reader cannot tell them apart from the file alone and must know +which it has out of band. They differ in page frame layout: version 2 used a +four-byte page header with no flags and no compressed-size prefix, while +version 3 uses a six-byte page header and, in the current encoding, a four-byte +compressed-size prefix. + +Version 4 adds per-page encryption and is the one version that *is* self +identifying, because it changed the magic to `LTX4`. Encryption is the only +feature version 4 adds, so the encoder writes an unencrypted file as version 3 +with the `LTX1` magic: an unencrypted version 4 header would be byte-identical +to a version 3 header anyway, and emitting `LTX4` would stop existing readers +accepting a file that had not otherwise changed. In practice `LTX4` therefore +means encrypted. + +An LTX file is composed of four sections, or five when encrypted: 1. Header -2. Page block -3. Page index -4. Trailer +2. Recipient block (encrypted files only) +3. Page block +4. Page index +5. Trailer -The header contains metadata about the file, the page block contains page +The header contains metadata about the file, the recipient block carries the +content encryption key wrapped to each recipient, the page block contains page frames, the page index enables random access to frames, and the trailer contains checksums for the file and the database end state. Unless otherwise specified, all fixed-width integer fields use big-endian byte order. @@ -33,7 +45,7 @@ range represented by the file. | Offset | Size | Field | Description | | ------ | ---- | ----------------- | ------------------------------------------------ | -| 0 | 4 | Magic | Always `LTX1`. | +| 0 | 4 | Magic | `LTX1` for version 3, `LTX4` for version 4. | | 4 | 4 | Flags | Header flags. | | 8 | 4 | PageSize | Database page size, in bytes. | | 12 | 4 | Commit | Database size after applying the file, in pages. | @@ -46,18 +58,47 @@ range represented by the file. | 64 | 4 | WALSalt1 | First WAL salt; zero for a journal or compaction. | | 68 | 4 | WALSalt2 | Second WAL salt; zero for journal or compaction. | | 72 | 8 | NodeID | Creator node ID; zero if unset. | -| 80 | 20 | Reserved | Written as zero by the current encoder. | +| 80 | 2 | RecipientCount | Encrypted recipients; zero if unencrypted. | +| 82 | 2 | KEMID | HPKE KEM identifier; zero if unencrypted. | +| 84 | 2 | KDFID | HPKE KDF identifier; zero if unencrypted. | +| 86 | 2 | AEADID | HPKE AEAD identifier; zero if unencrypted. | +| 88 | 12 | Reserved | Written as zero by the current encoder. | + +Bytes 80 through 87 are the version 4 encryption parameters. They fall inside +the region version 3 reserved and wrote as zero, which is why an unencrypted +version 4 header is byte-identical to a version 3 one. ##### Header flags -| Flag | Name | Description | -| ------------ | -------------------- | ----------------------------------- | -| `0x00000002` | HeaderFlagNoChecksum | Disable database checksum tracking. | +| Flag | Name | Description | +| ------------ | ----------------------- | ------------------------------------ | +| `0x00000002` | HeaderFlagNoChecksum | Disable database checksum tracking. | +| `0x00000004` | HeaderFlagEncryptedHPKE | Pages are encrypted (version 4 only). | `HeaderFlagNoChecksum` is bit 1 (`1 << 1`). When set, the pre-apply and -post-apply database checksums are zero. All other header flag bits are currently -invalid. The file checksum is still required when database checksum tracking is -disabled. +post-apply database checksums are zero. The file checksum is still required when +database checksum tracking is disabled. + +`HeaderFlagEncryptedHPKE` is bit 2 (`1 << 2`). When set, the file must be +version 4, `RecipientCount` must be non-zero, and a recipient block follows the +header. All other header flag bits are currently invalid. + + +#### Recipient block + +Present only when `HeaderFlagEncryptedHPKE` is set, immediately after the +header, and repeated immediately before the trailer. It holds +`RecipientCount` entries of 80 bytes each. + +Each entry is an HPKE (RFC 9180) single-shot sealing of the 32-byte content +encryption key (CEK) to one recipient public key, and is laid out as the +32-byte encapsulated key, the 32-byte wrapped CEK, and a 16-byte authentication +tag. A recipient recovers the CEK by trying to open each entry with its private +key. + +The CEK is generated fresh for every file. Two keys are derived from it with +HKDF-SHA256: a page key using the info string `ltx-page-key`, and an index key +using `ltx-index-key`. #### Page block @@ -88,6 +129,26 @@ bits are invalid. A six-byte zero page header terminates the page block and has no size prefix or page data. +##### Encrypted page frames + +When `HeaderFlagEncryptedHPKE` is set, the payload is the LZ4-compressed page +data sealed with ChaCha20-Poly1305 under the page key, and `CompressedSize` +counts the sealed payload rather than the compressed one: + +| Offset | Size | Field | Description | +| ------ | ---- | ---------- | ---------------------------------------------- | +| 0 | 12 | Nonce | Random per-page nonce. | +| 12 | M | Ciphertext | Sealed LZ4-compressed page data. | +| 12+M | 16 | Tag | Poly1305 authentication tag. | + +The additional authenticated data is the 32-byte SHA-256 hash of the header, +the four-byte page number, and the six-byte page header, concatenated in that +order. Binding those in means a frame cannot be moved to a different page +number, or into a different file, without detection. + +Encrypted files must use the size-prefixed frame layout; a frame without +`PageHeaderFlagSize` is rejected rather than treated as a legacy frame. + #### Page index @@ -103,6 +164,13 @@ A zero page-number varint terminates the entries. An eight-byte big-endian unsigned integer follows and contains the total byte size of the varint entries, including the zero terminator but excluding the size field itself. +In an encrypted file the index itself is not encrypted, but a 16-byte +Poly1305 tag follows it, computed over the index bytes as additional data under +the index key with an empty plaintext. The duplicate recipient block follows the +tag, and the trailer follows that. Verifying the tag requires the index key, so +a reader without a decryption key skips it; the file checksum still covers those +bytes. + #### Trailer @@ -135,3 +203,14 @@ structural metadata; it is not a byte-for-byte checksum of the file on disk. In particular, different valid LZ4 payload bytes produce the same checksum when they decompress to the same page data and do not change the hashed size or index values. + +Encrypted files differ on step 2: the sealed payload is hashed exactly as +written, rather than the plaintext it protects, and the recipient blocks are +hashed as well. That is deliberate. It means whoever holds the file can verify +its integrity without holding any decryption key, which is what lets `ltx +verify` check an archived encrypted file. Such a check covers structure and +bytes only; it says nothing about page contents, and the tool reports the +narrower guarantee rather than a bare `ok`. + +The post-apply database checksum is unaffected by encryption. It is always a +rolling checksum over plaintext pages, so verifying it does require a key. diff --git a/cmd/ltx/apply.go b/cmd/ltx/apply.go index 6b84f06..aa5cdc3 100644 --- a/cmd/ltx/apply.go +++ b/cmd/ltx/apply.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/hex" "flag" "fmt" "io" @@ -22,6 +23,7 @@ func NewApplyCommand() *ApplyCommand { func (c *ApplyCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("ltx-apply", flag.ContinueOnError) dbPath := fs.String("db", "", "database path") + keyHex := fs.String("key", "", "hex-encoded private key for decryption") fs.Usage = func() { fmt.Println(` The apply command applies one or more LTX files to a database file. @@ -45,6 +47,15 @@ Arguments: return fmt.Errorf("required: -db PATH") } + var decryptionKey []byte + if *keyHex != "" { + var err error + decryptionKey, err = hex.DecodeString(*keyHex) + if err != nil { + return fmt.Errorf("invalid -key: %w", err) + } + } + // Open database file. Create if it doesn't exist. dbFile, err := os.OpenFile(*dbPath, os.O_RDWR|os.O_CREATE, 0o666) if err != nil { @@ -54,7 +65,7 @@ Arguments: // Apply LTX files in order. for _, filename := range fs.Args() { - if err := c.applyLTXFile(ctx, dbFile, filename); err != nil { + if err := c.applyLTXFile(ctx, dbFile, filename, decryptionKey); err != nil { return fmt.Errorf("%s: %s", filename, err) } } @@ -66,7 +77,7 @@ Arguments: return dbFile.Close() } -func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename string) error { +func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename string, decryptionKey []byte) error { ltxFile, err := os.Open(filename) if err != nil { return err @@ -75,6 +86,9 @@ func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename // Read LTX header and verify initial checksum matches. dec := ltx.NewDecoder(ltxFile) + if decryptionKey != nil { + dec.SetDecryptionKey(decryptionKey) + } if err := dec.DecodeHeader(); err != nil { return fmt.Errorf("decode ltx header: %w", err) } diff --git a/cmd/ltx/dump.go b/cmd/ltx/dump.go index eaa21c3..98c4e17 100644 --- a/cmd/ltx/dump.go +++ b/cmd/ltx/dump.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/hex" "flag" "fmt" "io" @@ -23,6 +24,7 @@ func NewDumpCommand() *DumpCommand { // Run executes the command. func (c *DumpCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("ltx-dump", flag.ContinueOnError) + keyHex := fs.String("key", "", "hex-encoded private key for decryption") fs.Usage = func() { fmt.Println(` The dump command writes out all data for a single LTX file. @@ -44,6 +46,15 @@ Arguments: return fmt.Errorf("too many arguments") } + var decryptionKey []byte + if *keyHex != "" { + var err error + decryptionKey, err = hex.DecodeString(*keyHex) + if err != nil { + return fmt.Errorf("invalid -key: %w", err) + } + } + f, err := os.Open(fs.Arg(0)) if err != nil { return err @@ -51,6 +62,9 @@ Arguments: defer func() { _ = f.Close() }() dec := ltx.NewDecoder(f) + if decryptionKey != nil { + dec.SetDecryptionKey(decryptionKey) + } // Read & print header information. err = dec.DecodeHeader() @@ -67,6 +81,12 @@ Arguments: fmt.Printf("WAL offset: %d\n", hdr.WALOffset) fmt.Printf("WAL size: %d\n", hdr.WALSize) fmt.Printf("WAL salt: %08x %08x\n", hdr.WALSalt1, hdr.WALSalt2) + if hdr.Encrypted() { + fmt.Printf("Encrypted: yes (recipients=%d, KEM=0x%04x, KDF=0x%04x, AEAD=0x%04x)\n", + hdr.RecipientCount, hdr.KEMID, hdr.KDFID, hdr.AEADID) + } else { + fmt.Printf("Encrypted: no\n") + } fmt.Printf("\n") if err != nil { return err diff --git a/cmd/ltx/encode_db.go b/cmd/ltx/encode_db.go index 6dc26e6..a326acc 100644 --- a/cmd/ltx/encode_db.go +++ b/cmd/ltx/encode_db.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/binary" + "encoding/hex" "errors" "flag" "fmt" "io" "os" "path/filepath" + "strings" "time" "github.com/superfly/ltx" @@ -32,6 +34,7 @@ func NewEncodeDBCommand() *EncodeDBCommand { func (c *EncodeDBCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("ltx-encode-db", flag.ContinueOnError) outPath := fs.String("o", "", "output path") + encryptTo := fs.String("encrypt-to", "", "comma-separated hex-encoded public keys for encryption") fs.Usage = func() { fmt.Println(` The encode-db command encodes an SQLite database into an LTX file. @@ -61,6 +64,17 @@ Arguments: } defer func() { _ = db.Close() }() + var recipientKeys [][]byte + if *encryptTo != "" { + for _, s := range strings.Split(*encryptTo, ",") { + key, err := hex.DecodeString(strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("invalid -encrypt-to key: %w", err) + } + recipientKeys = append(recipientKeys, key) + } + } + dbInfo, err := db.Stat() if err != nil { return fmt.Errorf("stat DB file: %w", err) @@ -106,6 +120,11 @@ Arguments: if err != nil { return fmt.Errorf("create ltx encoder: %w", err) } + if len(recipientKeys) > 0 { + if err := enc.SetEncryption(recipientKeys); err != nil { + return fmt.Errorf("set encryption: %w", err) + } + } if err := enc.EncodeHeader(ltx.Header{ Version: ltx.Version, PageSize: hdr.pageSize, diff --git a/cmd/ltx/encode_db_test.go b/cmd/ltx/encode_db_test.go index b2bd798..27d66ca 100644 --- a/cmd/ltx/encode_db_test.go +++ b/cmd/ltx/encode_db_test.go @@ -36,7 +36,9 @@ func TestEncodeDBCommand(t *testing.T) { dec := ltx.NewDecoder(f) if err := dec.Verify(); err != nil { t.Fatal(err) - } else if got, want := dec.Header().Version, ltx.Version; got != want { + } else if got, want := dec.Header().Version, ltx.Version3; got != want { + // Unencrypted output stays at v3 so existing readers keep working; + // encryption is the only thing that requires v4. t.Fatalf("version=%d, want %d", got, want) } diff --git a/cmd/ltx/keygen.go b/cmd/ltx/keygen.go new file mode 100644 index 0000000..5d3a9fc --- /dev/null +++ b/cmd/ltx/keygen.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "encoding/hex" + "flag" + "fmt" + + "github.com/superfly/ltx" +) + +type KeygenCommand struct{} + +func NewKeygenCommand() *KeygenCommand { + return &KeygenCommand{} +} + +func (c *KeygenCommand) Run(_ context.Context, args []string) error { + fs := flag.NewFlagSet("ltx-keygen", flag.ContinueOnError) + fs.Usage = func() { + fmt.Println(` +The keygen command generates an X25519 keypair for HPKE encryption. + +Usage: + + ltx keygen + +Output is two lines of hex-encoded keys: public key, then private key. +`[1:]) + } + if err := fs.Parse(args); err != nil { + return err + } + + pub, priv, err := ltx.GenerateKeyPair() + if err != nil { + return fmt.Errorf("generate keypair: %w", err) + } + + fmt.Printf("public: %s\n", hex.EncodeToString(pub)) + fmt.Printf("private: %s\n", hex.EncodeToString(priv)) + + return nil +} diff --git a/cmd/ltx/list.go b/cmd/ltx/list.go index fbdd2a4..f6da58a 100644 --- a/cmd/ltx/list.go +++ b/cmd/ltx/list.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/hex" "flag" "fmt" "io" @@ -25,6 +26,7 @@ func NewListCommand() *ListCommand { func (c *ListCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("ltx-list", flag.ContinueOnError) tsv := fs.Bool("tsv", false, "output as tab-separated values") + keyHex := fs.String("key", "", "hex-encoded private key for decryption") fs.Usage = func() { fmt.Println(` The list command lists header & trailer information for a set of LTX files. @@ -44,6 +46,15 @@ Arguments: return fmt.Errorf("at least one LTX file is required") } + var decryptionKey []byte + if *keyHex != "" { + var err error + decryptionKey, err = hex.DecodeString(*keyHex) + if err != nil { + return fmt.Errorf("invalid -key: %w", err) + } + } + var w io.Writer = os.Stdout if !*tsv { tw := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) @@ -51,9 +62,9 @@ Arguments: w = tw } - _, _ = fmt.Fprintln(w, "min_txid\tmax_txid\tcommit\tpages\tpreapply\tpostapply\ttimestamp\twal_offset\twal_size\twal_salt") + _, _ = fmt.Fprintln(w, "min_txid\tmax_txid\tcommit\tpages\tpreapply\tpostapply\ttimestamp\twal_offset\twal_size\twal_salt\tencrypted") for _, arg := range fs.Args() { - if err := c.printFile(w, arg); err != nil { + if err := c.printFile(w, arg, decryptionKey); err != nil { _, _ = fmt.Fprintf(os.Stderr, "%s: %s\n", arg, err) } } @@ -61,7 +72,7 @@ Arguments: return nil } -func (c *ListCommand) printFile(w io.Writer, filename string) error { +func (c *ListCommand) printFile(w io.Writer, filename string, decryptionKey []byte) error { f, err := os.Open(filename) if err != nil { return err @@ -69,6 +80,9 @@ func (c *ListCommand) printFile(w io.Writer, filename string) error { defer func() { _ = f.Close() }() dec := ltx.NewDecoder(f) + if decryptionKey != nil { + dec.SetDecryptionKey(decryptionKey) + } if err := dec.Verify(); err != nil { return err } @@ -79,7 +93,12 @@ func (c *ListCommand) printFile(w io.Writer, filename string) error { timestamp = "" } - _, _ = fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\t%s\t%s\t%d\t%d\t%08x %08x\n", + encrypted := "no" + if dec.Header().Encrypted() { + encrypted = "yes" + } + + _, _ = fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\t%s\t%s\t%d\t%d\t%08x %08x\t%s\n", dec.Header().MinTXID.String(), dec.Header().MaxTXID.String(), dec.Header().Commit, @@ -90,6 +109,7 @@ func (c *ListCommand) printFile(w io.Writer, filename string) error { dec.Header().WALOffset, dec.Header().WALSize, dec.Header().WALSalt1, dec.Header().WALSalt2, + encrypted, ) return nil diff --git a/cmd/ltx/main.go b/cmd/ltx/main.go index 3f35b85..f759299 100644 --- a/cmd/ltx/main.go +++ b/cmd/ltx/main.go @@ -49,8 +49,12 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) { return NewDumpCommand().Run(ctx, args) case "encode-db": return NewEncodeDBCommand().Run(ctx, args) + case "keygen": + return NewKeygenCommand().Run(ctx, args) case "list": return NewListCommand().Run(ctx, args) + case "rekey": + return NewRekeyCommand().Run(ctx, args) case "verify": return NewVerifyCommand().Run(ctx, args) case "version": @@ -85,7 +89,10 @@ The commands are: apply applies a set of LTX files to a database checksum computes the LTX checksum of a database file dump writes out metadata and page headers for a set of LTX files + encode-db encodes an SQLite database into an LTX file + keygen generates an X25519 keypair for HPKE encryption list lists header & trailer fields for LTX files in a table + rekey re-encrypts an LTX file with new recipients verify reads & verifies checksums of a set of LTX files version prints the version `[1:]) diff --git a/cmd/ltx/rekey.go b/cmd/ltx/rekey.go new file mode 100644 index 0000000..c108834 --- /dev/null +++ b/cmd/ltx/rekey.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "encoding/hex" + "flag" + "fmt" + "os" + "strings" + + "github.com/superfly/ltx" +) + +type RekeyCommand struct{} + +func NewRekeyCommand() *RekeyCommand { + return &RekeyCommand{} +} + +func (c *RekeyCommand) Run(_ context.Context, args []string) error { + fs := flag.NewFlagSet("ltx-rekey", flag.ContinueOnError) + keyHex := fs.String("key", "", "hex-encoded private key for decryption") + encryptTo := fs.String("encrypt-to", "", "comma-separated hex-encoded public keys for encryption") + outPath := fs.String("o", "", "output path") + fs.Usage = func() { + fmt.Println(` +The rekey command reads an LTX file with one key and re-encrypts with new recipients. + +Usage: + + ltx rekey [arguments] PATH + +Arguments: +`[1:]) + fs.PrintDefaults() + fmt.Println() + } + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() == 0 { + fs.Usage() + return flag.ErrHelp + } + if *outPath == "" { + return fmt.Errorf("required: -o PATH") + } + + var decryptionKey []byte + if *keyHex != "" { + var err error + decryptionKey, err = hex.DecodeString(*keyHex) + if err != nil { + return fmt.Errorf("invalid -key: %w", err) + } + } + + var recipientKeys [][]byte + if *encryptTo != "" { + for _, s := range strings.Split(*encryptTo, ",") { + key, err := hex.DecodeString(strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("invalid -encrypt-to key: %w", err) + } + recipientKeys = append(recipientKeys, key) + } + } + + // Read input. + f, err := os.Open(fs.Arg(0)) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + var spec ltx.FileSpec + if _, err := spec.ReadFromWithKey(f, decryptionKey); err != nil { + return fmt.Errorf("read input: %w", err) + } + + // Write output with new encryption. + spec.RecipientPublicKeys = recipientKeys + out, err := os.Create(*outPath) + if err != nil { + return fmt.Errorf("create output: %w", err) + } + defer func() { _ = out.Close() }() + + if _, err := spec.WriteTo(out); err != nil { + return fmt.Errorf("write output: %w", err) + } + + if err := out.Sync(); err != nil { + return err + } + + return nil +} diff --git a/cmd/ltx/verify.go b/cmd/ltx/verify.go index aaa12a8..ba6182d 100644 --- a/cmd/ltx/verify.go +++ b/cmd/ltx/verify.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/hex" "flag" "fmt" "os" @@ -20,16 +21,19 @@ func NewVerifyCommand() *VerifyCommand { // Run executes the command. func (c *VerifyCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("ltx-verify", flag.ContinueOnError) + keyHex := fs.String("key", "", "hex-encoded private key for decryption") fs.Usage = func() { fmt.Println(` The verify command reads one or more LTX files and verifies its integrity. Usage: - ltx verify PATH [PATH...] + ltx verify [arguments] PATH [PATH...] -`[1:], - ) +Arguments: +`[1:]) + fs.PrintDefaults() + fmt.Println() } if err := fs.Parse(args); err != nil { return err @@ -37,9 +41,18 @@ Usage: return fmt.Errorf("at least one LTX file must be specified") } + var decryptionKey []byte + if *keyHex != "" { + var err error + decryptionKey, err = hex.DecodeString(*keyHex) + if err != nil { + return fmt.Errorf("invalid -key: %w", err) + } + } + var okN, errorN int for _, filename := range fs.Args() { - if err := c.verifyFile(ctx, filename); err != nil { + if err := c.verifyFile(ctx, filename, decryptionKey); err != nil { errorN++ fmt.Printf("%s: %s\n", filename, err) continue @@ -56,12 +69,26 @@ Usage: return nil } -func (c *VerifyCommand) verifyFile(_ context.Context, filename string) error { +func (c *VerifyCommand) verifyFile(_ context.Context, filename string, decryptionKey []byte) error { f, err := os.Open(filename) if err != nil { return err } defer func() { _ = f.Close() }() - return ltx.NewDecoder(f).Verify() + dec := ltx.NewDecoder(f) + if decryptionKey != nil { + dec.SetDecryptionKey(decryptionKey) + } + if err := dec.Verify(); err != nil { + return err + } + + // An encrypted file verified without a key has had its integrity checked + // against the file checksum, which covers the ciphertext. Say so, rather + // than letting a bare "ok" imply the contents were checked too. + if dec.Header().Encrypted() && decryptionKey == nil { + fmt.Printf("%s: integrity ok (encrypted; contents not verified, no key supplied)\n", filename) + } + return nil } diff --git a/compactor.go b/compactor.go index 2bdbda7..4e1fb39 100644 --- a/compactor.go +++ b/compactor.go @@ -42,6 +42,13 @@ type Compactor struct { // transaction IDs. This is false by default but can be enabled when // rebuilding snapshots with missing transactions. AllowNonContiguousTXIDs bool + + // DecryptionKey is the private key used to decrypt input files. + DecryptionKey []byte + + // OutputEncryption is the set of recipient public keys for the output file. + // If set, the output will be encrypted. If nil, the output will be unencrypted. + OutputEncryption [][]byte } // NewCompactor returns a new instance of Compactor with default settings. @@ -80,6 +87,20 @@ func (c *Compactor) Compact(ctx context.Context) error { return fmt.Errorf("at least one input reader required") } + // Set decryption key on all input decoders. + if c.DecryptionKey != nil { + for _, input := range c.inputs { + input.dec.SetDecryptionKey(c.DecryptionKey) + } + } + + // Set encryption on output encoder. + if len(c.OutputEncryption) > 0 { + if err := c.enc.SetEncryption(c.OutputEncryption); err != nil { + return fmt.Errorf("set output encryption: %w", err) + } + } + // Read headers from all inputs. for i, input := range c.inputs { if err := input.dec.DecodeHeader(); err != nil { diff --git a/compactor_test.go b/compactor_test.go index 95e1174..cf7789c 100644 --- a/compactor_test.go +++ b/compactor_test.go @@ -14,7 +14,7 @@ func TestCompactor_Compact(t *testing.T) { t.Run("SingleFilePageDataOnly", func(t *testing.T) { input := <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, PageSize: 512, Commit: 1, MinTXID: 1, @@ -103,7 +103,7 @@ func TestCompactor_Compact(t *testing.T) { assertFileSpecEqual(t, spec, <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, PageSize: 1024, Commit: 3, MinTXID: 1, @@ -183,7 +183,7 @@ func TestCompactor_Compact(t *testing.T) { assertFileSpecEqual(t, spec, <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, PageSize: 1024, Commit: 5, MinTXID: 2, @@ -227,7 +227,7 @@ func TestCompactor_Compact(t *testing.T) { assertFileSpecEqual(t, spec, <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, PageSize: 1024, Commit: 2, MinTXID: 2, @@ -327,6 +327,167 @@ func TestCompactor_Compact(t *testing.T) { } }) + t.Run("EncryptedToEncrypted", func(t *testing.T) { + pub, priv, _ := ltx.GenerateKeyPair() + + page1Data := bytes.Repeat([]byte{0x81}, 1024) + page2Data := bytes.Repeat([]byte{0x82}, 1024) + + // Create encrypted input. + var encBuf bytes.Buffer + enc, _ := ltx.NewEncoder(&encBuf) + _ = enc.SetEncryption([][]byte{pub}) + _ = enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 2, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }) + _ = enc.EncodePage(ltx.PageHeader{Pgno: 1}, page1Data) + _ = enc.EncodePage(ltx.PageHeader{Pgno: 2}, page2Data) + chksum := ltx.ChecksumFlag + chksum = ltx.ChecksumFlag | (chksum ^ ltx.ChecksumPage(1, page1Data)) + chksum = ltx.ChecksumFlag | (chksum ^ ltx.ChecksumPage(2, page2Data)) + enc.SetPostApplyChecksum(chksum) + _ = enc.Close() + + // Compact encrypted → encrypted. + var output bytes.Buffer + c, _ := ltx.NewCompactor(&output, []io.Reader{&encBuf}) + c.DecryptionKey = priv + c.OutputEncryption = [][]byte{pub} + if err := c.Compact(context.Background()); err != nil { + t.Fatal(err) + } + + // Verify output is decryptable. + dec := ltx.NewDecoder(&output) + dec.SetDecryptionKey(priv) + if err := dec.DecodeHeader(); err != nil { + t.Fatal(err) + } + if !dec.Header().Encrypted() { + t.Fatal("expected encrypted output") + } + + var hdr ltx.PageHeader + data := make([]byte, 1024) + if err := dec.DecodePage(&hdr, data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, page1Data) { + t.Fatal("page 1 data mismatch") + } + if err := dec.DecodePage(&hdr, data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, page2Data) { + t.Fatal("page 2 data mismatch") + } + if err := dec.DecodePage(&hdr, data); err != io.EOF { + t.Fatal("expected EOF") + } + if err := dec.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("EncryptedToUnencrypted", func(t *testing.T) { + pub, priv, _ := ltx.GenerateKeyPair() + + page1Data := bytes.Repeat([]byte{0x81}, 1024) + + var encBuf bytes.Buffer + enc, _ := ltx.NewEncoder(&encBuf) + _ = enc.SetEncryption([][]byte{pub}) + _ = enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }) + _ = enc.EncodePage(ltx.PageHeader{Pgno: 1}, page1Data) + enc.SetPostApplyChecksum(ltx.ChecksumFlag | ltx.ChecksumPage(1, page1Data)) + _ = enc.Close() + + // Compact encrypted → unencrypted. + var output bytes.Buffer + c, _ := ltx.NewCompactor(&output, []io.Reader{&encBuf}) + c.DecryptionKey = priv + if err := c.Compact(context.Background()); err != nil { + t.Fatal(err) + } + + // Verify output is unencrypted. + dec := ltx.NewDecoder(&output) + if err := dec.DecodeHeader(); err != nil { + t.Fatal(err) + } + if dec.Header().Encrypted() { + t.Fatal("expected unencrypted output") + } + + var hdr ltx.PageHeader + data := make([]byte, 1024) + if err := dec.DecodePage(&hdr, data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, page1Data) { + t.Fatal("data mismatch") + } + if err := dec.DecodePage(&hdr, data); err != io.EOF { + t.Fatal("expected EOF") + } + if err := dec.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("UnencryptedToEncrypted", func(t *testing.T) { + pub, priv, _ := ltx.GenerateKeyPair() + + page1Data := bytes.Repeat([]byte{0x81}, 1024) + + var buf bytes.Buffer + enc, _ := ltx.NewEncoder(&buf) + _ = enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }) + _ = enc.EncodePage(ltx.PageHeader{Pgno: 1}, page1Data) + enc.SetPostApplyChecksum(ltx.ChecksumFlag | ltx.ChecksumPage(1, page1Data)) + _ = enc.Close() + + // Compact unencrypted → encrypted. + var output bytes.Buffer + c, _ := ltx.NewCompactor(&output, []io.Reader{&buf}) + c.OutputEncryption = [][]byte{pub} + if err := c.Compact(context.Background()); err != nil { + t.Fatal(err) + } + + // Verify output is encrypted and decryptable. + dec := ltx.NewDecoder(&output) + dec.SetDecryptionKey(priv) + if err := dec.DecodeHeader(); err != nil { + t.Fatal(err) + } + if !dec.Header().Encrypted() { + t.Fatal("expected encrypted output") + } + + var hdr ltx.PageHeader + data := make([]byte, 1024) + if err := dec.DecodePage(&hdr, data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, page1Data) { + t.Fatal("data mismatch") + } + if err := dec.DecodePage(&hdr, data); err != io.EOF { + t.Fatal("expected EOF") + } + if err := dec.Close(); err != nil { + t.Fatal(err) + } + }) + t.Run("Status", func(t *testing.T) { bufs := make([]bytes.Buffer, 2) writeFileSpec(t, &bufs[0], <x.FileSpec{ diff --git a/crypto.go b/crypto.go new file mode 100644 index 0000000..2f76231 --- /dev/null +++ b/crypto.go @@ -0,0 +1,246 @@ +package ltx + +import ( + "crypto/ecdh" + "crypto/hkdf" + "crypto/hpke" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + + "golang.org/x/crypto/chacha20poly1305" +) + +const ( + CEKSize = 32 + X25519PubSize = 32 +) + +var ( + ErrDecryptionKeyRequired = fmt.Errorf("decryption key required for encrypted file") + ErrDecryptionKeyInvalid = fmt.Errorf("failed to decrypt: no matching recipient entry") + ErrIndexAuthFailed = fmt.Errorf("page index authentication failed") +) + +func hpkeKEM() hpke.KEM { return hpke.DHKEM(ecdh.X25519()) } +func hpkeKDF() hpke.KDF { return hpke.HKDFSHA256() } +func hpkeAEAD() hpke.AEAD { return hpke.ChaCha20Poly1305() } + +func GenerateKeyPair() (pub, priv []byte, err error) { + kem := hpkeKEM() + privKey, err := kem.GenerateKey() + if err != nil { + return nil, nil, fmt.Errorf("generate key: %w", err) + } + privBytes, err := privKey.Bytes() + if err != nil { + return nil, nil, fmt.Errorf("serialize private key: %w", err) + } + return privKey.PublicKey().Bytes(), privBytes, nil +} + +// GenerateCEK returns a fresh random content encryption key. +// +// A CEK MUST NOT be reused across files. Both AEAD constructions below depend +// on it: AuthenticateIndex uses a fixed nonce, and EncryptPage uses random +// 96-bit nonces. Each is safe only because the keys derived from a CEK +// authenticate and encrypt exactly one file. Reusing a CEK across files with +// differing content reuses a (key, nonce) pair, which for Poly1305 leaks the +// authenticator key and permits forgery. If a future change wants to avoid +// re-encrypting pages (during compaction, say), it must derive fresh nonces +// rather than reuse the CEK. +func GenerateCEK() ([]byte, error) { + cek := make([]byte, CEKSize) + if _, err := io.ReadFull(rand.Reader, cek); err != nil { + return nil, fmt.Errorf("generate CEK: %w", err) + } + return cek, nil +} + +type RecipientBlock struct { + Entries [][]byte +} + +func (rb *RecipientBlock) MarshalBinary() ([]byte, error) { + b := make([]byte, len(rb.Entries)*RecipientEntrySize) + for i, entry := range rb.Entries { + if len(entry) != RecipientEntrySize { + return nil, fmt.Errorf("invalid recipient entry size: %d", len(entry)) + } + copy(b[i*RecipientEntrySize:], entry) + } + return b, nil +} + +func (rb *RecipientBlock) UnmarshalBinary(b []byte, count int) error { + if len(b) < count*RecipientEntrySize { + return io.ErrShortBuffer + } + rb.Entries = make([][]byte, count) + for i := range count { + rb.Entries[i] = make([]byte, RecipientEntrySize) + copy(rb.Entries[i], b[i*RecipientEntrySize:]) + } + return nil +} + +func SealCEK(cek []byte, recipientPubKeys [][]byte) (*RecipientBlock, error) { + if len(cek) != CEKSize { + return nil, fmt.Errorf("invalid CEK size: %d", len(cek)) + } + + kem := hpkeKEM() + kdf := hpkeKDF() + aead := hpkeAEAD() + info := []byte("ltx-cek-wrap") + + rb := &RecipientBlock{Entries: make([][]byte, len(recipientPubKeys))} + + for i, pubKeyBytes := range recipientPubKeys { + pubKey, err := kem.NewPublicKey(pubKeyBytes) + if err != nil { + return nil, fmt.Errorf("parse public key %d: %w", i, err) + } + + sealed, err := hpke.Seal(pubKey, kdf, aead, info, cek) + if err != nil { + return nil, fmt.Errorf("seal CEK for recipient %d: %w", i, err) + } + + if len(sealed) != RecipientEntrySize { + return nil, fmt.Errorf("unexpected sealed size: got %d, want %d", len(sealed), RecipientEntrySize) + } + + rb.Entries[i] = sealed + } + + return rb, nil +} + +func OpenCEK(rb *RecipientBlock, privKeyBytes []byte) ([]byte, error) { + kem := hpkeKEM() + kdf := hpkeKDF() + aead := hpkeAEAD() + info := []byte("ltx-cek-wrap") + + privKey, err := kem.NewPrivateKey(privKeyBytes) + if err != nil { + return nil, fmt.Errorf("parse private key: %w", err) + } + + for _, entry := range rb.Entries { + cek, err := hpke.Open(privKey, kdf, aead, info, entry) + if err != nil { + continue + } + return cek, nil + } + + return nil, ErrDecryptionKeyInvalid +} + +func DerivePageKey(cek []byte) ([]byte, error) { + return deriveKey(cek, "ltx-page-key") +} + +func DeriveIndexKey(cek []byte) ([]byte, error) { + return deriveKey(cek, "ltx-index-key") +} + +func deriveKey(cek []byte, info string) ([]byte, error) { + key, err := hkdf.Expand(sha256.New, cek, info, CEKSize) + if err != nil { + return nil, fmt.Errorf("derive key (%s): %w", info, err) + } + return key, nil +} + +// EncryptPage seals one compressed page under a random nonce, returning +// nonce || ciphertext || tag. +// +// Nonces are random and 96 bits wide, so safety rests on pageKey being derived +// from a per-file CEK: the birthday bound only has to cover the pages of a +// single file, not every page ever written. See [GenerateCEK]. +func EncryptPage(pageKey, compressed, aad []byte) ([]byte, error) { + aead, err := chacha20poly1305.New(pageKey) + if err != nil { + return nil, fmt.Errorf("create AEAD: %w", err) + } + + nonce := make([]byte, NonceSize) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, fmt.Errorf("generate nonce: %w", err) + } + + ciphertext := aead.Seal(nil, nonce, compressed, aad) + + result := make([]byte, NonceSize+len(ciphertext)) + copy(result, nonce) + copy(result[NonceSize:], ciphertext) + return result, nil +} + +func DecryptPage(pageKey, encrypted, aad []byte) ([]byte, error) { + if len(encrypted) < NonceSize+AuthTagSize { + return nil, fmt.Errorf("encrypted data too short") + } + + aead, err := chacha20poly1305.New(pageKey) + if err != nil { + return nil, fmt.Errorf("create AEAD: %w", err) + } + + nonce := encrypted[:NonceSize] + ciphertext := encrypted[NonceSize:] + + plaintext, err := aead.Open(nil, nonce, ciphertext, aad) + if err != nil { + return nil, fmt.Errorf("decrypt page: %w", err) + } + return plaintext, nil +} + +func BuildPageAAD(headerHash [32]byte, pgno uint32, pageHeaderBytes []byte) []byte { + aad := make([]byte, 32+4+len(pageHeaderBytes)) + copy(aad, headerHash[:]) + binary.BigEndian.PutUint32(aad[32:], pgno) + copy(aad[36:], pageHeaderBytes) + return aad +} + +// AuthenticateIndex returns a Poly1305 tag over the page index bytes, which are +// passed as additional data with an empty plaintext. +// +// The nonce is a fixed constant. That is safe ONLY because indexKey is derived +// from a per-file CEK and therefore authenticates exactly one index. Two +// different indexes authenticated under the same key and this nonce would leak +// the Poly1305 key and allow index forgery. See [GenerateCEK]. +func AuthenticateIndex(indexKey, indexBytes []byte) ([]byte, error) { + aead, err := chacha20poly1305.New(indexKey) + if err != nil { + return nil, fmt.Errorf("create AEAD: %w", err) + } + + nonce := []byte("ltx-idx-auth") + tag := aead.Seal(nil, nonce, nil, indexBytes) + return tag, nil +} + +func VerifyIndexAuth(indexKey, indexBytes, tag []byte) error { + aead, err := chacha20poly1305.New(indexKey) + if err != nil { + return fmt.Errorf("create AEAD: %w", err) + } + + nonce := []byte("ltx-idx-auth") + if _, err := aead.Open(nil, nonce, tag, indexBytes); err != nil { + return ErrIndexAuthFailed + } + return nil +} + +func HeaderHash(headerBytes []byte) [32]byte { + return sha256.Sum256(headerBytes) +} diff --git a/crypto_test.go b/crypto_test.go new file mode 100644 index 0000000..b39f64e --- /dev/null +++ b/crypto_test.go @@ -0,0 +1,231 @@ +package ltx_test + +import ( + "bytes" + "testing" + + "github.com/superfly/ltx" +) + +func TestGenerateKeyPair(t *testing.T) { + pub, priv, err := ltx.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + if len(pub) != 32 { + t.Fatalf("public key length=%d, want 32", len(pub)) + } + if len(priv) != 32 { + t.Fatalf("private key length=%d, want 32", len(priv)) + } +} + +func TestSealOpenCEK(t *testing.T) { + t.Run("SingleRecipient", func(t *testing.T) { + pub, priv, err := ltx.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + + cek, err := ltx.GenerateCEK() + if err != nil { + t.Fatal(err) + } + + rb, err := ltx.SealCEK(cek, [][]byte{pub}) + if err != nil { + t.Fatal(err) + } + if len(rb.Entries) != 1 { + t.Fatalf("entries=%d, want 1", len(rb.Entries)) + } + + got, err := ltx.OpenCEK(rb, priv) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, cek) { + t.Fatal("CEK mismatch") + } + }) + + t.Run("MultiRecipient", func(t *testing.T) { + pub1, priv1, _ := ltx.GenerateKeyPair() + pub2, priv2, _ := ltx.GenerateKeyPair() + pub3, priv3, _ := ltx.GenerateKeyPair() + + cek, _ := ltx.GenerateCEK() + + rb, err := ltx.SealCEK(cek, [][]byte{pub1, pub2, pub3}) + if err != nil { + t.Fatal(err) + } + if len(rb.Entries) != 3 { + t.Fatalf("entries=%d, want 3", len(rb.Entries)) + } + + for i, priv := range [][]byte{priv1, priv2, priv3} { + got, err := ltx.OpenCEK(rb, priv) + if err != nil { + t.Fatalf("recipient %d: %v", i, err) + } + if !bytes.Equal(got, cek) { + t.Fatalf("recipient %d: CEK mismatch", i) + } + } + }) + + t.Run("WrongKey", func(t *testing.T) { + pub, _, _ := ltx.GenerateKeyPair() + _, wrongPriv, _ := ltx.GenerateKeyPair() + + cek, _ := ltx.GenerateCEK() + rb, err := ltx.SealCEK(cek, [][]byte{pub}) + if err != nil { + t.Fatal(err) + } + + if _, err := ltx.OpenCEK(rb, wrongPriv); err != ltx.ErrDecryptionKeyInvalid { + t.Fatalf("expected ErrDecryptionKeyInvalid, got: %v", err) + } + }) +} + +func TestRecipientBlock_MarshalUnmarshal(t *testing.T) { + pub, _, _ := ltx.GenerateKeyPair() + cek, _ := ltx.GenerateCEK() + + rb, err := ltx.SealCEK(cek, [][]byte{pub}) + if err != nil { + t.Fatal(err) + } + + data, err := rb.MarshalBinary() + if err != nil { + t.Fatal(err) + } + if len(data) != ltx.RecipientEntrySize { + t.Fatalf("marshal size=%d, want %d", len(data), ltx.RecipientEntrySize) + } + + var rb2 ltx.RecipientBlock + if err := rb2.UnmarshalBinary(data, 1); err != nil { + t.Fatal(err) + } + if len(rb2.Entries) != 1 { + t.Fatalf("entries=%d, want 1", len(rb2.Entries)) + } + if !bytes.Equal(rb2.Entries[0], rb.Entries[0]) { + t.Fatal("entry mismatch") + } +} + +func TestEncryptDecryptPage(t *testing.T) { + t.Run("RoundTrip", func(t *testing.T) { + cek, _ := ltx.GenerateCEK() + pageKey, _ := ltx.DerivePageKey(cek) + + compressed := []byte("compressed page data here for testing round-trip encryption") + aad := ltx.BuildPageAAD([32]byte{1, 2, 3}, 42, []byte{0, 0, 0, 42, 0, 1}) + + encrypted, err := ltx.EncryptPage(pageKey, compressed, aad) + if err != nil { + t.Fatal(err) + } + if len(encrypted) < ltx.NonceSize+ltx.AuthTagSize+len(compressed) { + t.Fatal("encrypted data too short") + } + + decrypted, err := ltx.DecryptPage(pageKey, encrypted, aad) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(decrypted, compressed) { + t.Fatal("plaintext mismatch") + } + }) + + t.Run("TamperedCiphertext", func(t *testing.T) { + cek, _ := ltx.GenerateCEK() + pageKey, _ := ltx.DerivePageKey(cek) + aad := ltx.BuildPageAAD([32]byte{}, 1, []byte{0, 0, 0, 1, 0, 0}) + + encrypted, _ := ltx.EncryptPage(pageKey, []byte("data"), aad) + encrypted[ltx.NonceSize+2] ^= 0xFF + + if _, err := ltx.DecryptPage(pageKey, encrypted, aad); err == nil { + t.Fatal("expected error on tampered ciphertext") + } + }) + + t.Run("WrongAAD", func(t *testing.T) { + cek, _ := ltx.GenerateCEK() + pageKey, _ := ltx.DerivePageKey(cek) + aad := ltx.BuildPageAAD([32]byte{}, 1, []byte{0, 0, 0, 1, 0, 0}) + wrongAAD := ltx.BuildPageAAD([32]byte{}, 2, []byte{0, 0, 0, 2, 0, 0}) + + encrypted, _ := ltx.EncryptPage(pageKey, []byte("data"), aad) + + if _, err := ltx.DecryptPage(pageKey, encrypted, wrongAAD); err == nil { + t.Fatal("expected error with wrong AAD") + } + }) +} + +func TestAuthenticateVerifyIndex(t *testing.T) { + t.Run("RoundTrip", func(t *testing.T) { + cek, _ := ltx.GenerateCEK() + indexKey, _ := ltx.DeriveIndexKey(cek) + indexBytes := []byte("varint encoded page index data here") + + tag, err := ltx.AuthenticateIndex(indexKey, indexBytes) + if err != nil { + t.Fatal(err) + } + if len(tag) != ltx.AuthTagSize { + t.Fatalf("tag size=%d, want %d", len(tag), ltx.AuthTagSize) + } + + if err := ltx.VerifyIndexAuth(indexKey, indexBytes, tag); err != nil { + t.Fatal(err) + } + }) + + t.Run("TamperedIndex", func(t *testing.T) { + cek, _ := ltx.GenerateCEK() + indexKey, _ := ltx.DeriveIndexKey(cek) + indexBytes := []byte("some index data") + + tag, _ := ltx.AuthenticateIndex(indexKey, indexBytes) + + tampered := append([]byte{}, indexBytes...) + tampered[0] ^= 0xFF + + if err := ltx.VerifyIndexAuth(indexKey, tampered, tag); err != ltx.ErrIndexAuthFailed { + t.Fatalf("expected ErrIndexAuthFailed, got: %v", err) + } + }) +} + +func TestDeriveKeys(t *testing.T) { + cek, _ := ltx.GenerateCEK() + + pageKey, err := ltx.DerivePageKey(cek) + if err != nil { + t.Fatal(err) + } + indexKey, err := ltx.DeriveIndexKey(cek) + if err != nil { + t.Fatal(err) + } + + if len(pageKey) != 32 { + t.Fatalf("page key length=%d, want 32", len(pageKey)) + } + if len(indexKey) != 32 { + t.Fatalf("index key length=%d, want 32", len(indexKey)) + } + if bytes.Equal(pageKey, indexKey) { + t.Fatal("page key and index key must be different") + } +} diff --git a/decoder.go b/decoder.go index 8419ba7..5eebfcd 100644 --- a/decoder.go +++ b/decoder.go @@ -32,6 +32,19 @@ type Decoder struct { hash hash.Hash64 pageN int // pages read n int64 // bytes read + + // Encryption fields + encrypted bool + // keyless is set when the file is encrypted but no decryption key was + // supplied. The file checksum covers ciphertext, so integrity can still be + // verified; page contents cannot. + keyless bool + privKey []byte + cek []byte + pageKey []byte + indexKey []byte + headerHash [32]byte + recipientBlock *RecipientBlock } // NewDecoder returns a new instance of Decoder. @@ -71,6 +84,12 @@ func (dec *Decoder) PageIndex() map[uint32]PageIndexElem { return dec.pageIndex } +// SetDecryptionKey sets the private key to use for decryption. +// Must be called before DecodeHeader. +func (dec *Decoder) SetDecryptionKey(privateKey []byte) { + dec.privKey = privateKey +} + // Close verifies the reader is at the end of the file and that the checksum matches. func (dec *Decoder) Close() error { if dec.state == stateClosed { @@ -84,33 +103,82 @@ func (dec *Decoder) Close() error { if err != nil { return fmt.Errorf("read all: %w", err) } - remaining := bytes.NewReader(remainingBytes) - // Write everything but the file checksum to the hash. - dec.writeToHash(remainingBytes[:len(remainingBytes)-ChecksumSize]) + if dec.encrypted { + // For encrypted files, remaining contains: + // page_index || index_auth_tag(16) || duplicate_recipient_block || trailer(16) + // We need to hash everything except the file checksum (last 8 bytes). + dec.writeToHash(remainingBytes[:len(remainingBytes)-ChecksumSize]) - // Read page index. - if dec.pageIndex, err = DecodePageIndex(remaining, 0, dec.header.MinTXID, dec.header.MaxTXID); err != nil { - return fmt.Errorf("read page index: %w", err) - } + // Parse the remaining data. We read from the end backwards: + // - Last 16 bytes: trailer + // - Before trailer: duplicate recipient block (RecipientCount * 80) + // - Before recipient block: index auth tag (16 bytes) + // - Rest: page index - // Read trailer. - b := make([]byte, TrailerSize) - if _, err := io.ReadFull(remaining, b); err != nil { - return err - } else if err := dec.trailer.UnmarshalBinary(b); err != nil { - return fmt.Errorf("unmarshal trailer: %w", err) - } + r := bytes.NewReader(remainingBytes) + + trailerStart := len(remainingBytes) - TrailerSize + dupRecipientSize := dec.header.RecipientBlockSize() + dupRecipientStart := trailerStart - dupRecipientSize + authTagStart := dupRecipientStart - AuthTagSize - // TODO: Ensure last read page is equal to the commit for snapshot LTX files + if authTagStart < 0 { + return fmt.Errorf("encrypted file too short for index auth + recipient block + trailer") + } + + // Read page index from the beginning to authTagStart. + indexBytes := remainingBytes[:authTagStart] + indexReader := bytes.NewReader(indexBytes) + + if dec.pageIndex, err = DecodePageIndex(indexReader, 0, dec.header.MinTXID, dec.header.MaxTXID); err != nil { + return fmt.Errorf("read page index: %w", err) + } + + // Verify index auth tag. Requires the index key, so it is skipped when + // decoding without a key; the file checksum still covers these bytes. + if !dec.keyless { + authTag := remainingBytes[authTagStart : authTagStart+AuthTagSize] + if err := VerifyIndexAuth(dec.indexKey, indexBytes, authTag); err != nil { + return err + } + } + + // Read trailer (skip duplicate recipient block, already consumed by hash). + _ = r // used implicitly via remainingBytes + trailerBuf := remainingBytes[trailerStart:] + if err := dec.trailer.UnmarshalBinary(trailerBuf); err != nil { + return fmt.Errorf("unmarshal trailer: %w", err) + } + } else { + // Write everything but the file checksum to the hash. + dec.writeToHash(remainingBytes[:len(remainingBytes)-ChecksumSize]) + + remaining := bytes.NewReader(remainingBytes) + + // Read page index. + if dec.pageIndex, err = DecodePageIndex(remaining, 0, dec.header.MinTXID, dec.header.MaxTXID); err != nil { + return fmt.Errorf("read page index: %w", err) + } + + // Read trailer. + b := make([]byte, TrailerSize) + if _, err := io.ReadFull(remaining, b); err != nil { + return err + } else if err := dec.trailer.UnmarshalBinary(b); err != nil { + return fmt.Errorf("unmarshal trailer: %w", err) + } + } // Compare file checksum with checksum in trailer. if chksum := ChecksumFlag | Checksum(dec.hash.Sum64()); chksum != dec.trailer.FileChecksum { return ErrChecksumMismatch } - // Verify post-apply checksum for snapshot files if checksums are being tracked. - if dec.header.IsSnapshot() && !dec.header.NoChecksum() { + // Verify post-apply checksum for snapshot files if checksums are being + // tracked. The rolling checksum is computed over plaintext pages, so it is + // unavailable when decoding without a key. + if dec.header.IsSnapshot() && !dec.header.NoChecksum() && !dec.keyless { if dec.trailer.PostApplyChecksum != dec.chksum { return fmt.Errorf("post-apply checksum in trailer (%s) does not match calculated checksum (%s)", dec.trailer.PostApplyChecksum, dec.chksum) } @@ -144,11 +212,68 @@ func (dec *Decoder) DecodeHeader() error { dec.chksum = ChecksumFlag } + // Handle encryption. + if dec.header.Encrypted() { + // Without a key the file can still be checked for integrity, because the + // file checksum covers the ciphertext as written. Page contents stay + // unreadable: DecodePage refuses to hand back plaintext in this mode. + dec.keyless = dec.privKey == nil + + dec.encrypted = true + dec.headerHash = HeaderHash(b) + + // Read recipient block. + rbSize := dec.header.RecipientBlockSize() + rbBytes := make([]byte, rbSize) + if _, err := io.ReadFull(dec.r, rbBytes); err != nil { + return fmt.Errorf("read recipient block: %w", err) + } + dec.writeToHash(rbBytes) + + var rb RecipientBlock + if err := rb.UnmarshalBinary(rbBytes, int(dec.header.RecipientCount)); err != nil { + return fmt.Errorf("unmarshal recipient block: %w", err) + } + dec.recipientBlock = &rb + + if !dec.keyless { + // Open CEK. + cek, err := OpenCEK(&rb, dec.privKey) + if err != nil { + return fmt.Errorf("open CEK: %w", err) + } + dec.cek = cek + + // Derive keys. + dec.pageKey, err = DerivePageKey(cek) + if err != nil { + return fmt.Errorf("derive page key: %w", err) + } + dec.indexKey, err = DeriveIndexKey(cek) + if err != nil { + return fmt.Errorf("derive index key: %w", err) + } + } + } + return nil } // DecodePage reads the next page header into hdr and associated page data. +// +// Returns ErrDecryptionKeyRequired for an encrypted file opened without a +// decryption key. Use Verify to check such a file's integrity instead. func (dec *Decoder) DecodePage(hdr *PageHeader, data []byte) error { + if dec.keyless { + return ErrDecryptionKeyRequired + } + return dec.decodePage(hdr, data) +} + +// decodePage reads the next page frame. When dec.keyless is set the ciphertext +// is read and folded into the file checksum but not decrypted, and data is left +// untouched. +func (dec *Decoder) decodePage(hdr *PageHeader, data []byte) error { if dec.state == stateClosed { return ErrDecoderClosed } else if dec.state == stateClose { @@ -179,9 +304,14 @@ func (dec *Decoder) DecodePage(hdr *PageHeader, data []byte) error { return err } + // Encrypted files must use the size-prefixed block format. + if dec.encrypted && hdr.Flags&PageHeaderFlagSize == 0 { + return fmt.Errorf("encrypted file contains page without size flag (pgno=%d)", hdr.Pgno) + } + // Read page data using format-specific approach. if hdr.Flags&PageHeaderFlagSize != 0 { - // New block format: read size prefix, then LZ4 block data. + // New block format: read size prefix, then data. sizeBuf := make([]byte, 4) if _, err := io.ReadFull(dec.r, sizeBuf); err != nil { return fmt.Errorf("read data size: %w", err) @@ -189,17 +319,42 @@ func (dec *Decoder) DecodePage(hdr *PageHeader, data []byte) error { dec.writeToHash(sizeBuf) dataSize := binary.BigEndian.Uint32(sizeBuf) - compressed := make([]byte, dataSize) - if _, err := io.ReadFull(dec.r, compressed); err != nil { - return fmt.Errorf("read compressed data: %w", err) + rawData := make([]byte, dataSize) + if _, err := io.ReadFull(dec.r, rawData); err != nil { + return fmt.Errorf("read data: %w", err) } - if _, err := lz4.UncompressBlock(compressed, data); err != nil { - return fmt.Errorf("decompress block: %w", err) + + if dec.encrypted { + // Hash the encrypted data (what's on disk). + _, _ = dec.hash.Write(rawData) + dec.n += int64(len(rawData)) + + // Without a key, the ciphertext has been folded into the file + // checksum and there is nothing further to do for this page. + if dec.keyless { + dec.pageN++ + return nil + } + + // Decrypt. + aad := BuildPageAAD(dec.headerHash, hdr.Pgno, b) + compressed, err := DecryptPage(dec.pageKey, rawData, aad) + if err != nil { + return fmt.Errorf("decrypt page %d: %w", hdr.Pgno, err) + } + + // Decompress. + if _, err := lz4.UncompressBlock(compressed, data); err != nil { + return fmt.Errorf("decompress block: %w", err) + } + } else { + // Hash and decompress unencrypted data. + if _, err := lz4.UncompressBlock(rawData, data); err != nil { + return fmt.Errorf("decompress block: %w", err) + } } } else { // Old format: use LimitedReader workaround for lz4 frame concatenation. - // The lz4 library peeks ahead after EOF to check for concatenated frames, - // so we limit reads to prevent it from reading into the next page header. dec.lr.R = dec.r dec.lr.N = math.MaxInt64 dec.zr.Reset(&dec.lr) @@ -215,7 +370,9 @@ func (dec *Decoder) DecodePage(hdr *PageHeader, data []byte) error { } } - dec.writeToHash(data) + if !dec.encrypted { + dec.writeToHash(data) + } dec.pageN++ // Calculate checksum while decoding snapshots if tracking checksums. @@ -238,7 +395,10 @@ func (dec *Decoder) Verify() error { var pageHeader PageHeader data := make([]byte, dec.header.PageSize) for i := 0; ; i++ { - if err := dec.DecodePage(&pageHeader, data); err == io.EOF { + // decodePage rather than DecodePage: for an encrypted file with no key + // this verifies the file checksum over the ciphertext without + // attempting to recover page contents. + if err := dec.decodePage(&pageHeader, data); err == io.EOF { break } else if err != nil { return fmt.Errorf("decode page %d: %w", i, err) diff --git a/decoder_test.go b/decoder_test.go index 4882c18..0b1b009 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -13,7 +13,7 @@ import ( func TestDecoder(t *testing.T) { spec := <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, PageSize: 1024, Commit: 2, MinTXID: 1, @@ -109,7 +109,7 @@ func TestDecoder(t *testing.T) { func TestDecoder_Decode_CommitZero(t *testing.T) { spec := <x.FileSpec{ Header: ltx.Header{ - Version: ltx.Version, + Version: ltx.Version3, Flags: 0, PageSize: 1024, Commit: 0, @@ -251,6 +251,300 @@ func TestDecoder_DecodeDatabaseTo(t *testing.T) { }) } +func TestDecoder_Encrypted(t *testing.T) { + t.Run("RoundTrip", func(t *testing.T) { + pub, priv, err := ltx.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + + page1Data := bytes.Repeat([]byte("x"), 1024) + page2Data := bytes.Repeat([]byte("y"), 1024) + + chksum := ltx.ChecksumFlag + chksum = ltx.ChecksumFlag | (chksum ^ ltx.ChecksumPage(1, page1Data)) + chksum = ltx.ChecksumFlag | (chksum ^ ltx.ChecksumPage(2, page2Data)) + + var buf bytes.Buffer + enc, err := ltx.NewEncoder(&buf) + if err != nil { + t.Fatal(err) + } + if err := enc.SetEncryption([][]byte{pub}); err != nil { + t.Fatal(err) + } + if err := enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, + PageSize: 1024, + Commit: 2, + MinTXID: 1, + MaxTXID: 1, + Timestamp: 1000, + }); err != nil { + t.Fatal(err) + } + if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, page1Data); err != nil { + t.Fatal(err) + } + if err := enc.EncodePage(ltx.PageHeader{Pgno: 2}, page2Data); err != nil { + t.Fatal(err) + } + enc.SetPostApplyChecksum(chksum) + if err := enc.Close(); err != nil { + t.Fatal(err) + } + + // Verify header has encryption flag. + hdr := enc.Header() + if !hdr.Encrypted() { + t.Fatal("expected encrypted flag") + } + if hdr.RecipientCount != 1 { + t.Fatalf("recipient count=%d, want 1", hdr.RecipientCount) + } + + // Decode. + dec := ltx.NewDecoder(&buf) + dec.SetDecryptionKey(priv) + if err := dec.DecodeHeader(); err != nil { + t.Fatal(err) + } + + var pageHdr ltx.PageHeader + data := make([]byte, 1024) + + if err := dec.DecodePage(&pageHdr, data); err != nil { + t.Fatal(err) + } + if pageHdr.Pgno != 1 { + t.Fatalf("pgno=%d, want 1", pageHdr.Pgno) + } + if !bytes.Equal(data, page1Data) { + t.Fatal("page 1 data mismatch") + } + + if err := dec.DecodePage(&pageHdr, data); err != nil { + t.Fatal(err) + } + if pageHdr.Pgno != 2 { + t.Fatalf("pgno=%d, want 2", pageHdr.Pgno) + } + if !bytes.Equal(data, page2Data) { + t.Fatal("page 2 data mismatch") + } + + if err := dec.DecodePage(&pageHdr, data); err != io.EOF { + t.Fatalf("expected EOF, got %v", err) + } + if err := dec.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("MultiRecipient", func(t *testing.T) { + pub1, priv1, _ := ltx.GenerateKeyPair() + pub2, priv2, _ := ltx.GenerateKeyPair() + + page1Data := bytes.Repeat([]byte("a"), 1024) + chksum := ltx.ChecksumFlag | ltx.ChecksumPage(1, page1Data) + + var buf bytes.Buffer + enc, err := ltx.NewEncoder(&buf) + if err != nil { + t.Fatal(err) + } + if err := enc.SetEncryption([][]byte{pub1, pub2}); err != nil { + t.Fatal(err) + } + if err := enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }); err != nil { + t.Fatal(err) + } + if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, page1Data); err != nil { + t.Fatal(err) + } + enc.SetPostApplyChecksum(chksum) + if err := enc.Close(); err != nil { + t.Fatal(err) + } + + encoded := buf.Bytes() + + // Both keys should decrypt. + for i, priv := range [][]byte{priv1, priv2} { + dec := ltx.NewDecoder(bytes.NewReader(encoded)) + dec.SetDecryptionKey(priv) + if err := dec.DecodeHeader(); err != nil { + t.Fatalf("recipient %d: decode header: %v", i, err) + } + + var hdr ltx.PageHeader + data := make([]byte, 1024) + if err := dec.DecodePage(&hdr, data); err != nil { + t.Fatalf("recipient %d: decode page: %v", i, err) + } + if !bytes.Equal(data, page1Data) { + t.Fatalf("recipient %d: data mismatch", i) + } + + if err := dec.DecodePage(&hdr, data); err != io.EOF { + t.Fatalf("recipient %d: expected EOF", i) + } + if err := dec.Close(); err != nil { + t.Fatalf("recipient %d: close: %v", i, err) + } + } + }) + + // encodeEncrypted builds a single-page encrypted file for the keyless tests. + encodeEncrypted := func(t *testing.T) []byte { + t.Helper() + + pub, _, err := ltx.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + enc, err := ltx.NewEncoder(&buf) + if err != nil { + t.Fatal(err) + } + if err := enc.SetEncryption([][]byte{pub}); err != nil { + t.Fatal(err) + } + if err := enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }); err != nil { + t.Fatal(err) + } + page := bytes.Repeat([]byte("s"), 1024) + if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, page); err != nil { + t.Fatal(err) + } + enc.SetPostApplyChecksum(ltx.ChecksumPage(1, page)) + if err := enc.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() + } + + // The file checksum of an encrypted file covers the ciphertext as written, + // so integrity is verifiable by a holder of the file alone. Contents are + // not: DecodePage must refuse rather than hand back an empty buffer. + t.Run("NoKeyVerifiesIntegrity", func(t *testing.T) { + b := encodeEncrypted(t) + + if err := ltx.NewDecoder(bytes.NewReader(b)).Verify(); err != nil { + t.Fatalf("keyless Verify should succeed on an intact file: %v", err) + } + }) + + t.Run("NoKeyRefusesPageContents", func(t *testing.T) { + b := encodeEncrypted(t) + + dec := ltx.NewDecoder(bytes.NewReader(b)) + if err := dec.DecodeHeader(); err != nil { + t.Fatalf("keyless DecodeHeader should succeed: %v", err) + } + + var hdr ltx.PageHeader + data := make([]byte, 1024) + if err := dec.DecodePage(&hdr, data); err != ltx.ErrDecryptionKeyRequired { + t.Fatalf("expected ErrDecryptionKeyRequired, got: %v", err) + } + }) + + t.Run("NoKeyDetectsCorruption", func(t *testing.T) { + b := encodeEncrypted(t) + + // Flip a bit in the final page's ciphertext. Keyless verification must + // still catch it, otherwise hashing ciphertext buys nothing. + corrupt := make([]byte, len(b)) + copy(corrupt, b) + corrupt[len(corrupt)-40] ^= 0x01 + + if err := ltx.NewDecoder(bytes.NewReader(corrupt)).Verify(); err == nil { + t.Fatal("keyless Verify accepted a corrupted file") + } + }) + + t.Run("WrongKeyForEncryptedFile", func(t *testing.T) { + pub, _, _ := ltx.GenerateKeyPair() + _, wrongPriv, _ := ltx.GenerateKeyPair() + + var buf bytes.Buffer + enc, _ := ltx.NewEncoder(&buf) + _ = enc.SetEncryption([][]byte{pub}) + _ = enc.EncodeHeader(ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }) + _ = enc.EncodePage(ltx.PageHeader{Pgno: 1}, make([]byte, 1024)) + enc.SetPostApplyChecksum(ltx.ChecksumFlag) + _ = enc.Close() + + dec := ltx.NewDecoder(&buf) + dec.SetDecryptionKey(wrongPriv) + if err := dec.DecodeHeader(); err == nil { + t.Fatal("expected error with wrong key") + } + }) +} + +func TestFileSpec_StripEncryptionWhenNoRecipients(t *testing.T) { + pub, priv, _ := ltx.GenerateKeyPair() + + pageData := make([]byte, 1024) + pageData[0] = 0xAB + + chksum := ltx.ChecksumFlag | ltx.ChecksumPage(1, pageData) + + // Create encrypted file. + var encBuf bytes.Buffer + spec := ltx.FileSpec{ + Header: ltx.Header{ + Version: ltx.Version, PageSize: 1024, Commit: 1, + MinTXID: 1, MaxTXID: 1, Timestamp: 1000, + }, + Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: pageData}}, + Trailer: ltx.Trailer{PostApplyChecksum: chksum}, + + RecipientPublicKeys: [][]byte{pub}, + } + if _, err := spec.WriteTo(&encBuf); err != nil { + t.Fatal(err) + } + + // Read back with key. + var spec2 ltx.FileSpec + if _, err := spec2.ReadFromWithKey(bytes.NewReader(encBuf.Bytes()), priv); err != nil { + t.Fatal(err) + } + + // Write without recipients — should produce a valid unencrypted file. + spec2.RecipientPublicKeys = nil + var plainBuf bytes.Buffer + if _, err := spec2.WriteTo(&plainBuf); err != nil { + t.Fatal(err) + } + + // Verify the output is readable without a key. + var spec3 ltx.FileSpec + if _, err := spec3.ReadFrom(bytes.NewReader(plainBuf.Bytes())); err != nil { + t.Fatalf("expected unencrypted file to be readable without key: %v", err) + } + if spec3.Header.Encrypted() { + t.Fatal("expected header to not have encrypted flag") + } + if !bytes.Equal(spec3.Pages[0].Data, pageData) { + t.Fatal("page data mismatch") + } +} + func TestDecoder_64KBPageSize(t *testing.T) { const pageSize = 65536 // 64KB - maximum SQLite page size diff --git a/dist/ltx b/dist/ltx deleted file mode 100755 index 75465d5..0000000 Binary files a/dist/ltx and /dev/null differ diff --git a/e2e_encryption_test.sh b/e2e_encryption_test.sh new file mode 100755 index 0000000..b9dbccc --- /dev/null +++ b/e2e_encryption_test.sh @@ -0,0 +1,759 @@ +#!/usr/bin/env bash +set -euo pipefail + +# E2E Encryption Tests for LTX HPKE (v4) +# Tests the full CLI workflow for per-page encryption using HPKE (RFC 9180). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LTX="" +TMPDIR="" +REPORT="" +PASS_COUNT=0 +FAIL_COUNT=0 +TOTAL_COUNT=0 +TEST_OUTPUTS="" + +# --- Helpers --- + +log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + log "PASS: $1" + TEST_OUTPUTS+="| $1 | PASS | ${2:-} |\n" +} + +fail() { + FAIL_COUNT=$((FAIL_COUNT + 1)) + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + log "FAIL: $1 — $2" + TEST_OUTPUTS+="| $1 | **FAIL** | ${2:-} |\n" +} + +millis() { python3 -c 'import time; print(int(time.time()*1000))'; } + +fmt_duration() { + local ms="$1" + if [[ $ms -ge 1000 ]]; then + printf "%d.%03ds" $((ms / 1000)) $((ms % 1000)) + else + printf "%dms" "$ms" + fi +} + +run_test() { + local name="$1" + local fn="$2" + log "--- Running: $name ---" + local start + start=$(millis) + if "$fn"; then + local dur=$(( $(millis) - start )) + pass "$name" "$(fmt_duration $dur)" + else + local dur=$(( $(millis) - start )) + fail "$name" "$(fmt_duration $dur)" + fi +} + +create_db() { + local path="$1" + local page_size="${2:-4096}" + local sql="$3" + sqlite3 "$path" </dev/null; then + echo "ASSERT FAILED: expected non-zero exit for: $desc" + return 1 + fi + return 0 +} + +assert_exit_zero() { + local desc="$1" + shift + if ! "$@" 2>&1; then + echo "ASSERT FAILED: expected zero exit for: $desc" + return 1 + fi + return 0 +} + +# --- Setup --- + +setup() { + log "Checking prerequisites..." + + if ! command -v go &>/dev/null; then + echo "ERROR: go is required" >&2; exit 1 + fi + local go_ver + go_ver=$(go version | grep -oE '[0-9]+\.[0-9]+') + log "Go version: $(go version)" + + if ! command -v sqlite3 &>/dev/null; then + echo "ERROR: sqlite3 is required" >&2; exit 1 + fi + log "sqlite3 version: $(sqlite3 --version)" + + HAS_DOCKER=false + if command -v docker &>/dev/null; then + HAS_DOCKER=true + log "Docker: available" + else + log "Docker: not found (Docker test will be skipped)" + fi + + log "Building ltx binary..." + mkdir -p "$SCRIPT_DIR/dist" + (cd "$SCRIPT_DIR" && go build -o ./dist/ltx ./cmd/ltx) + LTX="$SCRIPT_DIR/dist/ltx" + log "Binary: $LTX ($("$LTX" version 2>&1 || echo 'unknown'))" + + TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/ltx-e2e.XXXXXX") + log "Temp dir: $TMPDIR" + + REPORT="$TMPDIR/e2e_report.md" + local git_commit + git_commit=$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") + + cat > "$REPORT" <
&1 || echo 'unknown') +- **Platform:** $(uname -s)/$(uname -m) + +## Results + +| Test | Result | Duration | +|------|--------|----------| +HEADER + + log "Setup complete." +} + +teardown() { + echo -e "$TEST_OUTPUTS" >> "$REPORT" + + cat >> "$REPORT" <