Skip to content

Commit d280ccf

Browse files
authored
test(golang): send Go-sealed v4 rows through PostgreSQL, and drop the withdrawn integer equality types
Nothing in the repository put a value the generated Go code sealed into an eql_v4 domain or operator; the PostgreSQL check was manual. The Go test now writes v4_rows.json, the Profile rows the generated code seals over the deterministic EQL guest. On every later run it checks that file against today's output, envelopes and match terms byte for byte and ciphertexts by opening them (their AEAD nonce is random). The EQL suite opens those rows with stack-kms's DeterministicSource over the same seed, and its PostgreSQL test, which test:encryption:postgres runs in CI, inserts them into eql_v4_* columns and queries name @@ with the Go query operand from v4_queries.json. With IntegerEq and BigintEq held back, Profile drops Age and stores Balance as Bigint, and v4_queries.json carries TextMatch only. The one stored-versus-query check had used Age, so it is replaced by a TextMatch check that a contained needle's filter bits are a subset of the stored filter and an absent one's are not. Every stored field's i is checked against its table and column. The Go workflow now runs when the fixtures or EQL's lockfile change, and biome leaves the generated rows file alone.
1 parent 65233bc commit d280ccf

9 files changed

Lines changed: 681 additions & 109 deletions

File tree

‎.github/workflows/tests-golang.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ on:
3636
# The eql guest build links eql-bindings.
3737
- packages/eql/crates/eql-bindings/**
3838
- packages/eql/crates/eql-domains/**
39+
# The Go tests read the cross-language fixtures the EQL suite writes,
40+
# which are derived under EQL's lockfile.
41+
- packages/eql/tests/encryption/fixtures/**
42+
- packages/eql/Cargo.lock
3943
- languages/golang/**
4044
- scripts/check-wasm-imports.py
4145
- scripts/go-binding-test.sh
@@ -61,6 +65,10 @@ on:
6165
# The eql guest build links eql-bindings.
6266
- packages/eql/crates/eql-bindings/**
6367
- packages/eql/crates/eql-domains/**
68+
# The Go tests read the cross-language fixtures the EQL suite writes,
69+
# which are derived under EQL's lockfile.
70+
- packages/eql/tests/encryption/fixtures/**
71+
- packages/eql/Cargo.lock
6472
- languages/golang/**
6573
- scripts/check-wasm-imports.py
6674
- scripts/go-binding-test.sh

‎biome.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"!languages/typescript/packages/protect-ffi/target",
1212
"!languages/typescript/packages/protect-ffi/src/eql-v3-types",
1313
"!packages/stack-encrypt/tests/fixtures/record_lowering.json",
14+
"!packages/eql/tests/encryption/fixtures/v4_rows.json",
1415
"!packages/eql/crates/eql-bindings/bindings",
1516
"!packages/eql/crates/eql-bindings/schema",
1617
"!packages/eql/packages/eql/src/generated",

‎languages/golang/encrypt/README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -288,14 +288,16 @@ The EQL types the engine produces, and the Go field types each takes:
288288
| `TextEq` | `string` | `=`, `<>` |
289289
| `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) |
290290
| `Text` | `string` | none: stored and read only |
291-
| `IntegerEq` | `int32` | `=`, `<>` |
292291
| `Integer` | `int32` | none |
293-
| `BigintEq` | `int64` | `=`, `<>` |
292+
| `Bigint` | `int64` | none |
294293
| `Boolean` | `bool` | none |
295294
| `Real` | `float32` | none |
296295
| `Double` | `float64` | none |
297296

298-
The rest of the catalog (ordering, dates, decimals, JSON) is listed in
297+
A `TextMatch` value must be at least three characters long: shorter text has
298+
no 3-grams to index and is refused. `IntegerEq` and `BigintEq` are held back
299+
until the engine's integer equality term settles in vitaminc 0.6. The rest of
300+
the catalog (ordering, dates, decimals, JSON) is listed in
299301
`eql.Types` with the reason it is not produced yet, and `stashgen` refuses it
300302
with that reason. The guest builds the value: the generated code
301303
imports `encrypt/eql`, which embeds the build of the engine that holds the
Lines changed: 210 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package encrypt_test
22

33
import (
4+
"bytes"
45
"context"
56
"encoding/json"
67
"os"
@@ -15,15 +16,23 @@ import (
1516
// The EQL v4 types the engine produces besides TextEq, sealed through
1617
// generated code over the deterministic guest build with the EQL types:
1718
// each Go field type reaches its v4 type's own plan, and the values open
18-
// back to what went in.
19+
// back to what went in. PostgreSQL sees these values through the Rust
20+
// suite: TestV4RowsMatchTheCommittedFixture commits them to
21+
// v4_rows.json, and packages/eql/tests/encryption inserts that file into
22+
// eql_v4 columns and queries it.
1923

2024
var profiles = []testusers.Profile{
21-
{ID: 1, Age: 42, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice"},
22-
{ID: 2, Age: -7, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith"},
25+
{ID: 1, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice"},
26+
{ID: 2, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith"},
2327
}
2428

25-
// keysOf is the sorted top-level keys of an EQL value.
26-
func keysOf(t *testing.T, value []byte) string {
29+
func fixturePath(name string) string {
30+
return filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", name)
31+
}
32+
33+
// keysOf is the sorted top-level keys of an EQL value. It also checks the
34+
// envelope: version 4, and the table and column the value was stored under.
35+
func keysOf(t *testing.T, column string, value []byte) string {
2736
t.Helper()
2837
var raw map[string]json.RawMessage
2938
if err := json.Unmarshal(value, &raw); err != nil {
@@ -32,6 +41,10 @@ func keysOf(t *testing.T, value []byte) string {
3241
if string(raw["v"]) != "4" {
3342
t.Errorf("v = %s, want 4: %s", raw["v"], value)
3443
}
44+
var i struct{ T, C string }
45+
if err := json.Unmarshal(raw["i"], &i); err != nil || i.T != "profiles" || i.C != column {
46+
t.Errorf("i = %s, want {\"t\":\"profiles\",\"c\":%q}", raw["i"], column)
47+
}
3548
keys := make([]string, 0, len(raw))
3649
for k := range raw {
3750
keys = append(keys, k)
@@ -40,6 +53,19 @@ func keysOf(t *testing.T, value []byte) string {
4053
return strings.Join(keys, ",")
4154
}
4255

56+
// stored is each sealed field of an encrypted profile by column.
57+
func stored(e testusers.EncryptedProfile) map[string][]byte {
58+
return map[string][]byte{
59+
"visits": e.Visits,
60+
"balance": e.Balance,
61+
"active": e.Active,
62+
"ratio": e.Ratio,
63+
"score": e.Score,
64+
"note": e.Note,
65+
"name": e.Name,
66+
}
67+
}
68+
4369
func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) {
4470
c := deterministicEQLClient(t)
4571
ctx := context.Background()
@@ -49,22 +75,19 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) {
4975
if err != nil {
5076
t.Fatal(err)
5177
}
78+
want := map[string]string{
79+
"visits": "c,i,v",
80+
"balance": "c,i,v",
81+
"active": "c,i,v",
82+
"ratio": "c,i,v",
83+
"score": "c,i,v",
84+
"note": "c,i,v",
85+
"name": "bf,c,i,v",
86+
}
5287
for i, e := range encrypted {
53-
for name, tc := range map[string]struct {
54-
value []byte
55-
keys string
56-
}{
57-
"age": {e.Age, "c,hm,i,v"},
58-
"visits": {e.Visits, "c,i,v"},
59-
"balance": {e.Balance, "c,hm,i,v"},
60-
"active": {e.Active, "c,i,v"},
61-
"ratio": {e.Ratio, "c,i,v"},
62-
"score": {e.Score, "c,i,v"},
63-
"note": {e.Note, "c,i,v"},
64-
"name": {e.Name, "bf,c,i,v"},
65-
} {
66-
if got := keysOf(t, tc.value); got != tc.keys {
67-
t.Errorf("row %d %s: keys %s, want %s", i, name, got, tc.keys)
88+
for column, value := range stored(e) {
89+
if got := keysOf(t, column, value); got != want[column] {
90+
t.Errorf("row %d %s: keys %s, want %s", i, column, got, want[column])
6891
}
6992
}
7093
}
@@ -80,26 +103,43 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) {
80103
}
81104
}
82105

106+
// bloom is the set of filter positions in a TextMatch value or query.
107+
func bloom(t *testing.T, value []byte) map[int]bool {
108+
t.Helper()
109+
var v struct {
110+
BF []int `json:"bf"`
111+
}
112+
if err := json.Unmarshal(value, &v); err != nil {
113+
t.Fatal(err)
114+
}
115+
set := make(map[int]bool, len(v.BF))
116+
for _, p := range v.BF {
117+
set[p] = true
118+
}
119+
return set
120+
}
121+
122+
func subset(a, b map[int]bool) bool {
123+
for p := range a {
124+
if !b[p] {
125+
return false
126+
}
127+
}
128+
return true
129+
}
130+
83131
func TestV4QueriesMatchTheRustFixture(t *testing.T) {
84132
c := deterministicEQLClient(t)
85133
ctx := context.Background()
86134
cipher := c.DefaultKeyset()
87135

88-
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", "v4_queries.json"))
136+
raw, err := os.ReadFile(fixturePath("v4_queries.json"))
89137
if err != nil {
90138
t.Fatal(err)
91139
}
92140
var fixture struct {
93141
Table string
94-
Age struct {
95-
Plaintext int32
96-
Query string
97-
}
98-
Balance struct {
99-
Plaintext int64
100-
Query string
101-
}
102-
Name struct {
142+
Name struct {
103143
Plaintext string
104144
Query string
105145
}
@@ -112,44 +152,165 @@ func TestV4QueriesMatchTheRustFixture(t *testing.T) {
112152
}
113153

114154
// Byte for byte what eql-bindings' own dispatch derived under the same
115-
// index key: each Go field runs its v4 type's plan and no other.
116-
age, err := testusers.ProfileFields.Age.Query(ctx, cipher, fixture.Age.Plaintext)
155+
// index key: the Go field runs its v4 type's plan and no other.
156+
name, err := testusers.ProfileFields.Name.Query(ctx, cipher, fixture.Name.Plaintext)
117157
if err != nil {
118158
t.Fatal(err)
119159
}
120-
if string(age) != fixture.Age.Query {
121-
t.Errorf("Age.Query = %s\nwant %s", age, fixture.Age.Query)
160+
if string(name) != fixture.Name.Query {
161+
t.Errorf("Name.Query = %s\nwant %s", name, fixture.Name.Query)
122162
}
123-
balance, err := testusers.ProfileFields.Balance.Query(ctx, cipher, fixture.Balance.Plaintext)
163+
164+
// And a stored value holds every filter bit of a query for text it
165+
// contains, which is what `@@` tests, and not those of text it lacks.
166+
encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles[:1])
124167
if err != nil {
125168
t.Fatal(err)
126169
}
127-
if string(balance) != fixture.Balance.Query {
128-
t.Errorf("Balance.Query = %s\nwant %s", balance, fixture.Balance.Query)
170+
all := bloom(t, encrypted[0].Name)
171+
for needle, contained := range map[string]bool{"alice": true, "LIC": true, "zzz": false, "bob": false} {
172+
q, err := testusers.ProfileFields.Name.Query(ctx, cipher, needle)
173+
if err != nil {
174+
t.Fatal(err)
175+
}
176+
if got := subset(bloom(t, q), all); got != contained {
177+
t.Errorf("%q in %q: %v, want %v", needle, profiles[0].Name, got, contained)
178+
}
129179
}
130-
name, err := testusers.ProfileFields.Name.Query(ctx, cipher, fixture.Name.Plaintext)
180+
}
181+
182+
// rowsFixture is v4_rows.json: the profiles above as the generated code seals
183+
// them under the deterministic guest and testSeed. The Rust suite opens them
184+
// with stack-kms's DeterministicSource over the same seed, and inserts and
185+
// queries them in PostgreSQL.
186+
type rowsFixture struct {
187+
Comment string `json:"_comment"`
188+
Seed string `json:"seed"`
189+
Table string `json:"table"`
190+
Rows []fixtureRow `json:"rows"`
191+
}
192+
193+
type fixtureRow struct {
194+
ID int64 `json:"id"`
195+
Plaintext fixturePlain `json:"plaintext"`
196+
Stored map[string]json.RawMessage `json:"stored"`
197+
}
198+
199+
type fixturePlain struct {
200+
Visits int32 `json:"visits"`
201+
Balance int64 `json:"balance"`
202+
Active bool `json:"active"`
203+
Ratio float32 `json:"ratio"`
204+
Score float64 `json:"score"`
205+
Note string `json:"note"`
206+
Name string `json:"name"`
207+
}
208+
209+
// TestV4RowsMatchTheCommittedFixture writes v4_rows.json when
210+
// EQL_UPDATE_FIXTURES is set, and otherwise checks the committed file
211+
// against what the generated code seals today. The deterministic guest
212+
// derives every data key from the seed, so the envelopes and the terms
213+
// (`bf`) are the same every run; the AEAD nonce inside each `c` is random,
214+
// so `c` is not. The test therefore compares everything but `c`, and opens
215+
// the committed `c` values with today's code.
216+
func TestV4RowsMatchTheCommittedFixture(t *testing.T) {
217+
c := deterministicEQLClient(t)
218+
encrypted, err := testusers.EncryptProfile(context.Background(), c.DefaultKeyset(), profiles)
131219
if err != nil {
132220
t.Fatal(err)
133221
}
134-
if string(name) != fixture.Name.Query {
135-
t.Errorf("Name.Query = %s\nwant %s", name, fixture.Name.Query)
222+
fixture := rowsFixture{
223+
Comment: "Profile rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4Rows, do not edit.",
224+
Seed: string(testSeed[:]),
225+
Table: "profiles",
136226
}
137-
138-
// And a stored value carries the term its query probes for.
139-
stored, err := testusers.ProfileFields.Age.Encrypt(ctx, cipher, fixture.Age.Plaintext)
227+
for i, e := range encrypted {
228+
p := profiles[i]
229+
row := fixtureRow{
230+
ID: p.ID,
231+
Plaintext: fixturePlain{p.Visits, p.Balance, p.Active, p.Ratio, p.Score, p.Note, p.Name},
232+
Stored: map[string]json.RawMessage{},
233+
}
234+
for column, value := range stored(e) {
235+
row.Stored[column] = json.RawMessage(value)
236+
}
237+
fixture.Rows = append(fixture.Rows, row)
238+
}
239+
rendered, err := json.MarshalIndent(fixture, "", " ")
140240
if err != nil {
141241
t.Fatal(err)
142242
}
143-
var s, q struct {
144-
HM string `json:"hm"`
243+
rendered = append(rendered, '\n')
244+
path := fixturePath("v4_rows.json")
245+
if os.Getenv("EQL_UPDATE_FIXTURES") != "" {
246+
if err := os.WriteFile(path, rendered, 0o644); err != nil {
247+
t.Fatal(err)
248+
}
249+
}
250+
committedBytes, err := os.ReadFile(path)
251+
if err != nil {
252+
t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err)
145253
}
146-
if err := json.Unmarshal(stored, &s); err != nil {
254+
var committed rowsFixture
255+
if err := json.Unmarshal(committedBytes, &committed); err != nil {
147256
t.Fatal(err)
148257
}
149-
if err := json.Unmarshal(age, &q); err != nil {
258+
if !bytes.Equal(withoutCiphertexts(t, committed), withoutCiphertexts(t, fixture)) {
259+
t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path)
260+
}
261+
262+
// The committed ciphertexts still open, to the committed plaintexts.
263+
sealed := make([]testusers.EncryptedProfile, len(committed.Rows))
264+
for i, row := range committed.Rows {
265+
sealed[i] = testusers.EncryptedProfile{
266+
ID: row.ID,
267+
Visits: []byte(row.Stored["visits"]),
268+
Balance: []byte(row.Stored["balance"]),
269+
Active: []byte(row.Stored["active"]),
270+
Ratio: []byte(row.Stored["ratio"]),
271+
Score: []byte(row.Stored["score"]),
272+
Note: []byte(row.Stored["note"]),
273+
Name: []byte(row.Stored["name"]),
274+
}
275+
}
276+
opened, err := testusers.DecryptProfile(context.Background(), c.DefaultKeyset(), sealed)
277+
if err != nil {
150278
t.Fatal(err)
151279
}
152-
if s.HM != q.HM {
153-
t.Errorf("stored hm %s, query hm %s", s.HM, q.HM)
280+
for i, row := range committed.Rows {
281+
p := opened[i]
282+
got := fixturePlain{p.Visits, p.Balance, p.Active, p.Ratio, p.Score, p.Note, p.Name}
283+
if got != row.Plaintext {
284+
t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext)
285+
}
286+
}
287+
}
288+
289+
// withoutCiphertexts renders a fixture with every stored value's `c`
290+
// removed: the part of the file that is the same on every run.
291+
func withoutCiphertexts(t *testing.T, f rowsFixture) []byte {
292+
t.Helper()
293+
out := f
294+
out.Rows = nil
295+
for _, row := range f.Rows {
296+
stripped := fixtureRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}}
297+
for column, value := range row.Stored {
298+
var fields map[string]json.RawMessage
299+
if err := json.Unmarshal(value, &fields); err != nil {
300+
t.Fatal(err)
301+
}
302+
delete(fields, "c")
303+
b, err := json.Marshal(fields)
304+
if err != nil {
305+
t.Fatal(err)
306+
}
307+
stripped.Stored[column] = b
308+
}
309+
out.Rows = append(out.Rows, stripped)
310+
}
311+
b, err := json.Marshal(out)
312+
if err != nil {
313+
t.Fatal(err)
154314
}
315+
return b
155316
}

0 commit comments

Comments
 (0)