11package encrypt_test
22
33import (
4+ "bytes"
45 "context"
56 "encoding/json"
67 "os"
@@ -15,15 +16,23 @@ import (
1516// The EQL v4 types the engine produces besides TextEq, sealed through
1617// generated code over the deterministic guest build with the EQL types:
1718// each Go field type reaches its v4 type's own plan, and the values open
18- // back to what went in.
19+ // back to what went in. PostgreSQL sees these values through the Rust
20+ // suite: TestV4RowsMatchTheCommittedFixture commits them to
21+ // v4_rows.json, and packages/eql/tests/encryption inserts that file into
22+ // eql_v4 columns and queries it.
1923
2024var profiles = []testusers.Profile {
21- {ID : 1 , Age : 42 , Visits : 3 , Balance : 1234567890123 , Active : true , Ratio : 0.5 , Score : 2.25 , Note : "a private note" , Name : "alice" },
22- {ID : 2 , Age : - 7 , Visits : 0 , Balance : - 1 , Active : false , Ratio : - 1.5 , Score : 0 , Note : "" , Name : "bob smith" },
25+ {ID : 1 , Visits : 3 , Balance : 1234567890123 , Active : true , Ratio : 0.5 , Score : 2.25 , Note : "a private note" , Name : "alice" },
26+ {ID : 2 , Visits : 0 , Balance : - 1 , Active : false , Ratio : - 1.5 , Score : 0 , Note : "" , Name : "bob smith" },
2327}
2428
25- // keysOf is the sorted top-level keys of an EQL value.
26- func keysOf (t * testing.T , value []byte ) string {
29+ func fixturePath (name string ) string {
30+ return filepath .Join (".." , ".." , ".." , "packages" , "eql" , "tests" , "encryption" , "fixtures" , name )
31+ }
32+
33+ // keysOf is the sorted top-level keys of an EQL value. It also checks the
34+ // envelope: version 4, and the table and column the value was stored under.
35+ func keysOf (t * testing.T , column string , value []byte ) string {
2736 t .Helper ()
2837 var raw map [string ]json.RawMessage
2938 if err := json .Unmarshal (value , & raw ); err != nil {
@@ -32,6 +41,10 @@ func keysOf(t *testing.T, value []byte) string {
3241 if string (raw ["v" ]) != "4" {
3342 t .Errorf ("v = %s, want 4: %s" , raw ["v" ], value )
3443 }
44+ var i struct { T , C string }
45+ if err := json .Unmarshal (raw ["i" ], & i ); err != nil || i .T != "profiles" || i .C != column {
46+ t .Errorf ("i = %s, want {\" t\" :\" profiles\" ,\" c\" :%q}" , raw ["i" ], column )
47+ }
3548 keys := make ([]string , 0 , len (raw ))
3649 for k := range raw {
3750 keys = append (keys , k )
@@ -40,6 +53,19 @@ func keysOf(t *testing.T, value []byte) string {
4053 return strings .Join (keys , "," )
4154}
4255
56+ // stored is each sealed field of an encrypted profile by column.
57+ func stored (e testusers.EncryptedProfile ) map [string ][]byte {
58+ return map [string ][]byte {
59+ "visits" : e .Visits ,
60+ "balance" : e .Balance ,
61+ "active" : e .Active ,
62+ "ratio" : e .Ratio ,
63+ "score" : e .Score ,
64+ "note" : e .Note ,
65+ "name" : e .Name ,
66+ }
67+ }
68+
4369func TestV4FieldsSealToTheirEQLTypesAndOpenBack (t * testing.T ) {
4470 c := deterministicEQLClient (t )
4571 ctx := context .Background ()
@@ -49,22 +75,19 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) {
4975 if err != nil {
5076 t .Fatal (err )
5177 }
78+ want := map [string ]string {
79+ "visits" : "c,i,v" ,
80+ "balance" : "c,i,v" ,
81+ "active" : "c,i,v" ,
82+ "ratio" : "c,i,v" ,
83+ "score" : "c,i,v" ,
84+ "note" : "c,i,v" ,
85+ "name" : "bf,c,i,v" ,
86+ }
5287 for i , e := range encrypted {
53- for name , tc := range map [string ]struct {
54- value []byte
55- keys string
56- }{
57- "age" : {e .Age , "c,hm,i,v" },
58- "visits" : {e .Visits , "c,i,v" },
59- "balance" : {e .Balance , "c,hm,i,v" },
60- "active" : {e .Active , "c,i,v" },
61- "ratio" : {e .Ratio , "c,i,v" },
62- "score" : {e .Score , "c,i,v" },
63- "note" : {e .Note , "c,i,v" },
64- "name" : {e .Name , "bf,c,i,v" },
65- } {
66- if got := keysOf (t , tc .value ); got != tc .keys {
67- t .Errorf ("row %d %s: keys %s, want %s" , i , name , got , tc .keys )
88+ for column , value := range stored (e ) {
89+ if got := keysOf (t , column , value ); got != want [column ] {
90+ t .Errorf ("row %d %s: keys %s, want %s" , i , column , got , want [column ])
6891 }
6992 }
7093 }
@@ -80,26 +103,43 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) {
80103 }
81104}
82105
106+ // bloom is the set of filter positions in a TextMatch value or query.
107+ func bloom (t * testing.T , value []byte ) map [int ]bool {
108+ t .Helper ()
109+ var v struct {
110+ BF []int `json:"bf"`
111+ }
112+ if err := json .Unmarshal (value , & v ); err != nil {
113+ t .Fatal (err )
114+ }
115+ set := make (map [int ]bool , len (v .BF ))
116+ for _ , p := range v .BF {
117+ set [p ] = true
118+ }
119+ return set
120+ }
121+
122+ func subset (a , b map [int ]bool ) bool {
123+ for p := range a {
124+ if ! b [p ] {
125+ return false
126+ }
127+ }
128+ return true
129+ }
130+
83131func TestV4QueriesMatchTheRustFixture (t * testing.T ) {
84132 c := deterministicEQLClient (t )
85133 ctx := context .Background ()
86134 cipher := c .DefaultKeyset ()
87135
88- raw , err := os .ReadFile (filepath . Join ( ".." , ".." , ".." , "packages" , "eql" , "tests" , "encryption" , "fixtures" , "v4_queries.json" ))
136+ raw , err := os .ReadFile (fixturePath ( "v4_queries.json" ))
89137 if err != nil {
90138 t .Fatal (err )
91139 }
92140 var fixture struct {
93141 Table string
94- Age struct {
95- Plaintext int32
96- Query string
97- }
98- Balance struct {
99- Plaintext int64
100- Query string
101- }
102- Name struct {
142+ Name struct {
103143 Plaintext string
104144 Query string
105145 }
@@ -112,44 +152,165 @@ func TestV4QueriesMatchTheRustFixture(t *testing.T) {
112152 }
113153
114154 // Byte for byte what eql-bindings' own dispatch derived under the same
115- // index key: each Go field runs its v4 type's plan and no other.
116- age , err := testusers .ProfileFields .Age .Query (ctx , cipher , fixture .Age .Plaintext )
155+ // index key: the Go field runs its v4 type's plan and no other.
156+ name , err := testusers .ProfileFields .Name .Query (ctx , cipher , fixture .Name .Plaintext )
117157 if err != nil {
118158 t .Fatal (err )
119159 }
120- if string (age ) != fixture .Age .Query {
121- t .Errorf ("Age .Query = %s\n want %s" , age , fixture .Age .Query )
160+ if string (name ) != fixture .Name .Query {
161+ t .Errorf ("Name .Query = %s\n want %s" , name , fixture .Name .Query )
122162 }
123- balance , err := testusers .ProfileFields .Balance .Query (ctx , cipher , fixture .Balance .Plaintext )
163+
164+ // And a stored value holds every filter bit of a query for text it
165+ // contains, which is what `@@` tests, and not those of text it lacks.
166+ encrypted , err := testusers .EncryptProfile (ctx , cipher , profiles [:1 ])
124167 if err != nil {
125168 t .Fatal (err )
126169 }
127- if string (balance ) != fixture .Balance .Query {
128- t .Errorf ("Balance.Query = %s\n want %s" , balance , fixture .Balance .Query )
170+ all := bloom (t , encrypted [0 ].Name )
171+ for needle , contained := range map [string ]bool {"alice" : true , "LIC" : true , "zzz" : false , "bob" : false } {
172+ q , err := testusers .ProfileFields .Name .Query (ctx , cipher , needle )
173+ if err != nil {
174+ t .Fatal (err )
175+ }
176+ if got := subset (bloom (t , q ), all ); got != contained {
177+ t .Errorf ("%q in %q: %v, want %v" , needle , profiles [0 ].Name , got , contained )
178+ }
129179 }
130- name , err := testusers .ProfileFields .Name .Query (ctx , cipher , fixture .Name .Plaintext )
180+ }
181+
182+ // rowsFixture is v4_rows.json: the profiles above as the generated code seals
183+ // them under the deterministic guest and testSeed. The Rust suite opens them
184+ // with stack-kms's DeterministicSource over the same seed, and inserts and
185+ // queries them in PostgreSQL.
186+ type rowsFixture struct {
187+ Comment string `json:"_comment"`
188+ Seed string `json:"seed"`
189+ Table string `json:"table"`
190+ Rows []fixtureRow `json:"rows"`
191+ }
192+
193+ type fixtureRow struct {
194+ ID int64 `json:"id"`
195+ Plaintext fixturePlain `json:"plaintext"`
196+ Stored map [string ]json.RawMessage `json:"stored"`
197+ }
198+
199+ type fixturePlain struct {
200+ Visits int32 `json:"visits"`
201+ Balance int64 `json:"balance"`
202+ Active bool `json:"active"`
203+ Ratio float32 `json:"ratio"`
204+ Score float64 `json:"score"`
205+ Note string `json:"note"`
206+ Name string `json:"name"`
207+ }
208+
209+ // TestV4RowsMatchTheCommittedFixture writes v4_rows.json when
210+ // EQL_UPDATE_FIXTURES is set, and otherwise checks the committed file
211+ // against what the generated code seals today. The deterministic guest
212+ // derives every data key from the seed, so the envelopes and the terms
213+ // (`bf`) are the same every run; the AEAD nonce inside each `c` is random,
214+ // so `c` is not. The test therefore compares everything but `c`, and opens
215+ // the committed `c` values with today's code.
216+ func TestV4RowsMatchTheCommittedFixture (t * testing.T ) {
217+ c := deterministicEQLClient (t )
218+ encrypted , err := testusers .EncryptProfile (context .Background (), c .DefaultKeyset (), profiles )
131219 if err != nil {
132220 t .Fatal (err )
133221 }
134- if string (name ) != fixture .Name .Query {
135- t .Errorf ("Name.Query = %s\n want %s" , name , fixture .Name .Query )
222+ fixture := rowsFixture {
223+ Comment : "Profile rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4Rows, do not edit." ,
224+ Seed : string (testSeed [:]),
225+ Table : "profiles" ,
136226 }
137-
138- // And a stored value carries the term its query probes for.
139- stored , err := testusers .ProfileFields .Age .Encrypt (ctx , cipher , fixture .Age .Plaintext )
227+ for i , e := range encrypted {
228+ p := profiles [i ]
229+ row := fixtureRow {
230+ ID : p .ID ,
231+ Plaintext : fixturePlain {p .Visits , p .Balance , p .Active , p .Ratio , p .Score , p .Note , p .Name },
232+ Stored : map [string ]json.RawMessage {},
233+ }
234+ for column , value := range stored (e ) {
235+ row .Stored [column ] = json .RawMessage (value )
236+ }
237+ fixture .Rows = append (fixture .Rows , row )
238+ }
239+ rendered , err := json .MarshalIndent (fixture , "" , " " )
140240 if err != nil {
141241 t .Fatal (err )
142242 }
143- var s , q struct {
144- HM string `json:"hm"`
243+ rendered = append (rendered , '\n' )
244+ path := fixturePath ("v4_rows.json" )
245+ if os .Getenv ("EQL_UPDATE_FIXTURES" ) != "" {
246+ if err := os .WriteFile (path , rendered , 0o644 ); err != nil {
247+ t .Fatal (err )
248+ }
249+ }
250+ committedBytes , err := os .ReadFile (path )
251+ if err != nil {
252+ t .Fatalf ("%v: run with EQL_UPDATE_FIXTURES=1 to write it" , err )
145253 }
146- if err := json .Unmarshal (stored , & s ); err != nil {
254+ var committed rowsFixture
255+ if err := json .Unmarshal (committedBytes , & committed ); err != nil {
147256 t .Fatal (err )
148257 }
149- if err := json .Unmarshal (age , & q ); err != nil {
258+ if ! bytes .Equal (withoutCiphertexts (t , committed ), withoutCiphertexts (t , fixture )) {
259+ t .Errorf ("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1" , path )
260+ }
261+
262+ // The committed ciphertexts still open, to the committed plaintexts.
263+ sealed := make ([]testusers.EncryptedProfile , len (committed .Rows ))
264+ for i , row := range committed .Rows {
265+ sealed [i ] = testusers.EncryptedProfile {
266+ ID : row .ID ,
267+ Visits : []byte (row .Stored ["visits" ]),
268+ Balance : []byte (row .Stored ["balance" ]),
269+ Active : []byte (row .Stored ["active" ]),
270+ Ratio : []byte (row .Stored ["ratio" ]),
271+ Score : []byte (row .Stored ["score" ]),
272+ Note : []byte (row .Stored ["note" ]),
273+ Name : []byte (row .Stored ["name" ]),
274+ }
275+ }
276+ opened , err := testusers .DecryptProfile (context .Background (), c .DefaultKeyset (), sealed )
277+ if err != nil {
150278 t .Fatal (err )
151279 }
152- if s .HM != q .HM {
153- t .Errorf ("stored hm %s, query hm %s" , s .HM , q .HM )
280+ for i , row := range committed .Rows {
281+ p := opened [i ]
282+ got := fixturePlain {p .Visits , p .Balance , p .Active , p .Ratio , p .Score , p .Note , p .Name }
283+ if got != row .Plaintext {
284+ t .Errorf ("row %d opens to %+v, want %+v" , row .ID , got , row .Plaintext )
285+ }
286+ }
287+ }
288+
289+ // withoutCiphertexts renders a fixture with every stored value's `c`
290+ // removed: the part of the file that is the same on every run.
291+ func withoutCiphertexts (t * testing.T , f rowsFixture ) []byte {
292+ t .Helper ()
293+ out := f
294+ out .Rows = nil
295+ for _ , row := range f .Rows {
296+ stripped := fixtureRow {ID : row .ID , Plaintext : row .Plaintext , Stored : map [string ]json.RawMessage {}}
297+ for column , value := range row .Stored {
298+ var fields map [string ]json.RawMessage
299+ if err := json .Unmarshal (value , & fields ); err != nil {
300+ t .Fatal (err )
301+ }
302+ delete (fields , "c" )
303+ b , err := json .Marshal (fields )
304+ if err != nil {
305+ t .Fatal (err )
306+ }
307+ stripped .Stored [column ] = b
308+ }
309+ out .Rows = append (out .Rows , stripped )
310+ }
311+ b , err := json .Marshal (out )
312+ if err != nil {
313+ t .Fatal (err )
154314 }
315+ return b
155316}
0 commit comments