test: consistently analyze tables exactly once

nix/tools/withTools.nix

@@ -14,6 +14,11 @@ let
     { name, postgresql }:
     let
       commandName = "postgrest-with-${name}";
+      postgresqlConf = writeText "postgresql.conf" "
+        autovacuum = false
+        listen_addresses = ''
+        log_statement = all
+      ";
     in
     checkedShellScript
       {
@@ -72,6 +77,10 @@ let
           TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U postgres --auth=trust \
             >> "$setuplog"
 
+          # Append our own config to the one initdb created to avoid replacing
+          # default values created by the latter.
+          cat ${postgresqlConf} >> "$tmpdir/db/postgresql.conf"
+
           log "Starting the database cluster..."
 
           # Instead of listening on a local port, we will listen on a unix domain socket.
@@ -80,7 +89,7 @@ let
           # On MacOS, it's 104 chars
           # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
 
-          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
+          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c hba_file=$HBA_FILE -k $PGHOST " \
             >> "$setuplog"
 
           log "Creating a minimally privileged $PGUSER connection role..."
@@ -106,7 +115,7 @@ let
             log "Starting replica on $replica_host"
 
             # We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s)
-            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
+            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c hba_file=$HBA_FILE -k $replica_host -c max_standby_streaming_delay=\"3s\" " \
               >> "$setuplog"
 
             >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres"
@@ -135,6 +144,7 @@ let
           load_start=$SECONDS
           >&2 printf "${commandName}: Loading fixtures under the postgres role..."
           psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
+          psql -U postgres -v ON_ERROR_STOP=1 -c "VACUUM ANALYZE;" >> "$setuplog"
           load_end=$((SECONDS - load_start))
           >&2 printf " done in %ss. Running command...\n" "$load_end"
         fi

src/PostgREST/Config/PgVersion.hs

@@ -4,7 +4,6 @@ module PostgREST.Config.PgVersion
   ( PgVersion(..)
   , minimumPgVersion
   , pgVersion150
-  , pgVersion170
   , pgVersion180
   ) where
 
@@ -33,8 +32,5 @@ pgVersion140 = PgVersion 140000 "14.0" "14.0"
 pgVersion150 :: PgVersion
 pgVersion150 = PgVersion 150000 "15.0" "15.0"
 
-pgVersion170 :: PgVersion
-pgVersion170 = PgVersion 170000 "17.0" "17.0"
-
 pgVersion180 :: PgVersion
 pgVersion180 = PgVersion 180000 "18.0" "18.0"

test/io/test_io.py

@@ -938,12 +938,12 @@ def test_log_query(level, defaultenv):
         response = postgrest.session.get(
             "/projects", headers={"Prefer": "count=estimated"}
         )
-        assert response.status_code == 206
+        assert response.status_code == 200
 
         response = postgrest.session.get(
             "/projects", headers={"Prefer": "count=planned"}
         )
-        assert response.status_code == 206
+        assert response.status_code == 200
 
         response = postgrest.session.get("/infinite_recursion")
         assert response.status_code == 500

test/spec/Feature/Query/AggregateFunctionsSpec.hs

@@ -154,24 +154,24 @@ allowed =
             [json|[{"total_budget": 9501.06}]|]
             { matchHeaders = [matchContentTypeJson] }
         it "supports aggregates from a spread relationships grouped by spreaded fields from other relationships" $ do
-          get "/processes?select=...process_costs(cost.sum()),...process_categories(name)" `shouldRespondWith`
+          get "/processes?select=...process_costs(cost.sum()),...process_categories(name)&order=process_categories(name)" `shouldRespondWith`
             [json|[
               {"sum": 400.00, "name": "Batch"},
               {"sum": 350.00, "name": "Mass"}]|]
             { matchHeaders = [matchContentTypeJson] }
-          get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)" `shouldRespondWith`
+          get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)&order=process_categories(category)" `shouldRespondWith`
             [json|[
               {"cost_sum": 400.00, "category": "Batch"},
               {"cost_sum": 350.00, "category": "Mass"}]|]
             { matchHeaders = [matchContentTypeJson] }
         it "supports aggregates on spreaded fields from nested relationships" $ do
-          get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))" `shouldRespondWith`
+          get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith`
             [json|[
               {"factory_id": 3, "sum": 110.00},
               {"factory_id": 2, "sum": 500.00},
               {"factory_id": 1, "sum": 350.00}]|]
             { matchHeaders = [matchContentTypeJson] }
-          get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))" `shouldRespondWith`
+          get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith`
             [json|[
               {"factory_id": 3, "cost_sum": 110.00},
               {"factory_id": 2, "cost_sum": 500.00},

test/spec/Feature/Query/EmbedDisambiguationSpec.hs

@@ -236,7 +236,7 @@ spec =
               [json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"id": 1},"user":{"name": "Angela Martin"}}]|]
 
         it "can specify a view!fk" $
-          get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4" `shouldRespondWith`
+          get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4&order=id" `shouldRespondWith`
             [json|
               [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}},
                {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}},

test/spec/Feature/Query/PlanSpec.hs

@@ -15,12 +15,11 @@ import           Test.Hspec           hiding (pendingWith)
 import           Test.Hspec.Wai
 import           Test.Hspec.Wai.JSON
 
-import PostgREST.Config.PgVersion (PgVersion, pgVersion170)
-import Protolude                  hiding (get)
+import Protolude  hiding (get)
 import SpecHelper
 
-spec :: PgVersion -> SpecWith ((), Application)
-spec actualPgVersion = do
+spec :: SpecWith ((), Application)
+spec = do
   describe "read table/view plan" $ do
     it "outputs the total cost for a single filter on a table" $ do
       r <- request methodGet "/projects?id=in.(1,2,3)"
@@ -34,7 +33,7 @@ spec actualPgVersion = do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resHeaders `shouldSatisfy` notZeroContentLength
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.32 else 15.63)
+        totalCost `shouldBe` 1.11
 
     it "outputs the total cost for a single filter on a view" $ do
       r <- request methodGet "/projects_view?id=gt.2"
@@ -47,7 +46,7 @@ spec actualPgVersion = do
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` 24.28
+        totalCost `shouldBe` 1.1
 
     it "outputs blocks info when using the buffers option" $ do
       r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=buffers") ""
@@ -143,7 +142,7 @@ spec actualPgVersion = do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resHeaders `shouldSatisfy` notZeroContentLength
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` 8.23
+        totalCost `shouldBe` 1.13
 
     it "outputs the total cost for a delete" $ do
       r <- request methodDelete "/projects?id=in.(1,2,3)"
@@ -157,7 +156,7 @@ spec actualPgVersion = do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resHeaders `shouldSatisfy` notZeroContentLength
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.37 else 15.68)
+        totalCost `shouldBe` 1.16
 
     it "outputs the total cost for a single upsert" $ do
       r <- request methodPut "/tiobe_pls?name=eq.Go"
@@ -490,7 +489,7 @@ spec actualPgVersion = do
 
         liftIO $ do
           resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze; charset=utf-8")
-          totalCost `shouldSatisfy` (> 49.0)
+          totalCost `shouldSatisfy` (> 2.0)
           aggregateQty `shouldSatisfy` (> 1)
 
     context "functions with count=exact" $ do

test/spec/Feature/Query/QuerySpec.hs

@@ -692,13 +692,13 @@ spec = do
         { matchHeaders = [matchContentTypeJson] }
 
     it "requesting data using many<->many relation defined by composite keys" $
-      get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)" `shouldRespondWith`
-        [json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"},{"filename":"README.md","content":"# make $$$!"}]}]|]
+      get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)&files.order=filename" `shouldRespondWith`
+        [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"}]}]|]
         { matchHeaders = [matchContentTypeJson] }
 
     it "requesting data using many<->many (composite keys) relation using hint" $
-      get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)" `shouldRespondWith`
-        [json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"},{"filename":"README.md","content":"# make $$$!"}]}]|]
+      get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)&files.order=filename" `shouldRespondWith`
+        [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"}]}]|]
         { matchHeaders = [matchContentTypeJson] }
 
     it "requesting children with composite key" $
@@ -1607,11 +1607,11 @@ spec = do
         ] |]
         { matchHeaders = [matchContentTypeJson] }
     it "formats through join" $
-      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith`
+      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&order=id" `shouldRespondWith`
         [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
         { matchHeaders = [matchContentTypeJson] }
     it "formats through join with star select" $
-      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith`
+      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)&order=id" `shouldRespondWith`
         [json| [
           {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
           {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
@@ -1645,7 +1645,7 @@ spec = do
         ] |]
         { matchHeaders = [matchContentTypeJson] }
     it "uses text parser on value for filter across relations" $
-      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith`
+      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100&order=id" `shouldRespondWith`
         [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
         { matchHeaders = [matchContentTypeJson] }
     -- This is not supported by data reps (would be hard to make it work with high performance). So the test just

test/spec/Feature/Query/RelatedQueriesSpec.hs

@@ -323,7 +323,7 @@ spec = describe "related queries" $ do
         ]|]
         { matchStatus  = 206
         , matchHeaders = [ matchContentTypeJson
-                         , "Content-Range" <:> "0-3/1200" ]
+                         , "Content-Range" <:> "0-3/5" ]
         }
       request methodGet "/projects?select=name,clients()&clients=is.null"
         [("Prefer", "count=planned")] ""
@@ -340,9 +340,9 @@ spec = describe "related queries" $ do
           {"id":1,"name":"Walmart"},
           {"id":2,"name":"Target"}
         ]|]
-        { matchStatus  = 206
+        { matchStatus  = 200
         , matchHeaders = [ matchContentTypeJson
-                         , "Content-Range" <:> "0-1/952" ]
+                         , "Content-Range" <:> "0-1/2" ]
         }
 
     it "works with count=estimated" $ do
@@ -357,7 +357,7 @@ spec = describe "related queries" $ do
         ]|]
         { matchStatus  = 206
         , matchHeaders = [ matchContentTypeJson
-                         , "Content-Range" <:> "0-3/1200" ]
+                         , "Content-Range" <:> "0-3/5" ]
         }
       request methodGet "/projects?select=name,clients()&clients=is.null"
         [("Prefer", "count=estimated")] ""
@@ -374,7 +374,7 @@ spec = describe "related queries" $ do
           {"id":1,"name":"Walmart"},
           {"id":2,"name":"Target"}
         ]|]
-        { matchStatus  = 206
+        { matchStatus  = 200
         , matchHeaders = [ matchContentTypeJson
-                         , "Content-Range" <:> "0-1/952" ]
+                         , "Content-Range" <:> "0-1/2" ]
         }

test/spec/Main.hs

@@ -243,7 +243,7 @@ main = do
 
     -- this test runs with db-plan-enabled = true
     parallel $ before planEnabledApp $
-      describe "Feature.Query.PlanSpec.spec" $ Feature.Query.PlanSpec.spec actualPgVersion
+      describe "Feature.Query.PlanSpec.spec" Feature.Query.PlanSpec.spec
 
     -- this test runs with server-trace-header set
     parallel $ before obsApp $

test/spec/fixtures/data.sql

@@ -983,3 +983,5 @@ VALUES (1, '2025-01-01 10:00','2025-01-01 11:00', 'vacation'),
        (2, '2024-11-01 09:00','2024-11-01 10:00', 'vacation'),
        (3, '2024-12-02 13:00','2024-12-02 14:00', 'vacation'),
        (1, '2023-01-02 20:00','2023-01-01 21:00', 'work');
+
+INSERT INTO bets (id) SELECT generate_series(1,1000);