From 1befafebd570fbc3060b5fe4b5742b5cae9c9579 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 17 May 2026 17:14:29 +0200 Subject: [PATCH 1/2] nix(postgrest-with-pg-...): use postgresql.conf instead of CLI arguments This is easier to reason about and easier to extend. Also removes a tiny bit of repetition. --- nix/tools/withTools.nix | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 094f13b359..f428aec2e0 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -14,6 +14,10 @@ let { name, postgresql }: let commandName = "postgrest-with-${name}"; + postgresqlConf = writeText "postgresql.conf" " + listen_addresses = '' + log_statement = all + "; in checkedShellScript { @@ -72,6 +76,10 @@ let TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U postgres --auth=trust \ >> "$setuplog" + # Append our own config to the one initdb created to avoid replacing + # default values created by the latter. + cat ${postgresqlConf} >> "$tmpdir/db/postgresql.conf" + log "Starting the database cluster..." # Instead of listening on a local port, we will listen on a unix domain socket. @@ -80,7 +88,7 @@ let # On MacOS, it's 104 chars # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket - pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \ + pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c hba_file=$HBA_FILE -k $PGHOST " \ >> "$setuplog" log "Creating a minimally privileged $PGUSER connection role..." @@ -106,7 +114,7 @@ let log "Starting replica on $replica_host" # We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s) - pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \ + pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c hba_file=$HBA_FILE -k $replica_host -c max_standby_streaming_delay=\"3s\" " \ >> "$setuplog" >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres" From 45cd8ddcde25caacc8894a7310525bdbd64fd2b5 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 17 May 2026 17:12:53 +0200 Subject: [PATCH 2/2] test: consistently analyze tables exactly once Disabling the autovacuum daemon should also help reproducibility in theory, although I don't know of any cases where we hit a problem with that. VACUUM changes the order of rows that PostgreSQL returns for some table without explicit ordering, thus doing the latter to make it consistently reproducible. After ANALYZE estimates are 100% exact for the moment, so some requests which returned 206 Partial Response now return 200 instead. The fact that PostgREST returns 206 on an unfiltered endpoint can probably be considered a bug. --- nix/tools/withTools.nix | 2 ++ src/PostgREST/Config/PgVersion.hs | 4 ---- test/io/test_io.py | 4 ++-- .../Feature/Query/AggregateFunctionsSpec.hs | 8 ++++---- .../Feature/Query/EmbedDisambiguationSpec.hs | 2 +- test/spec/Feature/Query/PlanSpec.hs | 17 ++++++++--------- test/spec/Feature/Query/QuerySpec.hs | 14 +++++++------- test/spec/Feature/Query/RelatedQueriesSpec.hs | 12 ++++++------ test/spec/Main.hs | 2 +- test/spec/fixtures/data.sql | 2 ++ 10 files changed, 33 insertions(+), 34 deletions(-) diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index f428aec2e0..44fe246571 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -15,6 +15,7 @@ let let commandName = "postgrest-with-${name}"; postgresqlConf = writeText "postgresql.conf" " + autovacuum = false listen_addresses = '' log_statement = all "; @@ -143,6 +144,7 @@ let load_start=$SECONDS >&2 printf "${commandName}: Loading fixtures under the postgres role..." psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog" + psql -U postgres -v ON_ERROR_STOP=1 -c "VACUUM ANALYZE;" >> "$setuplog" load_end=$((SECONDS - load_start)) >&2 printf " done in %ss. Running command...\n" "$load_end" fi diff --git a/src/PostgREST/Config/PgVersion.hs b/src/PostgREST/Config/PgVersion.hs index 26db29e5c2..5129267f19 100644 --- a/src/PostgREST/Config/PgVersion.hs +++ b/src/PostgREST/Config/PgVersion.hs @@ -4,7 +4,6 @@ module PostgREST.Config.PgVersion ( PgVersion(..) , minimumPgVersion , pgVersion150 - , pgVersion170 , pgVersion180 ) where @@ -33,8 +32,5 @@ pgVersion140 = PgVersion 140000 "14.0" "14.0" pgVersion150 :: PgVersion pgVersion150 = PgVersion 150000 "15.0" "15.0" -pgVersion170 :: PgVersion -pgVersion170 = PgVersion 170000 "17.0" "17.0" - pgVersion180 :: PgVersion pgVersion180 = PgVersion 180000 "18.0" "18.0" diff --git a/test/io/test_io.py b/test/io/test_io.py index 56cdcba8a6..42f0abb0cc 100644 --- a/test/io/test_io.py +++ b/test/io/test_io.py @@ -938,12 +938,12 @@ def test_log_query(level, defaultenv): response = postgrest.session.get( "/projects", headers={"Prefer": "count=estimated"} ) - assert response.status_code == 206 + assert response.status_code == 200 response = postgrest.session.get( "/projects", headers={"Prefer": "count=planned"} ) - assert response.status_code == 206 + assert response.status_code == 200 response = postgrest.session.get("/infinite_recursion") assert response.status_code == 500 diff --git a/test/spec/Feature/Query/AggregateFunctionsSpec.hs b/test/spec/Feature/Query/AggregateFunctionsSpec.hs index 2d0d4d9d98..e0d67bb424 100644 --- a/test/spec/Feature/Query/AggregateFunctionsSpec.hs +++ b/test/spec/Feature/Query/AggregateFunctionsSpec.hs @@ -154,24 +154,24 @@ allowed = [json|[{"total_budget": 9501.06}]|] { matchHeaders = [matchContentTypeJson] } it "supports aggregates from a spread relationships grouped by spreaded fields from other relationships" $ do - get "/processes?select=...process_costs(cost.sum()),...process_categories(name)" `shouldRespondWith` + get "/processes?select=...process_costs(cost.sum()),...process_categories(name)&order=process_categories(name)" `shouldRespondWith` [json|[ {"sum": 400.00, "name": "Batch"}, {"sum": 350.00, "name": "Mass"}]|] { matchHeaders = [matchContentTypeJson] } - get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)" `shouldRespondWith` + get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)&order=process_categories(category)" `shouldRespondWith` [json|[ {"cost_sum": 400.00, "category": "Batch"}, {"cost_sum": 350.00, "category": "Mass"}]|] { matchHeaders = [matchContentTypeJson] } it "supports aggregates on spreaded fields from nested relationships" $ do - get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))" `shouldRespondWith` + get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith` [json|[ {"factory_id": 3, "sum": 110.00}, {"factory_id": 2, "sum": 500.00}, {"factory_id": 1, "sum": 350.00}]|] { matchHeaders = [matchContentTypeJson] } - get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))" `shouldRespondWith` + get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith` [json|[ {"factory_id": 3, "cost_sum": 110.00}, {"factory_id": 2, "cost_sum": 500.00}, diff --git a/test/spec/Feature/Query/EmbedDisambiguationSpec.hs b/test/spec/Feature/Query/EmbedDisambiguationSpec.hs index 48bf081972..2f7193722c 100644 --- a/test/spec/Feature/Query/EmbedDisambiguationSpec.hs +++ b/test/spec/Feature/Query/EmbedDisambiguationSpec.hs @@ -236,7 +236,7 @@ spec = [json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"id": 1},"user":{"name": "Angela Martin"}}]|] it "can specify a view!fk" $ - get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4" `shouldRespondWith` + get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4&order=id" `shouldRespondWith` [json| [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}}, {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}}, diff --git a/test/spec/Feature/Query/PlanSpec.hs b/test/spec/Feature/Query/PlanSpec.hs index ab468b7856..880b76d4ba 100644 --- a/test/spec/Feature/Query/PlanSpec.hs +++ b/test/spec/Feature/Query/PlanSpec.hs @@ -15,12 +15,11 @@ import Test.Hspec hiding (pendingWith) import Test.Hspec.Wai import Test.Hspec.Wai.JSON -import PostgREST.Config.PgVersion (PgVersion, pgVersion170) -import Protolude hiding (get) +import Protolude hiding (get) import SpecHelper -spec :: PgVersion -> SpecWith ((), Application) -spec actualPgVersion = do +spec :: SpecWith ((), Application) +spec = do describe "read table/view plan" $ do it "outputs the total cost for a single filter on a table" $ do r <- request methodGet "/projects?id=in.(1,2,3)" @@ -34,7 +33,7 @@ spec actualPgVersion = do resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` notZeroContentLength resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } - totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.32 else 15.63) + totalCost `shouldBe` 1.11 it "outputs the total cost for a single filter on a view" $ do r <- request methodGet "/projects_view?id=gt.2" @@ -47,7 +46,7 @@ spec actualPgVersion = do liftIO $ do resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } - totalCost `shouldBe` 24.28 + totalCost `shouldBe` 1.1 it "outputs blocks info when using the buffers option" $ do r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=buffers") "" @@ -143,7 +142,7 @@ spec actualPgVersion = do resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` notZeroContentLength resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } - totalCost `shouldBe` 8.23 + totalCost `shouldBe` 1.13 it "outputs the total cost for a delete" $ do r <- request methodDelete "/projects?id=in.(1,2,3)" @@ -157,7 +156,7 @@ spec actualPgVersion = do resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` notZeroContentLength resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } - totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.37 else 15.68) + totalCost `shouldBe` 1.16 it "outputs the total cost for a single upsert" $ do r <- request methodPut "/tiobe_pls?name=eq.Go" @@ -490,7 +489,7 @@ spec actualPgVersion = do liftIO $ do resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze; charset=utf-8") - totalCost `shouldSatisfy` (> 49.0) + totalCost `shouldSatisfy` (> 2.0) aggregateQty `shouldSatisfy` (> 1) context "functions with count=exact" $ do diff --git a/test/spec/Feature/Query/QuerySpec.hs b/test/spec/Feature/Query/QuerySpec.hs index 683d564f52..b35b60d801 100644 --- a/test/spec/Feature/Query/QuerySpec.hs +++ b/test/spec/Feature/Query/QuerySpec.hs @@ -692,13 +692,13 @@ spec = do { matchHeaders = [matchContentTypeJson] } it "requesting data using many<->many relation defined by composite keys" $ - get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)" `shouldRespondWith` - [json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include "},{"filename":"README.md","content":"# make $$$!"}]}]|] + get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)&files.order=filename" `shouldRespondWith` + [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include "}]}]|] { matchHeaders = [matchContentTypeJson] } it "requesting data using many<->many (composite keys) relation using hint" $ - get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)" `shouldRespondWith` - [json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include "},{"filename":"README.md","content":"# make $$$!"}]}]|] + get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)&files.order=filename" `shouldRespondWith` + [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include "}]}]|] { matchHeaders = [matchContentTypeJson] } it "requesting children with composite key" $ @@ -1607,11 +1607,11 @@ spec = do ] |] { matchHeaders = [matchContentTypeJson] } it "formats through join" $ - get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith` + get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&order=id" `shouldRespondWith` [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |] { matchHeaders = [matchContentTypeJson] } it "formats through join with star select" $ - get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith` + get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)&order=id" `shouldRespondWith` [json| [ {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}, {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}} @@ -1645,7 +1645,7 @@ spec = do ] |] { matchHeaders = [matchContentTypeJson] } it "uses text parser on value for filter across relations" $ - get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith` + get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100&order=id" `shouldRespondWith` [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |] { matchHeaders = [matchContentTypeJson] } -- This is not supported by data reps (would be hard to make it work with high performance). So the test just diff --git a/test/spec/Feature/Query/RelatedQueriesSpec.hs b/test/spec/Feature/Query/RelatedQueriesSpec.hs index de0dd8f2d0..c5a7d4e566 100644 --- a/test/spec/Feature/Query/RelatedQueriesSpec.hs +++ b/test/spec/Feature/Query/RelatedQueriesSpec.hs @@ -323,7 +323,7 @@ spec = describe "related queries" $ do ]|] { matchStatus = 206 , matchHeaders = [ matchContentTypeJson - , "Content-Range" <:> "0-3/1200" ] + , "Content-Range" <:> "0-3/5" ] } request methodGet "/projects?select=name,clients()&clients=is.null" [("Prefer", "count=planned")] "" @@ -340,9 +340,9 @@ spec = describe "related queries" $ do {"id":1,"name":"Walmart"}, {"id":2,"name":"Target"} ]|] - { matchStatus = 206 + { matchStatus = 200 , matchHeaders = [ matchContentTypeJson - , "Content-Range" <:> "0-1/952" ] + , "Content-Range" <:> "0-1/2" ] } it "works with count=estimated" $ do @@ -357,7 +357,7 @@ spec = describe "related queries" $ do ]|] { matchStatus = 206 , matchHeaders = [ matchContentTypeJson - , "Content-Range" <:> "0-3/1200" ] + , "Content-Range" <:> "0-3/5" ] } request methodGet "/projects?select=name,clients()&clients=is.null" [("Prefer", "count=estimated")] "" @@ -374,7 +374,7 @@ spec = describe "related queries" $ do {"id":1,"name":"Walmart"}, {"id":2,"name":"Target"} ]|] - { matchStatus = 206 + { matchStatus = 200 , matchHeaders = [ matchContentTypeJson - , "Content-Range" <:> "0-1/952" ] + , "Content-Range" <:> "0-1/2" ] } diff --git a/test/spec/Main.hs b/test/spec/Main.hs index 194aacba4f..e9d75f4b92 100644 --- a/test/spec/Main.hs +++ b/test/spec/Main.hs @@ -243,7 +243,7 @@ main = do -- this test runs with db-plan-enabled = true parallel $ before planEnabledApp $ - describe "Feature.Query.PlanSpec.spec" $ Feature.Query.PlanSpec.spec actualPgVersion + describe "Feature.Query.PlanSpec.spec" Feature.Query.PlanSpec.spec -- this test runs with server-trace-header set parallel $ before obsApp $ diff --git a/test/spec/fixtures/data.sql b/test/spec/fixtures/data.sql index 5b5486a9d8..a1187d1722 100644 --- a/test/spec/fixtures/data.sql +++ b/test/spec/fixtures/data.sql @@ -983,3 +983,5 @@ VALUES (1, '2025-01-01 10:00','2025-01-01 11:00', 'vacation'), (2, '2024-11-01 09:00','2024-11-01 10:00', 'vacation'), (3, '2024-12-02 13:00','2024-12-02 14:00', 'vacation'), (1, '2023-01-02 20:00','2023-01-01 21:00', 'work'); + +INSERT INTO bets (id) SELECT generate_series(1,1000);