From a3bc057213f6c074bc3f9ce89cbde385127ec828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C5=82eczek?= Date: Tue, 10 Feb 2026 13:37:53 +0100 Subject: [PATCH] fix: Add random delay in Listener.cacheReloader Triggering schema cache reload immediately upon receival of notification by the listener leads to thundering herd problem in PostgREST cluster. This change adds random delay between 0 and 1 seconds between notification retrieval and schema cache reload triggering. --- postgrest.cabal | 1 + src/PostgREST/Listener.hs | 9 ++++++++- test/io/postgrest.py | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/postgrest.cabal b/postgrest.cabal index fd5f3ec28b..8e1c4e2a76 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -158,6 +158,7 @@ library , stm-hamt >= 1.2 && < 2 , focus >= 1.0 && < 2 , some >= 1.0.4.1 && < 2 + , MonadRandom >= 0.6.2 && < 0.7 -- -fno-spec-constr may help keep compile time memory use in check, -- see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304 -- -optP-Wno-nonportable-include-path diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs index e801e1d69d..82d46e5a43 100644 --- a/src/PostgREST/Listener.hs +++ b/src/PostgREST/Listener.hs @@ -18,6 +18,7 @@ import qualified PostgREST.AppState as AppState import qualified PostgREST.Config as Config import Control.Arrow ((&&&)) +import Control.Monad.Random import Data.Bitraversable (bisequence) import Data.Either.Combinators (whenRight) import qualified Database.PostgreSQL.LibPQ as LibPQ @@ -102,7 +103,13 @@ retryingListen appState = do | msg == "reload config" -> observer (DBListenerGotConfigMsg channel) >> AppState.readInDbConfig False appState | otherwise -> pure () -- Do nothing if anything else than an empty message is sent - cacheReloader = + -- add a random delay between 0 and 1 second to avoid thundering herd problem + -- if multiple listeners receive the same notification at the same time + -- the cacheReloader is launched in a separate thread to avoid blocking the listener + -- during the random delay + cacheReloader = void $ forkIO $ do + delay <- getRandomR (0, 1000000) + threadDelay delay AppState.schemaCacheLoader appState releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release diff --git a/test/io/postgrest.py b/test/io/postgrest.py index 74f40ac75d..a859d03d09 100644 --- a/test/io/postgrest.py +++ b/test/io/postgrest.py @@ -18,7 +18,7 @@ def sleep_until_postgrest_scache_reload(): "Sleep until schema cache reload" - time.sleep(0.3) + time.sleep(1.3) def sleep_until_postgrest_config_reload(): @@ -28,7 +28,7 @@ def sleep_until_postgrest_config_reload(): def sleep_until_postgrest_full_reload(): "Sleep until schema cache plus config reload" - time.sleep(0.3) + time.sleep(1.3) class PostgrestTimedOut(Exception): @@ -71,7 +71,7 @@ def read_stdout(self, nlines=1): time.sleep(0.1) return output - def wait_until_scache_starts_loading(self, max_seconds=1): + def wait_until_scache_starts_loading(self, max_seconds=2): "Wait for the admin /ready return a status of 503" wait_until_status_code(