Skip to content

Commit 5928807

Browse files
Jimbo4350Jimbo4350
authored
Merge pull request #6620 from lambdasistemi/fix/sigterm-startup-shutdown
fix(node): handle SIGTERM across startup phases
2 parents 76d85a2 + e5c4fb0 commit 5928807

5 files changed

Lines changed: 170 additions & 26 deletions

File tree

‎cardano-node/cardano-node.cabal‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ test-suite cardano-node-test
249249
Test.Cardano.Node.Gen
250250
Test.Cardano.Node.Json
251251
Test.Cardano.Node.POM
252+
Test.Cardano.Node.TopLevel
252253
Test.Cardano.Tracing.NewTracing.Consistency
253254

254255
ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T"

‎cardano-node/src/Cardano/Node/Handlers/TopLevel.hs‎

Lines changed: 75 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
1+
{-# LANGUAGE CPP #-}
12
{-# LANGUAGE TypeApplications #-}
23

4+
#if !defined(mingw32_HOST_OS)
5+
#define UNIX
6+
#endif
7+
38
module Cardano.Node.Handlers.TopLevel
4-
( toplevelExceptionHandler
9+
( SigTermException (..)
10+
, SigTermPhase (..)
11+
, installSigTermHandler
12+
, throwSigTerm
13+
, toplevelExceptionHandler
514
) where
615

716
-- The code in this module derives from multiple authors over many years.
@@ -50,11 +59,69 @@ import qualified Ouroboros.Network.Diffusion.Types as Network
5059

5160
import Prelude
5261

62+
import Control.Concurrent
63+
( ThreadId
64+
#ifdef UNIX
65+
, mkWeakThreadId
66+
, myThreadId
67+
#endif
68+
)
5369
import Control.Exception
70+
#ifdef UNIX
71+
import Control.Monad (forM_, void)
72+
#endif
5473
import Control.Monad.Class.MonadAsync (ExceptionInLinkedThread (..))
74+
#ifdef UNIX
75+
import GHC.Weak (deRefWeak)
76+
#endif
5577
import System.Environment
5678
import System.Exit
5779
import System.IO
80+
#ifdef UNIX
81+
import qualified System.Posix.Signals as Signals
82+
#endif
83+
84+
-- | Internal async exception used to route SIGTERM through the top-level
85+
-- handler without letting ordinary exception handlers catch it.
86+
data SigTermException = SigTermException
87+
deriving Show
88+
89+
instance Exception SigTermException where
90+
toException = asyncExceptionToException
91+
fromException = asyncExceptionFromException
92+
93+
-- | Selects the exception used to terminate the node. Startup needs an async
94+
-- exception so configuration parsers cannot catch it. The diffusion layer
95+
-- recognises 'ExitCode' as an expected shutdown once startup is complete.
96+
data SigTermPhase
97+
= SigTermDuringStartup
98+
| SigTermDuringRuntime
99+
100+
-- | Throw the SIGTERM exception appropriate for the current node phase.
101+
throwSigTerm :: SigTermPhase -> ThreadId -> IO ()
102+
throwSigTerm phase threadId =
103+
case phase of
104+
SigTermDuringStartup -> throwTo threadId SigTermException
105+
SigTermDuringRuntime -> throwTo threadId ExitSuccess
106+
107+
-- | Ensure that SIGTERM throws an async exception to the main node thread.
108+
installSigTermHandler :: SigTermPhase -> IO ()
109+
#ifdef UNIX
110+
installSigTermHandler phase = do
111+
-- Similar implementation to the RTS's handling of SIGINT (see GHC's
112+
-- https://gitlab.haskell.org/ghc/ghc/-/blob/master/libraries/base/GHC/TopHandler.hs).
113+
runThreadIdWk <- mkWeakThreadId =<< myThreadId
114+
void $ Signals.installHandler
115+
Signals.sigTERM
116+
(Signals.CatchOnce $ do
117+
runThreadIdMay <- deRefWeak runThreadIdWk
118+
forM_ runThreadIdMay $ \runThreadId ->
119+
throwSigTerm phase runThreadId
120+
)
121+
Nothing
122+
#else
123+
installSigTermHandler _ = pure ()
124+
#endif
58125

59126
-- | An exception handler to use for a program top level, as an alternative to
60127
-- the default top level handler provided by GHC.
@@ -84,10 +151,14 @@ toplevelExceptionHandler prog = do
84151
rethrowAsyncExceptions :: SomeAsyncException -> IO a
85152
rethrowAsyncExceptions full@(SomeAsyncException e) =
86153
case fromException (toException e) of
87-
Just (ExceptionInLinkedThread _ eInner)
88-
| Just ExitSuccess <- fromException eInner
154+
Just SigTermException
89155
-> throwIO ExitSuccess
90-
_ -> throwIO full
156+
Nothing ->
157+
case fromException (toException e) of
158+
Just (ExceptionInLinkedThread _ eInner)
159+
| Just ExitSuccess <- fromException eInner
160+
-> throwIO ExitSuccess
161+
_ -> throwIO full
91162

92163
-- We don't want to print ExitCode, and it should be handled by the default
93164
-- top handler because that sets the actual OS process exit code.

‎cardano-node/src/Cardano/Node/Run.hs‎

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import Cardano.Node.Configuration.Socket (LocalSocketOrSocketInfo,
4141
import Cardano.Node.Configuration.TopologyP2P
4242
import qualified Cardano.Node.Configuration.TopologyP2P as TopologyP2P
4343
import Cardano.Node.Handlers.Shutdown
44+
import Cardano.Node.Handlers.TopLevel (SigTermPhase (..), installSigTermHandler)
4445
import Cardano.Node.Protocol (ProtocolInstantiationError (..), mkConsensusProtocol)
4546
import Cardano.Node.Protocol.Byron (ByronProtocolInstantiationError (CredentialsError))
4647
import Cardano.Node.Protocol.Cardano (CardanoProtocolInstantiationError (..))
@@ -58,7 +59,7 @@ import Cardano.Node.Tracing.StateRep (NodeState (NodeKernelOnline))
5859
import Cardano.Node.Tracing.Tracers.NodeVersion (getNodeVersion)
5960
import Cardano.Node.Tracing.Tracers.Startup (getStartupInfo)
6061
import Cardano.Node.Types
61-
import Cardano.Prelude (ExitCode (..), FatalError (..), bool, (:~:) (..))
62+
import Cardano.Prelude (FatalError (..), bool, (:~:) (..))
6263
import Cardano.Slotting.Slot (WithOrigin (..))
6364
import Cardano.Logging.Types (LogFormatting)
6465
import Cardano.Logging.Utils (showT)
@@ -122,7 +123,7 @@ import Ouroboros.Network.PeerSelection.State.LocalRootPeers (HotValenc
122123
import Ouroboros.Network.Protocol.ChainSync.Codec
123124

124125
import Control.Applicative (empty)
125-
import Control.Concurrent (killThread, mkWeakThreadId, myThreadId, getNumCapabilities)
126+
import Control.Concurrent (killThread, getNumCapabilities)
126127
import Control.Concurrent.Async
127128
import Control.Concurrent.Class.MonadSTM.Strict
128129
import Control.Exception (try, Exception, IOException)
@@ -157,7 +158,6 @@ import System.Directory (canonicalizePath, createDirectoryIfMissing, m
157158
import System.FilePath (takeDirectory, (</>))
158159
import System.IO (hPutStrLn)
159160
#ifdef UNIX
160-
import GHC.Weak (deRefWeak)
161161
import System.Posix.Files
162162
import qualified System.Posix.Signals as Signals
163163
import System.Posix.Types (FileMode)
@@ -175,7 +175,7 @@ runNode
175175
:: PartialNodeConfiguration
176176
-> IO ()
177177
runNode cmdPc = do
178-
installSigTermHandler
178+
installSigTermHandler SigTermDuringStartup
179179

180180
Crypto.cryptoInit
181181

@@ -214,24 +214,6 @@ buildNodeConfiguration partialConf = do
214214
pure
215215
$ makeNodeConfiguration (defaultPartialNodeConfiguration <> configYamlPc <> partialConf)
216216

217-
-- | Workaround to ensure that the main thread throws an async exception on
218-
-- receiving a SIGTERM signal.
219-
installSigTermHandler :: IO ()
220-
installSigTermHandler = do
221-
#ifdef UNIX
222-
-- Similar implementation to the RTS's handling of SIGINT (see GHC's
223-
-- https://gitlab.haskell.org/ghc/ghc/-/blob/master/libraries/base/GHC/TopHandler.hs).
224-
runThreadIdWk <- mkWeakThreadId =<< myThreadId
225-
_ <- Signals.installHandler
226-
Signals.sigTERM
227-
(Signals.CatchOnce $ do
228-
runThreadIdMay <- deRefWeak runThreadIdWk
229-
forM_ runThreadIdMay $ \runThreadId -> Exception.throwTo runThreadId ExitSuccess
230-
)
231-
Nothing
232-
#endif
233-
return ()
234-
235217
handleNodeWithTracers
236218
:: PartialNodeConfiguration
237219
-> NodeConfiguration
@@ -449,6 +431,7 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do
449431
#endif
450432
nForkPolicy <- getForkPolicy $ ncResponderCoreAffinityPolicy nc
451433
cForkPolicy <- getForkPolicy $ ncResponderCoreAffinityPolicy nc
434+
installSigTermHandler SigTermDuringRuntime
452435
void $
453436
let diffusionNodeArguments :: Cardano.Diffusion.CardanoNodeArguments IO
454437
diffusionNodeArguments = Cardano.Diffusion.CardanoNodeArguments {

‎cardano-node/test/Test/Cardano/Node/TopLevel.hs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
{-# LANGUAGE CPP #-}
2+
{-# LANGUAGE TemplateHaskell #-}
3+
{-# LANGUAGE TypeApplications #-}
4+
5+
#if !defined(mingw32_HOST_OS)
6+
#define UNIX
7+
#endif
8+
9+
module Test.Cardano.Node.TopLevel
10+
( tests
11+
) where
12+
13+
import Cardano.Node.Handlers.TopLevel
14+
15+
import Control.Concurrent (myThreadId)
16+
#ifdef UNIX
17+
import qualified Control.Concurrent as Concurrent
18+
#endif
19+
import Control.Exception
20+
import System.Exit
21+
#ifdef UNIX
22+
import qualified System.Posix.Signals as Signals
23+
#endif
24+
25+
import Hedgehog (Property, discover, (===))
26+
import qualified Hedgehog
27+
import Hedgehog.Internal.Property (failWith)
28+
29+
prop_sigTermExceptionIsAsync :: Property
30+
prop_sigTermExceptionIsAsync =
31+
Hedgehog.property $
32+
case fromException @SomeAsyncException $ toException SigTermException of
33+
Just{} -> Hedgehog.success
34+
Nothing ->
35+
failWith Nothing "SigTermException should be a SomeAsyncException"
36+
37+
prop_topLevelSigTermExitsSuccessfully :: Property
38+
prop_topLevelSigTermExitsSuccessfully =
39+
Hedgehog.property $ do
40+
result <-
41+
Hedgehog.evalIO $
42+
try @ExitCode $
43+
toplevelExceptionHandler $
44+
throwIO SigTermException
45+
result === (Left ExitSuccess :: Either ExitCode ())
46+
47+
prop_sigTermDuringStartupIsAsync :: Property
48+
prop_sigTermDuringStartupIsAsync =
49+
Hedgehog.property $ do
50+
result <-
51+
Hedgehog.evalIO $
52+
try @SomeAsyncException $
53+
myThreadId >>= throwSigTerm SigTermDuringStartup
54+
case result of
55+
Left{} -> Hedgehog.success
56+
Right{} ->
57+
failWith Nothing "startup SIGTERM should throw an async exception"
58+
59+
prop_sigTermDuringRuntimeExitsSuccessfully :: Property
60+
prop_sigTermDuringRuntimeExitsSuccessfully =
61+
Hedgehog.property $ do
62+
result <-
63+
Hedgehog.evalIO $
64+
try @ExitCode $
65+
myThreadId >>= throwSigTerm SigTermDuringRuntime
66+
result === (Left ExitSuccess :: Either ExitCode ())
67+
68+
prop_installedSigTermHandlerExitsSuccessfully :: Property
69+
#ifdef UNIX
70+
prop_installedSigTermHandlerExitsSuccessfully =
71+
Hedgehog.property $ do
72+
result <-
73+
Hedgehog.evalIO $
74+
try @ExitCode $
75+
toplevelExceptionHandler $ do
76+
installSigTermHandler SigTermDuringStartup
77+
Signals.raiseSignal Signals.sigTERM
78+
Concurrent.threadDelay 1000000
79+
result === (Left ExitSuccess :: Either ExitCode ())
80+
#else
81+
prop_installedSigTermHandlerExitsSuccessfully =
82+
Hedgehog.property Hedgehog.success
83+
#endif
84+
85+
tests :: IO Bool
86+
tests =
87+
Hedgehog.checkParallel $$discover

‎cardano-node/test/cardano-node-test.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import qualified Test.Cardano.Node.FilePermissions
1313
#endif
1414
import qualified Test.Cardano.Node.Json
1515
import qualified Test.Cardano.Node.POM
16+
import qualified Test.Cardano.Node.TopLevel
1617
import qualified Test.Cardano.Tracing.NewTracing.Consistency
1718

1819
import qualified Cardano.Crypto.Init as Crypto
@@ -33,5 +34,6 @@ main = do
3334
[ Test.Cardano.Config.Mainnet.tests
3435
, Test.Cardano.Node.Json.tests
3536
, Test.Cardano.Node.POM.tests
37+
, Test.Cardano.Node.TopLevel.tests
3638
, Test.Cardano.Tracing.NewTracing.Consistency.tests
3739
]

0 commit comments

Comments
 (0)