Leios: late-join support, rebased

ouroboros-consensus-cardano/src/byron/Ouroboros/Consensus/Byron/Ledger/Block.hs

@@ -56,7 +56,7 @@ import Ouroboros.Consensus.Block
 import qualified Ouroboros.Consensus.Byron.EBBs as EBBs
 import Ouroboros.Consensus.Byron.Ledger.Conversions
 import Ouroboros.Consensus.Byron.Ledger.Orphans ()
-import Ouroboros.Consensus.Storage.LedgerDB (ResolveLeiosBlock)
+import Ouroboros.Consensus.Storage.LedgerDB (IsCertRB (..), ResolveLeiosBlock (..))
 import Ouroboros.Consensus.Util (ShowProxy (..))
 import Ouroboros.Consensus.Util.Condense
 import Ouroboros.Network.SizeInBytes (SizeInBytes)
@@ -105,7 +105,9 @@ instance Condense ByronBlock where
 instance ShowProxy ByronBlock
 
 -- | Default 'ResolveLeiosBlock' — Byron blocks never carry Leios certs.
-instance ResolveLeiosBlock ByronBlock
+instance ResolveLeiosBlock ByronBlock where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
 
 instance NFData ByronBlock where
   rnf ByronBlock{byronBlockRaw, byronBlockSlotNo, byronBlockHash} =

ouroboros-consensus-cardano/src/ouroboros-consensus-cardano/Ouroboros/Consensus/Cardano/Block.hs

@@ -238,7 +238,7 @@ import Ouroboros.Consensus.Protocol.TPraos (TPraos)
 import Ouroboros.Consensus.Shelley.Eras
 import Ouroboros.Consensus.Shelley.Ledger (ShelleyBlock)
 import Ouroboros.Consensus.Shelley.Ledger.Block (ShelleyCompatible)
-import Ouroboros.Consensus.Storage.LedgerDB (ResolveLeiosBlock (..))
+import Ouroboros.Consensus.Storage.LedgerDB (IsCertRB (..), ResolveLeiosBlock (..))
 import Ouroboros.Consensus.TypeFamilyWrappers
 
 {-------------------------------------------------------------------------------
@@ -1544,3 +1544,11 @@ instance
   headerLeiosAnnouncement hdr = case hdr of
     HeaderDijkstra dHdr -> headerLeiosAnnouncement dHdr
     _ -> Nothing
+
+  headerIsCertRB hdr = case hdr of
+    HeaderDijkstra dHdr -> headerIsCertRB dHdr
+    _ -> NotCertRB
+
+  headerEbAnnouncement hdr = case hdr of
+    HeaderDijkstra dHdr -> headerEbAnnouncement dHdr
+    _ -> Nothing

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/HFEras.hs

@@ -40,7 +40,7 @@ import Ouroboros.Consensus.Shelley.Ledger.Protocol ()
 import Ouroboros.Consensus.Shelley.Protocol.Praos ()
 import Ouroboros.Consensus.Shelley.Protocol.TPraos ()
 import Ouroboros.Consensus.Shelley.ShelleyHFC ()
-import Ouroboros.Consensus.Storage.LedgerDB (ResolveLeiosBlock)
+import Ouroboros.Consensus.Storage.LedgerDB (IsCertRB (..), ResolveLeiosBlock (..))
 
 {-------------------------------------------------------------------------------
   Hard fork eras
@@ -115,9 +115,21 @@ instance Praos.PraosCrypto c => ShelleyCompatible (Praos c) DijkstraEra
   lives in "Ouroboros.Consensus.Shelley.Ledger.Ledger".
 -------------------------------------------------------------------------------}
 
-instance ResolveLeiosBlock (ShelleyBlock (TPraos c) ShelleyEra)
-instance ResolveLeiosBlock (ShelleyBlock (TPraos c) AllegraEra)
-instance ResolveLeiosBlock (ShelleyBlock (TPraos c) MaryEra)
-instance ResolveLeiosBlock (ShelleyBlock (TPraos c) AlonzoEra)
-instance ResolveLeiosBlock (ShelleyBlock (Praos c) BabbageEra)
-instance ResolveLeiosBlock (ShelleyBlock (Praos c) ConwayEra)
+instance ResolveLeiosBlock (ShelleyBlock (TPraos c) ShelleyEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
+instance ResolveLeiosBlock (ShelleyBlock (TPraos c) AllegraEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
+instance ResolveLeiosBlock (ShelleyBlock (TPraos c) MaryEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
+instance ResolveLeiosBlock (ShelleyBlock (TPraos c) AlonzoEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
+instance ResolveLeiosBlock (ShelleyBlock (Praos c) BabbageEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
+instance ResolveLeiosBlock (ShelleyBlock (Praos c) ConwayEra) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/Forge.hs

@@ -36,6 +36,7 @@ import LeiosDemoDb
 import LeiosDemoTypes
   ( EbAnnouncement (..)
   , ForgedLeiosEb (..)
+  , IsCertRB (..)
   , LeiosPoint (..)
   , TraceLeiosKernel (..)
   , forgeLeiosEb
@@ -97,6 +98,7 @@ forgeShelleyBlock hotKey cbl ForgeBlockArgs{..} = do
       actualBodySize
       protocolVersion
       mayEbAnn
+      (case mayLeiosCert of SJust _ -> CertRB; SNothing -> NotCertRB)
   let blk = mkShelleyBlock $ SL.Block hdr body
   return $
     assert (verifyBlockIntegrity (configSlotsPerKESPeriod $ configConsensus fbConfig) blk) $
@@ -203,7 +205,7 @@ forgeShelleyBlock hotKey cbl ForgeBlockArgs{..} = do
           , mempoolRestMeasure = ByteSize32 0
           }
       leiosDbInsertEbPoint fbLeiosDb (forgedEb.point) ebSize
-      leiosDbInsertEbBody fbLeiosDb (forgedEb.point) (forgedEb.body)
+      void $ leiosDbInsertEbBody fbLeiosDb (forgedEb.point) (forgedEb.body)
       void $ leiosDbInsertTxs fbLeiosDb (forgedEb.txClosure)
       traceWith fbLeiosTracer $
         TraceLeiosBlockStored

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/Ledger.hs

@@ -142,7 +142,7 @@ import Ouroboros.Consensus.Protocol.Ledger.Util (isNewEpoch)
 import Ouroboros.Consensus.Protocol.Praos (Praos, PraosState (..))
 import Ouroboros.Consensus.Protocol.Praos.Header
   ( Header (Header, headerBody)
-  , HeaderBody (hbLeiosEbAnnouncement, hbSlotNo)
+  , HeaderBody (hbIsCertRB, hbLeiosEbAnnouncement, hbSlotNo)
   )
 import Ouroboros.Consensus.Shelley.Ledger.Block
 import Ouroboros.Consensus.Shelley.Ledger.Config
@@ -1045,6 +1045,12 @@ instance
     annBody :: HeaderBody c
     Header{headerBody = annBody} = shelleyHeaderRaw hdr
 
+  headerIsCertRB hdr =
+    hbIsCertRB (headerBody (shelleyHeaderRaw hdr))
+
+  headerEbAnnouncement hdr =
+    strictMaybeToMaybe $ hbLeiosEbAnnouncement (headerBody (shelleyHeaderRaw hdr))
+
 -- | Deserialise a transaction supplied as Leios-stored bytes.
 deserialiseLeiosTx :: forall era. ShelleyBasedEra era => BS.ByteString -> Tx TopTx era
 deserialiseLeiosTx bs =

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Protocol/Abstract.hs

@@ -50,7 +50,7 @@ import Data.Kind (Type)
 import Data.Typeable (Typeable)
 import Data.Word (Word64)
 import GHC.Generics (Generic)
-import LeiosDemoTypes (EbAnnouncement)
+import LeiosDemoTypes (EbAnnouncement, IsCertRB)
 import NoThunks.Class (NoThunks)
 import Numeric.Natural (Natural)
 import Ouroboros.Consensus.Protocol.Abstract
@@ -169,6 +169,8 @@ class ProtocolHeaderSupportsKES proto where
     ProtVer ->
     -- | Optional Leios EB announcement. Only used by Praos.
     Maybe EbAnnouncement ->
+    -- | Whether this RB certifies a previously-announced EB
+    IsCertRB ->
     m (ShelleyProtocolHeader proto)
 
   -- | Extract the most recently announced (and not yet certified) Leios EB

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Protocol/Praos.hs

@@ -145,7 +145,7 @@ instance PraosCrypto c => ProtocolHeaderSupportsKES (Praos c) where
           currentKesPeriod - startOfKesPeriod
       | otherwise =
           0
-  mkHeader hk cbl il slotNo blockNo prevHash bbHash sz protVer mayEbAnn = do
+  mkHeader hk cbl il slotNo blockNo prevHash bbHash sz protVer mayEbAnn isCertRB = do
     PraosFields{praosSignature, praosToSign} <- forgePraosFields hk cbl il mkBhBodyBytes
     pure $ Header praosToSign praosSignature
    where
@@ -168,6 +168,7 @@ instance PraosCrypto c => ProtocolHeaderSupportsKES (Praos c) where
           , hbOCert = praosToSignOCert
           , hbProtVer = protVer
           , hbLeiosEbAnnouncement = maybe SNothing SJust mayEbAnn
+          , hbIsCertRB = isCertRB
           }
 
   protocolStateLeiosInfo _ cs =
@@ -187,6 +188,7 @@ instance PraosCrypto c => ProtocolHeaderSupportsProtocol (Praos c) where
       , hvSlotNo = hbSlotNo headerBody
       , hvSigned = headerBody
       , hvSignature = headerSig
+      , hvIsCertRB = hbIsCertRB headerBody
       }
   pHeaderIssuer = hbVk . headerBody
   pHeaderIssueNo = SL.ocertN . hbOCert . headerBody

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Protocol/TPraos.hs

@@ -90,7 +90,7 @@ instance PraosCrypto c => ProtocolHeaderSupportsKES (TPraos c) where
           currentKesPeriod - startOfKesPeriod
       | otherwise =
           0
-  mkHeader hotKey canBeLeader isLeader curSlot curNo prevHash bbHash actualBodySize protVer _mayEbAnn = do
+  mkHeader hotKey canBeLeader isLeader curSlot curNo prevHash bbHash actualBodySize protVer _mayEbAnn _isCertRB = do
     TPraosFields{tpraosSignature, tpraosToSign} <-
       forgeTPraosFields hotKey canBeLeader isLeader mkBhBody
     pure $ SL.BHeader tpraosToSign tpraosSignature

ouroboros-consensus-cardano/src/unstable-cardano-testlib/Test/ThreadNet/Infra/ShelleyBasedHardFork.hs

@@ -120,7 +120,9 @@ type ShelleyBasedHardForkBlock proto1 era1 proto2 era2 =
 -- Real-era resolution (the Dijkstra splice) lives in
 -- "Ouroboros.Consensus.Cardano.Block"; these test stacks never enter
 -- Dijkstra.
-instance ResolveLeiosBlock (ShelleyBasedHardForkBlock proto1 era1 proto2 era2)
+instance ResolveLeiosBlock (ShelleyBasedHardForkBlock proto1 era1 proto2 era2) where
+  headerIsCertRB _ = NotCertRB
+  headerEbAnnouncement _ = Nothing
 
 {-------------------------------------------------------------------------------
   Pattern synonyms, for encapsulation and legibility

ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBImmutaliser/Run.hs

@@ -52,6 +52,7 @@ import Ouroboros.Consensus.Storage.ImmutableDB
   , tipToPoint
   )
 import qualified Ouroboros.Consensus.Storage.ImmutableDB as ImmutableDB
+import Ouroboros.Consensus.Storage.LedgerDB (ResolveLeiosBlock)
 import Ouroboros.Consensus.Storage.VolatileDB
   ( VolatileDB
   , VolatileDbArgs (..)
@@ -108,6 +109,7 @@ withDBs ::
   , ImmutableDB.ImmutableDbSerialiseConstraints blk
   , VolatileDB.VolatileDbSerialiseConstraints blk
   , NodeInitStorage blk
+  , ResolveLeiosBlock blk
   ) =>
   TopLevelConfig blk ->
   ResourceRegistry m ->

ouroboros-consensus-cardano/src/unstable-shelley-testlib/Test/Consensus/Shelley/Examples.hs

@@ -37,6 +37,7 @@ import Data.List.NonEmpty (NonEmpty ((:|)))
 import qualified Data.Map as Map
 import Data.Maybe.Strict (StrictMaybe (..))
 import qualified Data.Set as Set
+import LeiosDemoTypes (IsCertRB (..))
 import Lens.Micro
 import Ouroboros.Consensus.Block
 import Ouroboros.Consensus.HeaderValidation
@@ -287,6 +288,7 @@ fromShelleyLedgerExamplesPraos
           , hbOCert = SL.bheaderOCert bhBody
           , hbProtVer = SL.bprotver bhBody
           , hbLeiosEbAnnouncement = SNothing
+          , hbIsCertRB = NotCertRB
           }
       hSig = coerce bhSig
     hash = ShelleyHash $ SL.unHashHeader pleHashHeader

ouroboros-consensus-cardano/src/unstable-shelley-testlib/Test/Consensus/Shelley/Generators.hs

@@ -21,6 +21,7 @@ import Cardano.Slotting.EpochInfo
 import Control.Monad (replicateM)
 import Data.Coerce (coerce)
 import Data.Maybe.Strict (StrictMaybe (..))
+import LeiosDemoTypes (IsCertRB (..))
 import Ouroboros.Consensus.Block
 import Ouroboros.Consensus.HeaderValidation
 import Ouroboros.Consensus.Ledger.Abstract
@@ -139,6 +140,7 @@ instance
           , Praos.hbOCert = SL.bheaderOCert bhBody
           , Praos.hbProtVer = SL.bprotver bhBody
           , Praos.hbLeiosEbAnnouncement = SNothing
+          , Praos.hbIsCertRB = NotCertRB
           }
       hSig = coerce bhSig
 

ouroboros-consensus-cardano/test/cardano-test/Test/ThreadNet/Leios.hs

@@ -107,8 +107,10 @@ import Test.Consensus.Cardano.ProtocolInfo (Era (Dijkstra), hardForkInto)
 import Test.QuickCheck
   ( Property
   , Testable
+  , choose
   , conjoin
   , counterexample
+  , forAll
   , tabulate
   , (.&&.)
   , (.||.)
@@ -144,7 +146,7 @@ import Test.ThreadNet.Network
   , TraceThreadNetNode (FromLeios, FromLeiosPeer, FromMempool)
   )
 import Test.ThreadNet.TxGen.Cardano (CardanoTxGenExtra (..))
-import Test.ThreadNet.Util.NodeJoinPlan (trivialNodeJoinPlan)
+import Test.ThreadNet.Util.NodeJoinPlan (NodeJoinPlan (..), trivialNodeJoinPlan)
 import Test.ThreadNet.Util.NodeRestarts (noRestarts)
 import Test.ThreadNet.Util.NodeToNodeVersion (newestVersion)
 import Test.ThreadNet.Util.NodeTopology (meshNodeTopology)
@@ -159,6 +161,8 @@ tests =
     "Leios ThreadNet"
     [ adjustQuickCheckTests (`div` 10) $
         testProperty "basic functionality" prop_leios
+    , adjustQuickCheckTests (`div` 10) $
+        testProperty "late join" prop_leios_late_join
     ]
 
 -- | Verify a suite of basic Leios ThreadNet invariants in a single run:
@@ -190,7 +194,9 @@ prop_leios seed =
   numSlots = 200 :: Word64
 
   (testOutput, ProtocolInfo{pInfoConfig, pInfoInitLedger}) =
-    runThreadNet seed (NumSlots numSlots) (NumCoreNodes $ fromIntegral numNodes)
+    runThreadNet seed (NumSlots numSlots) numCoreNodes (trivialNodeJoinPlan numCoreNodes)
+
+  numCoreNodes = NumCoreNodes $ fromIntegral numNodes
 
   traces = testOutput.allTraces
 
@@ -343,6 +349,54 @@ prop_leios seed =
       | (s1, s2) <- zip certifyingBlocks (drop 1 certifyingBlocks)
       ]
 
+-- | A late-joining node must catch up and converge with its peers.
+--
+-- 4 nodes, 200 slots. Nodes 0–2 join at slot 0; node 3 joins at a random
+-- slot, after some CertRBs may already have been produced.  With the
+-- late-join machinery in place the node must both (a) not crash in
+-- 'resolveLeiosBlock' on a CertRB whose certified EB closure it never
+-- observed live, and (b) fetch the missing closures and end up on the
+-- same chain as everyone else.
+prop_leios_late_join :: Seed -> Property
+prop_leios_late_join seed =
+  -- Cap the join slot at numSlots/4 so the late node always has at least
+  -- 3/4 of the run to catch up. Samples near numSlots would otherwise fail
+  -- the chain-consistency assertion for catch-up-bandwidth reasons
+  -- unrelated to the late-join logic under test.
+  forAll (choose (1, fromIntegral numSlots `div` 4)) $ \lateJoinSlot ->
+    let
+      joinPlan =
+        NodeJoinPlan $
+          Map.fromList
+            [ (CoreNodeId 0, SlotNo 0)
+            , (CoreNodeId 1, SlotNo 0)
+            , (CoreNodeId 2, SlotNo 0)
+            , (CoreNodeId 3, SlotNo $ fromIntegral (lateJoinSlot :: Int))
+            ]
+
+      numCoreNodes = NumCoreNodes 4
+
+      (testOutput, _) =
+        runThreadNet seed (NumSlots numSlots) numCoreNodes joinPlan
+
+      nodeChains =
+        Chain.toOldestFirst . nodeOutputFinalChain <$> testOutput.testOutputNodes
+     in
+      conjoin
+        [ not (null nodeChains)
+            & counterexample "test output was empty"
+        , -- ChainSel hides a CertRB whose EB closure is missing instead of
+          -- crashing; the EB-completion re-trigger ('ebCompletionRunner')
+          -- plus the late-join fetch then let the node select it, so all
+          -- nodes converge on the same chain.
+          all (== head (Map.elems nodeChains)) nodeChains
+            & counterexample "nodes have different chains"
+            & counterexample ("chain lengths: " <> show (fmap length nodeChains))
+        ]
+        & counterexample ("late join slot: " <> show lateJoinSlot)
+ where
+  numSlots = 200 :: Word64
+
 -- | Independently compute cumulative tx bytes by resolving each block in the
 -- chain (filling in EB closures from the LeiosDB) and summing individual
 -- 'sizeTxF' values per transaction.
@@ -427,8 +481,9 @@ runThreadNet ::
   Seed ->
   NumSlots ->
   NumCoreNodes ->
+  NodeJoinPlan ->
   (TestOutput (CardanoBlock StandardCrypto), ProtocolInfo (CardanoBlock StandardCrypto))
-runThreadNet initSeed numSlots numCoreNodes =
+runThreadNet initSeed numSlots numCoreNodes joinPlan =
   ( runTestNetwork
       testConfig
       testConfigB
@@ -514,7 +569,7 @@ runThreadNet initSeed numSlots numCoreNodes =
       { forgeEbbEnv = Nothing
       , future = EraFinal slotLength shelleyGenesis.sgEpochLength
       , messageDelay = noCalcMessageDelay
-      , nodeJoinPlan = trivialNodeJoinPlan numCoreNodes
+      , nodeJoinPlan = joinPlan
       , nodeRestarts = noRestarts
       , txGenExtra =
           CardanoTxGenExtra