Add leios-late-ib-inclusion to the Haskell simulator (#413)

* Support leios-late-ib-inclusion
* Update SimulatorModel.md

Author: nfrisby

data/simulation/config.d.ts

@@ -39,10 +39,10 @@ export interface Config {
    * Only supported by Haskell simulation. */
   "leios-vote-send-recv-stages": boolean;
   /**
-   * Extends Leios so that EB producers include IBs directly from previous pipelines
-   * where no certified EB was observed.
-   * 
-   * Only supported by Rust simulation. */
+   * Extends Leios so that EB producers include IBs directly from previous pipelines.
+   * Due to casuality, the EB must always include them, even if those IBs end up being
+   * certified in their own pipeline.
+   */
   "leios-late-ib-inclusion": boolean;
   /**
    * The expected time it takes a header to fully diffuse across the network.

data/simulation/variants/config.full.yaml

@@ -4,7 +4,7 @@ leios-variant: full
 # "Random" is meant to ensure that different producers include different TXs.
 leios-mempool-sampling-strategy: random
 
-# Allow EBs to include IBs from previous slots
+# Allow EBs to include IBs from previous pipelines
 leios-late-ib-inclusion: true
 
 # Chain quality controls how far back EB recursion can reach.

leios-trace-hs/src/LeiosConfig.hs

@@ -99,6 +99,7 @@ data Config = Config
   , leiosStageActiveVotingSlots :: Word
   , leiosVoteSendRecvStages :: Bool
   , leiosVariant :: LeiosVariant
+  , leiosLateIbInclusion :: Bool
   , leiosHeaderDiffusionTimeMs :: DurationMs
   , praosChainQuality :: Double
   , txGenerationDistribution :: Distribution
@@ -171,6 +172,7 @@ instance Default Config where
       , leiosStageActiveVotingSlots = 1
       , leiosVoteSendRecvStages = False
       , leiosVariant = Short
+      , leiosLateIbInclusion = True
       , leiosHeaderDiffusionTimeMs = 1000
       , praosChainQuality = 40
       , txGenerationDistribution = Exp{lambda = 0.85, scale = Just 1000}
@@ -243,6 +245,7 @@ configToKVsWith getter cfg =
     , get @"treatBlocksAsFull" getter cfg
     , get @"cleanupPolicies" getter cfg
     , get @"leiosVariant" getter cfg
+    , get @"leiosLateIbInclusion" getter cfg
     , get @"leiosHeaderDiffusionTimeMs" getter cfg
     , get @"praosChainQuality" getter cfg
     , get @"simulateTransactions" getter cfg
@@ -329,6 +332,7 @@ instance FromJSON Config where
     treatBlocksAsFull <- parseFieldOrDefault @Config @"treatBlocksAsFull" obj
     cleanupPolicies <- parseFieldOrDefault @Config @"cleanupPolicies" obj
     leiosVariant <- parseFieldOrDefault @Config @"leiosVariant" obj
+    leiosLateIbInclusion <- parseFieldOrDefault @Config @"leiosLateIbInclusion" obj
     leiosHeaderDiffusionTimeMs <- parseFieldOrDefault @Config @"leiosHeaderDiffusionTimeMs" obj
     praosChainQuality <- parseFieldOrDefault @Config @"praosChainQuality" obj
     simulateTransactions <- parseFieldOrDefault @Config @"simulateTransactions" obj

leios-trace-verifier/hs-src/test/Spec/Scenario.hs

@@ -19,7 +19,7 @@ import qualified Data.Map.Strict as M
 import qualified Data.Set as S
 
 config :: Config
-config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0}
+config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosLateIbInclusion = False, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0}
 
 topology :: Topology 'COORD2D
 topology = Topology{nodes = M.fromList [(NodeName "node-0", Node{nodeInfo = NodeInfo{stake = 500, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.12000040231003672, _2 = 0.1631004621065356}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-1", LinkInfo{latencyMs = 141.01364015418432, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-2", LinkInfo{latencyMs = 254.6249782835189, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-1", Node{nodeInfo = NodeInfo{stake = 200, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.34276660615051174, _2 = 0.2636899791034371}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-2", LinkInfo{latencyMs = 175.32530255486685, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-3", LinkInfo{latencyMs = 379.1167948193313, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-2", Node{nodeInfo = NodeInfo{stake = 100, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.5150493264153491, _2 = 0.27873594531347595}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-3", LinkInfo{latencyMs = 248.31457793649423, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-3", Node{nodeInfo = NodeInfo{stake = 0, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.3503537969220088, _2 = 0.13879558055660354}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-0", LinkInfo{latencyMs = 140.19739576271448, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]})]}

simulation/docs/SimulatorModel.md

@@ -96,7 +96,8 @@ Each EB (see `LeiosProtocol.Common.EndorseBlock`) consists of the following fiel
 
 More details for some fields.
 
-- An EB from iteration `i` includes the IDs of all IBs that were already adopted, are also from iteration `i`, and arrived before the end of `i`'s Deliver2 stage.
+- If `leios-late-ib-inclusion` is disabled, an EB from iteration `i` includes the IDs of all IBs that were already adopted, are also from iteration `i`, and arrived before the end of `i`'s Deliver2 stage.
+- If `leios-late-ib-inclusion` is enabled, an EB from iteration `i` includes the IDs of all IBs that were already adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s contemporary Deliver2 stage.
 - If the Leios variant is set to `short`, this EB includes no EB IDs.
 - If the Leios variant is set to `full`, an EB from iteration `i` includes the ID of the best eligible EB from each iteration with any eligible EBs.
     - An eligible EB has already been adopted, has already been certified, and is from an iteration in the closed interval `[i - min i (2 + pipelinesToReferenceFromEB), i-3]`.
@@ -129,8 +130,10 @@ More details for some fields.
 - A VB from iteration `i` includes the IDs of all EBs that satisfy the following.
     - The EB must have already been adopted.
     - The EB must also be from iteration `i`.
-    - The EB must only include IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Endorse stage.
-    - The EB must include all IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Deliver1 stage.
+    - If `leios-late-ib-inclusion` is disabled, the EB must only include IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Endorse stage.
+    - If `leios-late-ib-inclusion` is disabled, the EB must include all IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Deliver1 stage.
+    - If `leios-late-ib-inclusion` is enabled, the EB must only include IBs that have already been adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s Endorse stage.
+    - If `leios-late-ib-inclusion` is enabled, the EB must include all IBs that have already been adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s Deliver1 stage.
     - If the Leios variant is set to `full`, then let X be the EB's included EBs in iteration order; let Y be the EBs this node would have considered eligible if it were to retroactively create an EB for iteration `i` right now with the only extra restriction being ignore EBs that arrived within Δ_hdr of the end of iteration `i`; then `and (zipWith elem X Y)` must be `True`.
       (TODO the `zipWith` is suspicious; whether it would misbehave in various scenarios depends on many implementation details.)
 - The byte size is computed as `voteBundleSizeBytesConstant + voteBundleSizeBytesPerEb * #EBs` (which implies the weighted-vote perspective).
@@ -192,7 +195,7 @@ TODO discuss the other Relay parameters, backpressure, pipelining, etc?
 When an IB header arrives, its validation task is enqueued on the model CPU---for VBs and EBs it's just an ID, not a header, so there's no validation.
 Once that finishes, the Relay logic will decide whether it needs to fetch the body.
 
-- An IB body is not fetched if it's older than the slot to which the buffer as has already been pruned or if it's already in the buffer.
+- An IB body is not fetched if it exists earlier than it should, it's being offered later than it should be, or if it's already in the buffer.
 - An EB is not fetched if it's older than the slot to which the buffer has already been pruned, it's too old to be included by an RB (see `maxEndorseBlockAgeSlots`), or if it's already in the buffer.
 - A VB is not fetched if it's older than the slot to which the buffer has already been pruned or if it's already in the buffer.
 
@@ -233,7 +236,9 @@ Because those threads use STM to read both the state of pending tasks as well as
 
 The existence of those threads enable very simple logic for the adoption tasks.
 
-- The node adopts a validated IB by starting to diffuse it, adding its `UTCTime` arrival to `ibDeliveryTimesVar`, and removing the IB from the todo lists in `ibsNeededForEBVar`.
+- The node adopts a validated IB by starting to diffuse it, removing the IB from the todo lists in `ibsNeededForEBVar`, and recording its ID and which stage it arrived during.
+  See `iBsForEBsAndVotesVar`.
+  If it arrived during the IB's iteration's Propose stage (aka "early") or after the IB's iteration's Endorse stage (aka "tardy"), then the IB is discarded.
 - The node adopts a validated EB by starting to diffuse it, adding it to `relayEBState`, and adding a corresponding todo list of the not-already-available IBs to `ibsNeededForEBVar`.
 - The node adopts a validated VB by starting to diffuse it and adding it to `votesForEBVar`.
 - The node adopts a validated RB by starting to diffuse it and including it whenever calculating its selection; see `preferredChain`.
@@ -245,17 +250,18 @@ The Relay component invokes the given callback when some object arrives, and tha
 ## Pruning threads
 
 - *IBs 1*.
-  At the end of the Vote(Send) stage for iteration `i`, the node stops diffusing all IBs from `i`.
-  (TODO this should happen at the end of the Endorse stage, but this buffer is being abused as the adoption buffer as well.)
-  It also forgets any of those IBs it had adopted, with the exception of their arrival time, which is used when generating VBs.
+  At the end of the Endorse stage for iteration `i`, the node stops diffusing all IBs from `i`.
   See `relayIBState`.
+- *IBs 2*.
+  If `leios-late-ib-inclusion` is disabled, then at the end of the Vote(Send) stage for iteration `i`, the node forgets the arrival times of all IBs from `i`.
+  If `leios-late-ib-inclusion` is enabled, the node instead does that two stages later.
+  See `iBsForEBsAndVotesVar`.
 - *EBs 1*.
   At the end of the Vote(Recv) stage for iteration `i`, the node stops diffusing and completely forgets all EBs from `i` that are not already certified.
   See `relayEBState`, `votesForEBVar`, and `ibsNeededForEBVar`.
-- *VBs* and *IBs 2*.
+- *VBs*.
   At the end of the Vote(Recv) stage for iteration `i`, the node stops diffusing and completely forgets all VBs from `i`, except that certified EBs from `i` remember the ID and multiplicity of the VBs that first met quorum.
-  It also forgets the arrival time of IBs from `i`.
-  See `relayVoteState` and `ibDeliveryTimesVar`.
+  See `relayVoteState`.
 - *EBs 2*.
   If the Leios variant is set to `short`, then `maxEndorseBlockAgeSlots` after the end of the Endorse stage for iteration `i`, the node stops diffusing and forgets all EBs from `i` that were certified but are not included by an RB on the selected chain.
   (TODO these blocks should have stopped diffusing a long time ago, assuming `maxEndorseBlockAgeSlots >> sliceLength`)
@@ -281,7 +287,7 @@ TODO include `taskQueue`
 
 TODO `relayIBState` abuse
 
-TODO `ibDeliveryTimesVar`
+TODO `iBsForEBsAndVotesVar`
 
 ## Adopted EBs state
 

simulation/ouroboros-leios-sim.cabal

@@ -218,6 +218,7 @@ test-suite ols-test
     , bytestring
   other-modules:
     Paths_ouroboros_leios_sim
-    Test.Topology
     Test.Config
+    Test.ShortToFull
+    Test.Topology
   default-language: Haskell2010

simulation/src/LeiosProtocol/Relay.hs

@@ -498,7 +498,7 @@ data SubmitPolicy = SubmitInOrder | SubmitAll
 
 data RelayConsumerConfig id header body m = RelayConsumerConfig
   { relay :: !RelayConfig
-  , shouldIgnore :: m (header -> Bool)
+  , shouldNotRequest :: m (header -> Bool)
   -- ^ headers to ignore, e.g. already received or coming too late.
   , validateHeaders :: [header] -> m ()
   , headerId :: !(header -> id)
@@ -761,7 +761,7 @@ relayConsumerPipelined config sst =
     if (min (Map.size lst0.available) (fromIntegral config.maxBodiesToRequest)) == 0
       then return (Left lst0)
       else return . Right . TS.Effect $ do
-        isIgnored <- config.shouldIgnore
+        isIgnored <- config.shouldNotRequest
         atomically $ do
           -- New headers are filtered before becoming available, but we have
           -- to filter `lst.available` again in the same STM tx that sets them as
@@ -987,7 +987,7 @@ relayConsumerPipelined config sst =
     m (RelayConsumerLocalState id header body n)
   acknowledgeIds lst idsSeq _ | Seq.null idsSeq = pure lst
   acknowledgeIds lst idsSeq idsMap = do
-    isIgnored <- config.shouldIgnore
+    isIgnored <- config.shouldNotRequest
     inFlight <- readTVarIO sst.inFlightVar
 
     let lst1 =