diff --git a/data/simulation/config.d.ts b/data/simulation/config.d.ts index 02b0b648a..e52572244 100644 --- a/data/simulation/config.d.ts +++ b/data/simulation/config.d.ts @@ -39,10 +39,10 @@ export interface Config { * Only supported by Haskell simulation. */ "leios-vote-send-recv-stages": boolean; /** - * Extends Leios so that EB producers include IBs directly from previous pipelines - * where no certified EB was observed. - * - * Only supported by Rust simulation. */ + * Extends Leios so that EB producers include IBs directly from previous pipelines. + * Due to casuality, the EB must always include them, even if those IBs end up being + * certified in their own pipeline. + */ "leios-late-ib-inclusion": boolean; /** * The expected time it takes a header to fully diffuse across the network. diff --git a/data/simulation/variants/config.full.yaml b/data/simulation/variants/config.full.yaml index 98bd69dba..fc73ec405 100644 --- a/data/simulation/variants/config.full.yaml +++ b/data/simulation/variants/config.full.yaml @@ -4,7 +4,7 @@ leios-variant: full # "Random" is meant to ensure that different producers include different TXs. leios-mempool-sampling-strategy: random -# Allow EBs to include IBs from previous slots +# Allow EBs to include IBs from previous pipelines leios-late-ib-inclusion: true # Chain quality controls how far back EB recursion can reach. diff --git a/leios-trace-hs/src/LeiosConfig.hs b/leios-trace-hs/src/LeiosConfig.hs index 6ff097025..c0a2902e1 100644 --- a/leios-trace-hs/src/LeiosConfig.hs +++ b/leios-trace-hs/src/LeiosConfig.hs @@ -99,6 +99,7 @@ data Config = Config , leiosStageActiveVotingSlots :: Word , leiosVoteSendRecvStages :: Bool , leiosVariant :: LeiosVariant + , leiosLateIbInclusion :: Bool , leiosHeaderDiffusionTimeMs :: DurationMs , praosChainQuality :: Double , txGenerationDistribution :: Distribution @@ -171,6 +172,7 @@ instance Default Config where , leiosStageActiveVotingSlots = 1 , leiosVoteSendRecvStages = False , leiosVariant = Short + , leiosLateIbInclusion = True , leiosHeaderDiffusionTimeMs = 1000 , praosChainQuality = 40 , txGenerationDistribution = Exp{lambda = 0.85, scale = Just 1000} @@ -243,6 +245,7 @@ configToKVsWith getter cfg = , get @"treatBlocksAsFull" getter cfg , get @"cleanupPolicies" getter cfg , get @"leiosVariant" getter cfg + , get @"leiosLateIbInclusion" getter cfg , get @"leiosHeaderDiffusionTimeMs" getter cfg , get @"praosChainQuality" getter cfg , get @"simulateTransactions" getter cfg @@ -329,6 +332,7 @@ instance FromJSON Config where treatBlocksAsFull <- parseFieldOrDefault @Config @"treatBlocksAsFull" obj cleanupPolicies <- parseFieldOrDefault @Config @"cleanupPolicies" obj leiosVariant <- parseFieldOrDefault @Config @"leiosVariant" obj + leiosLateIbInclusion <- parseFieldOrDefault @Config @"leiosLateIbInclusion" obj leiosHeaderDiffusionTimeMs <- parseFieldOrDefault @Config @"leiosHeaderDiffusionTimeMs" obj praosChainQuality <- parseFieldOrDefault @Config @"praosChainQuality" obj simulateTransactions <- parseFieldOrDefault @Config @"simulateTransactions" obj diff --git a/leios-trace-verifier/hs-src/test/Spec/Scenario.hs b/leios-trace-verifier/hs-src/test/Spec/Scenario.hs index d311116b2..0fa8deb15 100644 --- a/leios-trace-verifier/hs-src/test/Spec/Scenario.hs +++ b/leios-trace-verifier/hs-src/test/Spec/Scenario.hs @@ -19,7 +19,7 @@ import qualified Data.Map.Strict as M import qualified Data.Set as S config :: Config -config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0} +config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosLateIbInclusion = False, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0} topology :: Topology 'COORD2D topology = Topology{nodes = M.fromList [(NodeName "node-0", Node{nodeInfo = NodeInfo{stake = 500, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.12000040231003672, _2 = 0.1631004621065356}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-1", LinkInfo{latencyMs = 141.01364015418432, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-2", LinkInfo{latencyMs = 254.6249782835189, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-1", Node{nodeInfo = NodeInfo{stake = 200, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.34276660615051174, _2 = 0.2636899791034371}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-2", LinkInfo{latencyMs = 175.32530255486685, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-3", LinkInfo{latencyMs = 379.1167948193313, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-2", Node{nodeInfo = NodeInfo{stake = 100, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.5150493264153491, _2 = 0.27873594531347595}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-3", LinkInfo{latencyMs = 248.31457793649423, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-3", Node{nodeInfo = NodeInfo{stake = 0, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.3503537969220088, _2 = 0.13879558055660354}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-0", LinkInfo{latencyMs = 140.19739576271448, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]})]} diff --git a/simulation/docs/SimulatorModel.md b/simulation/docs/SimulatorModel.md index 5e3974ffd..1618b0388 100644 --- a/simulation/docs/SimulatorModel.md +++ b/simulation/docs/SimulatorModel.md @@ -96,7 +96,8 @@ Each EB (see `LeiosProtocol.Common.EndorseBlock`) consists of the following fiel More details for some fields. -- An EB from iteration `i` includes the IDs of all IBs that were already adopted, are also from iteration `i`, and arrived before the end of `i`'s Deliver2 stage. +- If `leios-late-ib-inclusion` is disabled, an EB from iteration `i` includes the IDs of all IBs that were already adopted, are also from iteration `i`, and arrived before the end of `i`'s Deliver2 stage. +- If `leios-late-ib-inclusion` is enabled, an EB from iteration `i` includes the IDs of all IBs that were already adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s contemporary Deliver2 stage. - If the Leios variant is set to `short`, this EB includes no EB IDs. - If the Leios variant is set to `full`, an EB from iteration `i` includes the ID of the best eligible EB from each iteration with any eligible EBs. - An eligible EB has already been adopted, has already been certified, and is from an iteration in the closed interval `[i - min i (2 + pipelinesToReferenceFromEB), i-3]`. @@ -129,8 +130,10 @@ More details for some fields. - A VB from iteration `i` includes the IDs of all EBs that satisfy the following. - The EB must have already been adopted. - The EB must also be from iteration `i`. - - The EB must only include IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Endorse stage. - - The EB must include all IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Deliver1 stage. + - If `leios-late-ib-inclusion` is disabled, the EB must only include IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Endorse stage. + - If `leios-late-ib-inclusion` is disabled, the EB must include all IBs that have already been adopted, are from iteration `i`, and arrived before the end of `i`'s Deliver1 stage. + - If `leios-late-ib-inclusion` is enabled, the EB must only include IBs that have already been adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s Endorse stage. + - If `leios-late-ib-inclusion` is enabled, the EB must include all IBs that have already been adopted, are from an iteration `j` in the closed interval `[max 0 (i-2), i]`, and arrived before the end of `j`'s Deliver1 stage. - If the Leios variant is set to `full`, then let X be the EB's included EBs in iteration order; let Y be the EBs this node would have considered eligible if it were to retroactively create an EB for iteration `i` right now with the only extra restriction being ignore EBs that arrived within Δ_hdr of the end of iteration `i`; then `and (zipWith elem X Y)` must be `True`. (TODO the `zipWith` is suspicious; whether it would misbehave in various scenarios depends on many implementation details.) - The byte size is computed as `voteBundleSizeBytesConstant + voteBundleSizeBytesPerEb * #EBs` (which implies the weighted-vote perspective). @@ -192,7 +195,7 @@ TODO discuss the other Relay parameters, backpressure, pipelining, etc? When an IB header arrives, its validation task is enqueued on the model CPU---for VBs and EBs it's just an ID, not a header, so there's no validation. Once that finishes, the Relay logic will decide whether it needs to fetch the body. -- An IB body is not fetched if it's older than the slot to which the buffer as has already been pruned or if it's already in the buffer. +- An IB body is not fetched if it exists earlier than it should, it's being offered later than it should be, or if it's already in the buffer. - An EB is not fetched if it's older than the slot to which the buffer has already been pruned, it's too old to be included by an RB (see `maxEndorseBlockAgeSlots`), or if it's already in the buffer. - A VB is not fetched if it's older than the slot to which the buffer has already been pruned or if it's already in the buffer. @@ -233,7 +236,9 @@ Because those threads use STM to read both the state of pending tasks as well as The existence of those threads enable very simple logic for the adoption tasks. -- The node adopts a validated IB by starting to diffuse it, adding its `UTCTime` arrival to `ibDeliveryTimesVar`, and removing the IB from the todo lists in `ibsNeededForEBVar`. +- The node adopts a validated IB by starting to diffuse it, removing the IB from the todo lists in `ibsNeededForEBVar`, and recording its ID and which stage it arrived during. + See `iBsForEBsAndVotesVar`. + If it arrived during the IB's iteration's Propose stage (aka "early") or after the IB's iteration's Endorse stage (aka "tardy"), then the IB is discarded. - The node adopts a validated EB by starting to diffuse it, adding it to `relayEBState`, and adding a corresponding todo list of the not-already-available IBs to `ibsNeededForEBVar`. - The node adopts a validated VB by starting to diffuse it and adding it to `votesForEBVar`. - The node adopts a validated RB by starting to diffuse it and including it whenever calculating its selection; see `preferredChain`. @@ -245,17 +250,18 @@ The Relay component invokes the given callback when some object arrives, and tha ## Pruning threads - *IBs 1*. - At the end of the Vote(Send) stage for iteration `i`, the node stops diffusing all IBs from `i`. - (TODO this should happen at the end of the Endorse stage, but this buffer is being abused as the adoption buffer as well.) - It also forgets any of those IBs it had adopted, with the exception of their arrival time, which is used when generating VBs. + At the end of the Endorse stage for iteration `i`, the node stops diffusing all IBs from `i`. See `relayIBState`. +- *IBs 2*. + If `leios-late-ib-inclusion` is disabled, then at the end of the Vote(Send) stage for iteration `i`, the node forgets the arrival times of all IBs from `i`. + If `leios-late-ib-inclusion` is enabled, the node instead does that two stages later. + See `iBsForEBsAndVotesVar`. - *EBs 1*. At the end of the Vote(Recv) stage for iteration `i`, the node stops diffusing and completely forgets all EBs from `i` that are not already certified. See `relayEBState`, `votesForEBVar`, and `ibsNeededForEBVar`. -- *VBs* and *IBs 2*. +- *VBs*. At the end of the Vote(Recv) stage for iteration `i`, the node stops diffusing and completely forgets all VBs from `i`, except that certified EBs from `i` remember the ID and multiplicity of the VBs that first met quorum. - It also forgets the arrival time of IBs from `i`. - See `relayVoteState` and `ibDeliveryTimesVar`. + See `relayVoteState`. - *EBs 2*. If the Leios variant is set to `short`, then `maxEndorseBlockAgeSlots` after the end of the Endorse stage for iteration `i`, the node stops diffusing and forgets all EBs from `i` that were certified but are not included by an RB on the selected chain. (TODO these blocks should have stopped diffusing a long time ago, assuming `maxEndorseBlockAgeSlots >> sliceLength`) @@ -281,7 +287,7 @@ TODO include `taskQueue` TODO `relayIBState` abuse -TODO `ibDeliveryTimesVar` +TODO `iBsForEBsAndVotesVar` ## Adopted EBs state diff --git a/simulation/ouroboros-leios-sim.cabal b/simulation/ouroboros-leios-sim.cabal index 62535694d..a6ddfee44 100644 --- a/simulation/ouroboros-leios-sim.cabal +++ b/simulation/ouroboros-leios-sim.cabal @@ -218,6 +218,7 @@ test-suite ols-test , bytestring other-modules: Paths_ouroboros_leios_sim - Test.Topology Test.Config + Test.ShortToFull + Test.Topology default-language: Haskell2010 diff --git a/simulation/src/LeiosProtocol/Relay.hs b/simulation/src/LeiosProtocol/Relay.hs index 2042a6ed8..2fa3a0408 100644 --- a/simulation/src/LeiosProtocol/Relay.hs +++ b/simulation/src/LeiosProtocol/Relay.hs @@ -498,7 +498,7 @@ data SubmitPolicy = SubmitInOrder | SubmitAll data RelayConsumerConfig id header body m = RelayConsumerConfig { relay :: !RelayConfig - , shouldIgnore :: m (header -> Bool) + , shouldNotRequest :: m (header -> Bool) -- ^ headers to ignore, e.g. already received or coming too late. , validateHeaders :: [header] -> m () , headerId :: !(header -> id) @@ -761,7 +761,7 @@ relayConsumerPipelined config sst = if (min (Map.size lst0.available) (fromIntegral config.maxBodiesToRequest)) == 0 then return (Left lst0) else return . Right . TS.Effect $ do - isIgnored <- config.shouldIgnore + isIgnored <- config.shouldNotRequest atomically $ do -- New headers are filtered before becoming available, but we have -- to filter `lst.available` again in the same STM tx that sets them as @@ -987,7 +987,7 @@ relayConsumerPipelined config sst = m (RelayConsumerLocalState id header body n) acknowledgeIds lst idsSeq _ | Seq.null idsSeq = pure lst acknowledgeIds lst idsSeq idsMap = do - isIgnored <- config.shouldIgnore + isIgnored <- config.shouldNotRequest inFlight <- readTVarIO sst.inFlightVar let lst1 = diff --git a/simulation/src/LeiosProtocol/Short.hs b/simulation/src/LeiosProtocol/Short.hs index 7d9b35280..834e95eb8 100644 --- a/simulation/src/LeiosProtocol/Short.hs +++ b/simulation/src/LeiosProtocol/Short.hs @@ -111,6 +111,10 @@ data LeiosConfig = forall p. IsPipeline p => LeiosConfig , variant :: LeiosVariant , headerDiffusionTime :: NominalDiffTime -- ^ Δ_{hdr}. + , lateIbInclusion :: Bool + -- ^ Whether an EB also includes IBs from the two previous iterations. + -- + -- TODO Merely one previous iteration if 'pipeline' is 'SingleVote'? , pipelinesToReferenceFromEB :: Int -- ^ how many older pipelines to reference from an EB when `variant = Full`. , votingFrequencyPerStage :: Double @@ -147,6 +151,7 @@ convertConfig disk = , cleanupPolicies = disk.cleanupPolicies , variant = disk.leiosVariant , headerDiffusionTime = realToFrac $ durationMsToDiffTime disk.leiosHeaderDiffusionTimeMs + , lateIbInclusion = disk.leiosLateIbInclusion , pipelinesToReferenceFromEB = if disk.leiosVariant == Full then @@ -285,6 +290,7 @@ delaysAndSizesAsFull cfg@LeiosConfig{pipeline, voteSendStage} = , cleanupPolicies = cfg.cleanupPolicies , variant = cfg.variant , headerDiffusionTime = cfg.headerDiffusionTime + , lateIbInclusion = cfg.lateIbInclusion , pipelinesToReferenceFromEB = cfg.pipelinesToReferenceFromEB , activeVotingStageLength = cfg.activeVotingStageLength , votingFrequencyPerStage = cfg.votingFrequencyPerStage @@ -461,21 +467,38 @@ isStage cfg stage slot = fromEnum slot >= cfg.sliceLength * fromEnum stage newtype PipelineNo = PipelineNo Word64 deriving (Bounded, Enum, Show, Eq, Ord) +pipelineMonus :: PipelineNo -> Word64 -> PipelineNo +pipelineMonus (PipelineNo w) i = PipelineNo $ w - min w i + stageRangeOf :: forall p. IsPipeline p => LeiosConfig -> PipelineNo -> Stage p -> (SlotNo, SlotNo) stageRangeOf cfg pl stage = fromMaybe undefined (stageRange cfg minBound (toEnum (fromEnum pl * cfg.sliceLength)) stage) +-- | WARNING This fails if the slot is earlier than the beginning of the stage +-- in the first iteration (ie @'PipelineNo' 0@) pipelineOf :: forall p. IsPipeline p => LeiosConfig -> Stage p -> SlotNo -> PipelineNo pipelineOf cfg stage sl = - toEnum $ - fromMaybe undefined (fromEnum <$> stageStart cfg stage sl minBound) - `div` cfg.sliceLength + maybe err cnv $ stageStart cfg stage sl minBound + where + cnv = toEnum . (`div` cfg.sliceLength) . fromEnum + + err = error $ show (cfg.sliceLength, x, stage, sl) + + x :: String + x = case cfg of + LeiosConfig{pipeline} -> case pipeline of + SingSingleVote -> "SingleVote" + SingSplitVote -> "SplitVote" forEachPipeline :: (forall p. Stage p) -> (forall p. IsPipeline p => Stage p -> a) -> [a] forEachPipeline s k = [k @SingleVote s, k @SplitVote s] +lastEndorse :: LeiosConfig -> PipelineNo -> SlotNo +lastEndorse leios@LeiosConfig{pipeline = _ :: SingPipeline p} pipelineNo = + snd $ stageRangeOf @p leios pipelineNo Endorse + lastVoteSend :: LeiosConfig -> PipelineNo -> SlotNo lastVoteSend leios@LeiosConfig{pipeline} pipelineNo = case pipeline of SingSingleVote -> snd (stageRangeOf leios pipelineNo Vote) @@ -657,28 +680,76 @@ data EndorseBlocksSnapshot = EndorseBlocksSnapshot , certifiedEndorseBlocks :: (PipelineNo, PipelineNo) -> [(PipelineNo, [(EndorseBlock, Certificate, UTCTime)])] } +-- | In which contemporary stage was an IB delivered +-- +-- IBs cannot be deliver earlier than any of these options, due to the +-- 'LeiosProtocol.Relay.shouldNotRequest' logic of the +-- 'LeiosProtocol.Short.Node.relayIBState'. +-- +-- IBs that are delivered later than any of these options are discarded, +-- ignored. +data IbDeliveryStage + = -- | The node will not vote for an EB that excludes IBs that arrived during + -- Propose or Deliver1. + -- + -- The node will include IBs that arrived during Propose or Deliver1 in an + -- EB it makes. + IbDuringProposeOrDeliver1 + | -- | The node will include IBs that arrived during Deliver2 in an EB it makes. + IbDuringDeliver2 + | -- | The node will not vote for an EB that includes IBs that arrived later + -- than Endorse. + IbDuringEndorse + deriving (Bounded, Enum, Eq, Ord, Show) + -- | Both constraints are inclusive. data InputBlocksQuery = InputBlocksQuery - { generatedBetween :: (SlotNo, SlotNo) - , receivedBy :: SlotNo + { generatedBetween :: (PipelineNo, PipelineNo) + , receivedBy :: IbDeliveryStage -- ^ This is checked against time the body is downloaded, before validation. } +ibWasDeliveredLate :: LeiosConfig -> SlotConfig -> SlotNo -> UTCTime -> Bool +ibWasDeliveredLate cfg slotCfg sl deliveryTime = + case ibDeliveryStage cfg slotCfg sl deliveryTime of + Nothing -> True + Just{} -> False + +ibDeliveryStage :: LeiosConfig -> SlotConfig -> SlotNo -> UTCTime -> Maybe IbDeliveryStage +ibDeliveryStage + cfg@LeiosConfig{pipeline = _ :: SingPipeline p} + slotCfg + ibSlot + deliveryTime + | before loPropose = Nothing -- TODO future blocks? + | before loDeliver2 = Just IbDuringProposeOrDeliver1 + | before loEndorse = Just IbDuringDeliver2 + | before (succ hiEndorse) = Just IbDuringEndorse + | otherwise = Nothing -- TODO late blocks? + where + p = pipelineOf @p cfg Propose ibSlot + + before sl = deliveryTime < slotTime slotCfg sl + + (loPropose, _) = stageRangeOf @p cfg p Propose + (loDeliver2, _) = stageRangeOf @p cfg p Deliver2 + (loEndorse, hiEndorse) = stageRangeOf @p cfg p Endorse + inputBlocksToEndorse :: LeiosConfig -> -- | current slot SlotNo -> InputBlocksSnapshot -> [InputBlockId] -inputBlocksToEndorse cfg@LeiosConfig{pipeline = _ :: SingPipeline p} current buffer = fromMaybe [] $ do - generatedBetween <- stageRange @p cfg Endorse current Propose - receivedBy <- stageEnd @p cfg Endorse current Deliver2 - pure $ - buffer.validInputBlocks - InputBlocksQuery - { generatedBetween - , receivedBy - } +inputBlocksToEndorse cfg@LeiosConfig{pipeline = _ :: SingPipeline p} current buffer = + buffer.validInputBlocks + InputBlocksQuery + { generatedBetween = (lo, hi) + , receivedBy = IbDuringDeliver2 + } + where + hi = pipelineOf @p cfg Endorse current + lo = if cfg.lateIbInclusion then pipelineMonus hi 2 else hi -- | Returns possible EBs to reference from current pipeline EB. endorseBlocksToReference :: @@ -734,21 +805,24 @@ shouldVoteOnEB :: shouldVoteOnEB cfg@LeiosConfig{voteSendStage} _ slot _buffers _ -- checks whether a pipeline has been started before. | Nothing <- stageRange cfg voteSendStage slot Propose = const False -shouldVoteOnEB cfg@LeiosConfig{voteSendStage} slotConfig slot buffers ebuffers = cond +shouldVoteOnEB cfg@LeiosConfig{voteSendStage = voteSendStage :: Stage p} slotConfig slot buffers ebuffers = cond where - generatedBetween = fromMaybe (error "impossible") $ stageRange cfg voteSendStage slot Propose + generatedBetween = (lo, hi) + where + hi = pipelineOf @p cfg voteSendStage slot + lo = if cfg.lateIbInclusion then pipelineMonus hi 2 else hi receivedByEndorse = buffers.validInputBlocks InputBlocksQuery { generatedBetween - , receivedBy = fromMaybe (error "impossible") $ stageEnd cfg voteSendStage slot Endorse + , receivedBy = IbDuringEndorse } receivedByDeliver1 = buffers.validInputBlocks q where q = InputBlocksQuery { generatedBetween - , receivedBy = fromMaybe (error "impossible") $ stageEnd cfg voteSendStage slot Deliver1 + , receivedBy = IbDuringProposeOrDeliver1 } -- Order of references in EndorseBlock matters for ledger state, so we stick to lists. -- Note: maybe order on (slot, subSlot, vrf proof) should be used instead? diff --git a/simulation/src/LeiosProtocol/Short/Node.hs b/simulation/src/LeiosProtocol/Short/Node.hs index 273686c3d..9c40dd3e0 100644 --- a/simulation/src/LeiosProtocol/Short/Node.hs +++ b/simulation/src/LeiosProtocol/Short/Node.hs @@ -19,7 +19,7 @@ import Control.Category ((>>>)) import Control.Concurrent.Class.MonadMVar import Control.Concurrent.Class.MonadSTM.TSem import Control.Exception (assert) -import Control.Monad (forever, guard, replicateM, when) +import Control.Monad (forever, guard, replicateM, unless, when) import Control.Monad.Class.MonadAsync import Control.Monad.Class.MonadFork import Control.Monad.Class.MonadThrow @@ -122,16 +122,29 @@ data LeiosNodeConfig = LeiosNodeConfig data LeiosNodeState m = LeiosNodeState { praosState :: !(PraosNode.PraosNodeState RankingBlockBody m) , relayIBState :: !(RelayIBState m) - , prunedIBStateToVar :: !(TVar m SlotNo) + -- ^ validated IBs that are still young enough to be diffusing + , iBsForEBsAndVotesVar :: !(TVar m (Map PipelineNo (Map InputBlockId IbDeliveryStage))) + -- ^ IBs that are relevant to an EB or Vote this node might need to issue + -- + -- Each of these IBs arrived during its contemporary Propose, Deliver1, + -- Deliver2, or Endorse stages, has been validated, and is not too old. + -- + -- INVARIANT: In basic Short Leios, none of these IBs are older than + -- @4*'sliceLength'@. With the @leios-late-ib-inclusion@ extension enabled, + -- none of these IBs are older than @(4+2)*'sliceLength'@. + -- + -- Note that some IBs that are too old to be included in this variable + -- might still be needed in order to apply some RB. + -- + -- INVARIANT: @all (\_k v -> not $ null v)@. + -- + -- INVARIANT: @all (\k v -> all ((k ==) . pipelineOf cfg Propose) v)@. , relayEBState :: !(RelayEBState m) , prunedUnadoptedEBStateToVar :: !(TVar m SlotNo) , prunedUncertifiedEBStateToVar :: !(TVar m SlotNo) , relayVoteState :: !(RelayVoteState m) , prunedVoteStateToVar :: !(TVar m SlotNo) -- ^ TODO: refactor into RelayState. - , ibDeliveryTimesVar :: !(TVar m (Map InputBlockId (SlotNo, UTCTime))) - -- ^ records time we received the input block. - -- Also stores the SlotNo of the IB to ease pruning. , taskQueue :: !(TaskMultiQueue LeiosNodeTask m) , waitingForRBVar :: !(TVar m (Map (HeaderHash RankingBlock) [STM m ()])) -- ^ waiting for RB block itself to be validated. @@ -186,7 +199,7 @@ data LedgerState = LedgerState data ValidationRequest m = ValidateRB !RankingBlock !(m ()) - | ValidateIBS ![(InputBlockHeader, InputBlockBody)] !UTCTime !([(InputBlockHeader, InputBlockBody)] -> STM m ()) + | ValidateIBs ![((InputBlockHeader, InputBlockBody), IbDeliveryStage)] !([(InputBlockHeader, InputBlockBody)] -> STM m ()) | ValidateEBS ![EndorseBlock] !([EndorseBlock] -> STM m ()) | ValidateVotes ![VoteMsg] !UTCTime !([VoteMsg] -> STM m ()) @@ -296,10 +309,10 @@ relayIBConfig _tracer cfg validateHeaders submitBlocks st = , maxHeadersToRequest = cfg.leios.ibDiffusion.maxHeadersToRequest , maxBodiesToRequest = cfg.leios.ibDiffusion.maxBodiesToRequest , submitBlocks - , shouldIgnore = atomically $ do - prunedTo <- readTVar st.prunedIBStateToVar + , shouldNotRequest = (getCurrentTime >>=) $ \deliveryTime -> atomically $ do + let late h = ibWasDeliveredLate cfg.leios cfg.slotConfig h.slot deliveryTime buff <- readTVar st.relayIBState.relayBufferVar - return $ \h -> h.slot < prunedTo || h.id `RB.member` buff + return $ \h -> late h || h.id `RB.member` buff } relayEBConfig :: @@ -321,7 +334,7 @@ relayEBConfig _tracer cfg@LeiosNodeConfig{leios = LeiosConfig{pipeline = (_ :: S , maxBodiesToRequest = cfg.leios.ebDiffusion.maxBodiesToRequest , submitBlocks = \hbs t k -> submitBlocks (map (first (.id)) hbs) t (k . map (\(i, b) -> (RelayHeader i b.slot, b))) - , shouldIgnore = do + , shouldNotRequest = do -- We possibly prune certified EBs (not referenced in the -- chain) after maxEndorseBlockAgeSlots, so we should not end -- up asking for their bodies again, in the remote possibility @@ -362,7 +375,7 @@ relayVoteConfig _tracer cfg submitBlocks _ leiosState = , maxBodiesToRequest = cfg.leios.voteDiffusion.maxBodiesToRequest , submitBlocks = \hbs t k -> submitBlocks (map (first (.id)) hbs) t (k . map (\(i, b) -> (RelayHeader i b.slot, b))) - , shouldIgnore = atomically $ do + , shouldNotRequest = atomically $ do buffer <- readTVar leiosState.relayVoteState.relayBufferVar prunedTo <- readTVar leiosState.prunedVoteStateToVar return $ \hd -> @@ -389,13 +402,12 @@ newLeiosNodeState :: newLeiosNodeState cfg = do praosState <- PraosNode.newPraosNodeState cfg.baseChain relayIBState <- newRelayState - prunedIBStateToVar <- newTVarIO (toEnum 0) + iBsForEBsAndVotesVar <- newTVarIO Map.empty relayEBState <- newRelayState prunedUnadoptedEBStateToVar <- newTVarIO (toEnum 0) prunedUncertifiedEBStateToVar <- newTVarIO (toEnum 0) relayVoteState <- newRelayState prunedVoteStateToVar <- newTVarIO (toEnum 0) - ibDeliveryTimesVar <- newTVarIO Map.empty ibsNeededForEBVar <- newTVarIO Map.empty ledgerStateVar <- newTVarIO Map.empty waitingForRBVar <- newTVarIO Map.empty @@ -432,7 +444,9 @@ leiosNode tracer cfg followers peers = do let submitRB rb completion = dispatch $! ValidateRB rb completion let submitIB xs deliveryTime completion = do traceReceived xs $ EventIB . uncurry InputBlock - dispatch $! ValidateIBS xs deliveryTime completion + let annotate x = (,) x <$> ibDeliveryStage cfg.leios cfg.slotConfig (fst x).slot deliveryTime + let xs' = mapMaybe annotate xs -- TODO what to do with early/late arrivals? + unless (null xs') $ dispatch $! ValidateIBs xs' completion let submitVote (map snd -> xs) deliveryTime completion = do traceReceived xs EventVote dispatch $! ValidateVotes xs deliveryTime $ completion . map (\v -> (v.id, v)) @@ -516,7 +530,10 @@ leiosNode tracer cfg followers peers = do -- TODO: could expire EBs not referenced by young enough EBs. cfg.leios.variant /= Full ] - , [ pruneExpiredIBs tracer cfg leiosState + , [ pruneRelayIBState tracer cfg leiosState + | CleanupExpiredIb `isEnabledIn` cfg.leios.cleanupPolicies + ] + , [ pruneIBsForEBsAndVotesVar tracer cfg leiosState | CleanupExpiredIb `isEnabledIn` cfg.leios.cleanupPolicies ] ] @@ -533,17 +550,15 @@ leiosNode tracer cfg followers peers = do , pruningThreads ] --- Actually prunes IBs we should stop delivering. -pruneExpiredIBs :: (Monad m, MonadDelay m, MonadSTM m, MonadTime m) => Tracer m LeiosNodeEvent -> LeiosNodeConfig -> LeiosNodeState m -> m () -pruneExpiredIBs _tracer LeiosNodeConfig{leios, slotConfig} st = go (toEnum 0) +-- Prunes 'relayIBState' +pruneRelayIBState :: (Monad m, MonadDelay m, MonadSTM m, MonadTime m) => Tracer m LeiosNodeEvent -> LeiosNodeConfig -> LeiosNodeState m -> m () +pruneRelayIBState _tracer LeiosNodeConfig{leios, slotConfig} st = go (toEnum 0) where go p = do - -- TODO: could end when Endorse ends, but we want them around for voting. - let endOfIBDiffusion = succ $ lastVoteSend leios p + let expiry = succ $ lastEndorse leios p let pruneTo = succ $ snd $ proposeRange leios p - _ <- waitNextSlot slotConfig endOfIBDiffusion + _ <- waitNextSlot slotConfig expiry _ibsPruned <- atomically $ do - writeTVar st.prunedIBStateToVar $! pruneTo partitionRBVar st.relayIBState.relayBufferVar $ \ibEntry -> (fst ibEntry.value).slot < pruneTo -- TODO: batch these, too many events @@ -551,6 +566,19 @@ pruneExpiredIBs _tracer LeiosNodeConfig{leios, slotConfig} st = go (toEnum 0) -- traceWith tracer $! LeiosNodeEvent Pruned (EventIB ib) go (succ p) +pruneIBsForEBsAndVotesVar :: (Monad m, MonadDelay m, MonadSTM m, MonadTime m) => Tracer m LeiosNodeEvent -> LeiosNodeConfig -> LeiosNodeState m -> m () +pruneIBsForEBsAndVotesVar _tracer LeiosNodeConfig{leios, slotConfig} st = go (toEnum 0) + where + go p = do + let expiry = succ $ lastVoteSend leios $ (if leios.lateIbInclusion then succ . succ else id) $ p + _ <- waitNextSlot slotConfig expiry + _ibsPruned <- atomically $ do + modifyTVar' + st.iBsForEBsAndVotesVar + (snd . Map.split p) + -- TODO Pruned events + go (succ p) + -- rEB slots after the end of Endorse, -- prune EBs that became certified but were not adopted by an RB. pruneExpiredUnadoptedEBs :: @@ -665,13 +693,10 @@ pruneExpiredVotes :: pruneExpiredVotes _tracer LeiosNodeConfig{leios = leios@LeiosConfig{pipeline = _ :: SingPipeline p}, slotConfig} st = go (toEnum 0) where go p = do - let pruneIBDeliveryTo = succ $ snd (stageRangeOf @p leios p Short.Propose) let pruneTo = succ (lastVoteSend leios p) _ <- waitNextSlot slotConfig (succ (lastVoteRecv leios p)) _votesPruned <- atomically $ do writeTVar st.prunedVoteStateToVar $! pruneTo - -- delivery times for IBs are only needed to vote, so they can be pruned too. - modifyTVar' st.ibDeliveryTimesVar $ Map.filter $ \(slot, _) -> slot >= pruneIBDeliveryTo partitionRBVar st.relayVoteState.relayBufferVar $ \voteEntry -> let voteSlot = (snd voteEntry.value).slot @@ -782,20 +807,27 @@ validateIBsOfCertifiedEBs _trace _cfg st = forever . atomically $ do modifyTVar' st.ibsValidationActionsVar $ Map.delete ibId m -adoptIB :: MonadSTM m => LeiosNodeState m -> InputBlock -> UTCTime -> STM m () -adoptIB leiosState ib deliveryTime = do +-- | This is called once the IB has been validated +-- +-- An IB might be validated a long while after it arrived. +-- +-- An IB that arrived later than it should have will not even be validated. +adoptIB :: MonadSTM m => LeiosConfig -> LeiosNodeState m -> InputBlock -> IbDeliveryStage -> STM m () +adoptIB cfg leiosState ib deliveryStage = do let !ibSlot = ib.header.slot - -- NOTE: voting relies on delivery times for IBs + !p = case cfg of + LeiosConfig{pipeline = _ :: SingPipeline p} -> + pipelineOf @p cfg Short.Propose ibSlot modifyTVar' - leiosState.ibDeliveryTimesVar - (Map.insertWith (\(_, x) (s, y) -> (,) s $! min x y) ib.id (ibSlot, deliveryTime)) + leiosState.iBsForEBsAndVotesVar + (Map.insertWith (Map.unionWith min) p $ Map.singleton ib.id deliveryStage) -- TODO: likely needs optimization, although EBs also grow slowly. modifyTVar' leiosState.ibsNeededForEBVar (Map.map (Set.delete ib.id)) adoptEB :: MonadSTM m => LeiosNodeState m -> EndorseBlock -> STM m () adoptEB leiosState eb = do - ibs <- RB.keySet <$> readTVar leiosState.relayIBState.relayBufferVar + ibs <- Set.unions . Map.map Map.keysSet <$> readTVar leiosState.iBsForEBsAndVotesVar let ibsNeeded = Map.fromList [(eb.id, Set.fromList eb.inputBlocks Set.\\ ibs)] modifyTVar' leiosState.ibsNeededForEBVar (`Map.union` ibsNeeded) @@ -821,12 +853,12 @@ dispatchValidation tracer cfg leiosState req = valRB rb m = do let task prefix = cpuTask prefix cfg.leios.praos.blockValidationDelay rb labelTask (ValRB, (task, m)) - valIB x@(uncurry InputBlock -> ib) deliveryTime completion = + valIB x@(uncurry InputBlock -> ib) deliveryStage completion = let delay prefix = cpuTask prefix cfg.leios.delays.inputBlockValidation ib task = atomically $ do completion [x] - adoptIB leiosState ib deliveryTime + adoptIB cfg.leios leiosState ib deliveryStage in labelTask (ValIB, (delay, task >> traceEnterState [uncurry InputBlock x] EventIB)) valEB eb completion = labelTask . (ValEB,) . (\p -> cpuTask p cfg.leios.delays.endorseBlockValidation eb,) $ do @@ -856,7 +888,7 @@ dispatchValidation tracer cfg leiosState req = else leiosState.waitingForLedgerStateVar waitFor var [(prev, [queue [task]])] return [] - ValidateIBS ibs deliveryTime completion -> do + ValidateIBs ibs completion -> do -- NOTE: IBs with an RB reference have to wait for ledger state of that RB. -- However, if they get referenced by the chain they should be validated anyway. -- We use a map to store the validation logic, so we can force it happening in the latter case. @@ -868,14 +900,14 @@ dispatchValidation tracer cfg leiosState req = Map.delete ibId m Nothing -> pure () - let storeAction rbHash ib@(h, _) = do + let storeAction rbHash ib@(h, _) deliveryStage = do modifyTVar' leiosState.ibsValidationActionsVar $ - Map.insert h.id (queue [valIB ib deliveryTime completion]) + Map.insert h.id (queue [valIB ib deliveryStage completion]) return (rbHash, [lookupValAction $ (fst ib).id]) waitingLedgerState <- sequence $ - [ storeAction rbHash ib - | ib <- ibs + [ storeAction rbHash ib deliveryStage + | (ib, deliveryStage) <- ibs , BlockHash rbHash <- [(fst ib).rankingBlock] ] @@ -884,7 +916,7 @@ dispatchValidation tracer cfg leiosState req = leiosState.waitingForLedgerStateVar waitingLedgerState - return [valIB ib deliveryTime completion | ib@(h, _) <- ibs, GenesisHash <- [h.rankingBlock]] + return [valIB ib deliveryStage completion | (ib@(h, _), deliveryStage) <- ibs, GenesisHash <- [h.rankingBlock]] ValidateEBS ebs completion -> do -- NOTE: block references are only inspected during voting. return [valEB eb completion | eb <- ebs] @@ -922,10 +954,10 @@ generator tracer cfg st = do traceWith tracer (PraosNodeEvent (PraosNodeEventGenerate rb)) traceWith tracer (PraosNodeEvent (PraosNodeEventNewTip newChain)) SomeAction Generate.Propose{} ib -> (GenIB,) $ do - now <- getCurrentTime atomically $ do + -- TODO should not be added to 'relayIBState' before it's validated modifyTVar' st.relayIBState.relayBufferVar (RB.snocIfNew ib.header.id (ib.header, ib.body)) - adoptIB st ib now + adoptIB cfg.leios st ib IbDuringProposeOrDeliver1 traceWith tracer (LeiosNodeEvent Generate (EventIB ib)) SomeAction Generate.Endorse eb -> (GenEB,) $ do atomically $ do @@ -952,7 +984,6 @@ mkBuffersView cfg st = BuffersView{..} chain <- PraosNode.preferredChain st.praosState bufferEB <- readTVar st.relayEBState.relayBufferVar votesForEB <- readTVar st.votesForEBVar - ibsNeededForEB <- readTVar st.ibsNeededForEBVar -- RBs in the same chain should not contain certificates for the same pipeline. let pipelinesInChain = Set.fromList $ @@ -964,9 +995,6 @@ mkBuffersView cfg st = BuffersView{..} let tryCertify eb = do Certified{cert} <- Map.lookup eb.id votesForEB guard (not $ Set.member (endorseBlockPipeline cfg.leios eb) pipelinesInChain) - -- Note: we expect to have received the IBs for any - -- certified EB, but degraded network could mean we do not. - guard (Map.lookup eb.id ibsNeededForEB == Just Set.empty) return $! (eb.id,) $! cert -- TODO: cache index of EBs ordered by .slot? @@ -995,20 +1023,25 @@ mkBuffersView cfg st = BuffersView{..} let txsPayload = cfg.leios.sizes.inputBlockBodyAvgSize return $ NewInputBlockData{referenceRankingBlock, txsPayload} ibs = do - buffer <- readTVar st.relayIBState.relayBufferVar - times <- readTVar st.ibDeliveryTimesVar - let generatedCheck r = - map (.id) - . filter (\ib -> ib.slot `inRange` r) - . map fst - . RB.values - $ buffer - receivedByCheck slot = - filter - ( maybe False ((<= slotTime cfg.slotConfig slot) . snd) - . flip Map.lookup times - ) - validInputBlocks q = receivedByCheck q.receivedBy $ generatedCheck q.generatedBetween + let splitLE k m = + let (lt, mbEq, _gt) = Map.splitLookup k m + in case mbEq of + Nothing -> lt + Just eq -> Map.insert k eq lt + splitGE k m = + let (_lt, mbEq, gt) = Map.splitLookup k m + in case mbEq of + Nothing -> gt + Just eq -> Map.insert k eq gt + generatedCheck (lo, hi) = + Map.unions + . splitGE lo + . splitLE hi + receivedByCheck hi = + mapMaybe (\(ibId, deliveryStage) -> do guard (deliveryStage <= hi); Just ibId) + . Map.toList + xs <- readTVar st.iBsForEBsAndVotesVar + let validInputBlocks q = receivedByCheck q.receivedBy . generatedCheck q.generatedBetween $ xs return InputBlocksSnapshot{..} ebs = do buffer <- readTVar st.relayEBState.relayBufferVar diff --git a/simulation/src/LeiosProtocol/Short/Sim.hs b/simulation/src/LeiosProtocol/Short/Sim.hs index 6efd57392..0d3f4f884 100644 --- a/simulation/src/LeiosProtocol/Short/Sim.hs +++ b/simulation/src/LeiosProtocol/Short/Sim.hs @@ -427,6 +427,7 @@ traceRelayLink1 connectionOptions = , cleanupPolicies = def , variant = Short , headerDiffusionTime = 1 + , lateIbInclusion = False , pipelinesToReferenceFromEB = 0 , activeVotingStageLength = 1 , pipeline = SingSingleVote diff --git a/simulation/src/LeiosProtocol/SimTestRelay.hs b/simulation/src/LeiosProtocol/SimTestRelay.hs index 6eeea453a..86b7684a9 100644 --- a/simulation/src/LeiosProtocol/SimTestRelay.hs +++ b/simulation/src/LeiosProtocol/SimTestRelay.hs @@ -138,7 +138,7 @@ relayNode let relayConsumerConfig = RelayConsumerConfig { relay = relayConfig - , shouldIgnore = atomically $ do + , shouldNotRequest = atomically $ do rb <- readTVar relayBufferVar return $ \hd -> RB.member (testHeaderId hd) rb , -- sequential validation of headers diff --git a/simulation/test/Main.hs b/simulation/test/Main.hs index 0969f5796..25ce1f8c1 100644 --- a/simulation/test/Main.hs +++ b/simulation/test/Main.hs @@ -1,6 +1,7 @@ module Main where import qualified Test.Config +import qualified Test.ShortToFull import Test.Tasty (defaultMain, testGroup) import qualified Test.Topology @@ -9,4 +10,5 @@ main = defaultMain . testGroup "ouroboros-leios-sim" $ [ Test.Topology.tests , Test.Config.tests + , Test.ShortToFull.tests ] diff --git a/simulation/test/Test/ShortToFull.hs b/simulation/test/Test/ShortToFull.hs new file mode 100644 index 000000000..9f9b47103 --- /dev/null +++ b/simulation/test/Test/ShortToFull.hs @@ -0,0 +1,85 @@ +{-# LANGUAGE OverloadedRecordDot #-} +{-# LANGUAGE ParallelListComp #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +module Test.ShortToFull where + +import Data.Default (def) +import Data.List (sort) +import LeiosProtocol.Common (InputBlockId (..), NodeId (..), SlotNo) +import LeiosProtocol.Config (Config) +import qualified LeiosProtocol.Short as Short +import Test.QuickCheck as Q +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) + +tests :: TestTree +tests = + testGroup + "ShortToFull" + [ testProperty "test_leiosLateIbInclusion" test_leiosLateIbInclusion + ] + +data Test_leiosLateIbInclusion = Test_leiosLateIbInclusion + { ibTable :: [(Int, Short.PipelineNo, Short.IbDeliveryStage)] -- the Int is @zip [0..]@, /before shrinking/ + , ebLo :: SlotNo + -- ^ this first slot in which an EB could actually exist + , ebSlot :: SlotNo + } + deriving (Show) + +instance Q.Arbitrary Test_leiosLateIbInclusion where + arbitrary = do + let cfg = Short.convertConfig def + let (ebLo, ebHi) = case cfg of + Short.LeiosConfig{pipeline = _ :: Short.SingPipeline p} -> + Short.stageRangeOf @p cfg (toEnum 0) Short.Endorse + ebSlot <- toEnum <$> Q.choose (fromEnum ebLo, fromEnum ebHi + 9 * cfg.sliceLength) + ibTable <- do + n <- Q.choose (0, 1000) + gens <- Q.vectorOf n $ fmap toEnum $ Q.choose (0, 20) + delays <- Q.vectorOf n $ Q.elements [minBound .. maxBound] + pure $ [(i, gen, delay) | i <- [0 ..] | gen <- sort gens | delay <- delays] + pure Test_leiosLateIbInclusion{..} + + shrink testSetup = + [ testSetup{ibTable = ibTable'} + | ibTable' <- shrinkList (const []) testSetup.ibTable + ] + +test_leiosLateIbInclusion :: Test_leiosLateIbInclusion -> Property +test_leiosLateIbInclusion testSetup = + Q.counterexample (show (cfg.sliceLength, iterations)) + $ Q.counterexample ("ebPipeline " <> show (fromEnum ebSlot `div` cfg.sliceLength)) + $ Q.counterexample ("on " <> show (map (.num) on)) + $ Q.counterexample + (unlines $ ("off" :) $ map (show . map (.num) . off) $ iterations) + $ on Q.=== concatMap off iterations + where + Test_leiosLateIbInclusion{..} = testSetup + cfg = Short.convertConfig def + ibSnapshot = + Short.InputBlocksSnapshot + { validInputBlocks = \q -> + [ InputBlockId{node = NodeId 0, num = i} + | (i, gen, recv) <- ibTable + , gen `Short.inRange` q.generatedBetween + , recv <= q.receivedBy + ] + } + on = + Short.inputBlocksToEndorse + cfg{Short.lateIbInclusion = True} + ebSlot + ibSnapshot + off sl = + Short.inputBlocksToEndorse + cfg{Short.lateIbInclusion = False} + sl + ibSnapshot + iterations = + let capL = fromIntegral cfg.sliceLength + in -- discard underflow and negative iterations + dropWhile (\sl -> sl > ebSlot || sl < ebLo) [ebSlot - 2 * capL, ebSlot - capL, ebSlot]