Merge pull request #1270 from psgreco/master-trim-headers-v2

Restore full functionality to trim_headers (untrim)

Author: delta1

src/bench/rpc_blockchain.cpp

@@ -15,7 +15,7 @@
 namespace {
 
 struct TestBlockAndIndex {
-    const std::unique_ptr<const TestingSetup> testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)};
+    std::unique_ptr<TestingSetup> testing_setup{MakeNoLogFileContext<TestingSetup>(CBaseChainParams::MAIN)};
     CBlock block{};
     uint256 blockHash{};
     CBlockIndex blockindex{};
@@ -28,6 +28,7 @@ struct TestBlockAndIndex {
 
         stream >> block;
 
+        CBlockIndex::SetNodeContext(&(testing_setup->m_node));
         blockHash = block.GetHash();
         blockindex.phashBlock = &blockHash;
         blockindex.nBits = 403014710;

src/chain.cpp

@@ -5,6 +5,11 @@
 
 #include <chain.h>
 #include <util/time.h>
+#include <validation.h>
+#include <node/context.h>
+
+
+node::NodeContext *CBlockIndex::m_pcontext;
 
 std::string CBlockFileInfo::ToString() const
 {
@@ -51,6 +56,27 @@ CBlockLocator CChain::GetLocator(const CBlockIndex *pindex) const {
     return CBlockLocator(vHave);
 }
 
+void CBlockIndex::untrim() EXCLUSIVE_LOCKS_REQUIRED(::cs_main){
+    AssertLockHeld(::cs_main);
+    if (!trimmed())
+        return;
+    CBlockIndex tmp;
+    const CBlockIndex *pindexfull = untrim_to(&tmp);
+    assert(pindexfull!=this);
+    m_trimmed = false;
+    set_stored();
+    proof = pindexfull->proof;
+    m_dynafed_params = pindexfull->m_dynafed_params;
+    m_signblock_witness = pindexfull->m_signblock_witness;
+    m_pcontext->chainman->m_blockman.m_dirty_blockindex.insert(this);
+}
+
+const CBlockIndex *CBlockIndex::untrim_to(CBlockIndex *pindexNew) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
+{
+    AssertLockHeld(::cs_main);
+    return m_pcontext->chainman->m_blockman.m_block_tree_db->RegenerateFullIndex(this, pindexNew);
+}
+
 const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const {
     if (pindex == nullptr) {
         return nullptr;

src/chain.h

@@ -16,6 +16,9 @@
 
 #include <vector>
 
+namespace node {
+struct NodeContext;
+}
 /**
  * Maximum amount of time that a block timestamp is allowed to exceed the
  * current network-adjusted time before the block will be accepted.
@@ -215,26 +218,41 @@ class CBlockIndex
 
     bool m_trimmed{false};
     bool m_trimmed_dynafed_block{false};
+    bool m_stored_lvl{false};
 
     friend class CBlockTreeDB;
 
+    static node::NodeContext *m_pcontext;
+
 public:
+    static void SetNodeContext(node::NodeContext *context) {m_pcontext = context;};
 
     // Irrevocably remove blocksigning and dynafed-related stuff from this
     // in-memory copy of the block header.
-    void trim() {
+    bool trim() {
         assert_untrimmed();
+        if (!m_stored_lvl) {
+            // We can't trim in-memory data if it's not on disk yet, but we can if it's already been recovered once
+            return false;
+        }
         m_trimmed = true;
         m_trimmed_dynafed_block = !m_dynafed_params.value().IsNull();
         proof = std::nullopt;
         m_dynafed_params = std::nullopt;
         m_signblock_witness = std::nullopt;
+        return true;
     }
 
+    void untrim();
+    const CBlockIndex * untrim_to(CBlockIndex *pindexNew) const;
+
     inline bool trimmed() const {
         return m_trimmed;
     }
 
+    inline void set_stored() {
+        m_stored_lvl = true;
+    }
     inline void assert_untrimmed() const {
         assert(!m_trimmed);
     }
@@ -501,6 +519,9 @@ class CDiskBlockIndex : public CBlockIndex
 
         // For compatibility with elements 0.14 based chains
         if (g_signed_blocks) {
+            if (!ser_action.ForRead()) {
+                obj.assert_untrimmed();
+            }
             if (is_dyna) {
                 READWRITE(obj.m_dynafed_params.value());
                 READWRITE(obj.m_signblock_witness.value().stack);

src/dynafed.cpp

@@ -1,6 +1,8 @@
 
 #include <dynafed.h>
 #include <hash.h>
+#include <validation.h>
+#include <node/context.h>
 
 bool NextBlockIsParameterTransition(const CBlockIndex* pindexPrev, const Consensus::Params& consensus, DynaFedParamEntry& winning_entry)
 {
@@ -15,6 +17,10 @@ bool NextBlockIsParameterTransition(const CBlockIndex* pindexPrev, const Consens
     for (int32_t height = next_height - 1; height >= (int32_t)(next_height - consensus.dynamic_epoch_length); --height) {
         const CBlockIndex* p_epoch_walk = pindexPrev->GetAncestor(height);
         assert(p_epoch_walk);
+        if (node::fTrimHeaders) {
+            LOCK(cs_main);
+            ForceUntrimHeader(p_epoch_walk);
+        }
         const DynaFedParamEntry& proposal = p_epoch_walk->dynafed_params().m_proposed;
         const uint256 proposal_root = proposal.CalculateRoot();
         vote_tally[proposal_root]++;
@@ -60,6 +66,10 @@ DynaFedParamEntry ComputeNextBlockFullCurrentParameters(const CBlockIndex* pinde
     // may be pre-dynafed params
     const CBlockIndex* p_epoch_start = pindexPrev->GetAncestor(epoch_start_height);
     assert(p_epoch_start);
+    if (node::fTrimHeaders) {
+        LOCK(cs_main);
+        ForceUntrimHeader(p_epoch_start);
+    }
     if (p_epoch_start->dynafed_params().IsNull()) {
         // We need to construct the "full" current parameters of pre-dynafed
         // consensus
@@ -93,6 +103,10 @@ DynaFedParamEntry ComputeNextBlockCurrentParameters(const CBlockIndex* pindexPre
 {
     assert(pindexPrev);
 
+    if (node::fTrimHeaders) {
+        LOCK(cs_main);
+        ForceUntrimHeader(pindexPrev);
+    }
     DynaFedParamEntry entry = ComputeNextBlockFullCurrentParameters(pindexPrev, consensus);
 
     uint32_t next_height = pindexPrev->nHeight+1;

src/init.cpp

@@ -1011,13 +1011,13 @@ bool AppInitParameterInteraction(const ArgsManager& args)
     }
 
     if (args.GetBoolArg("-trim_headers", false)) {
-        LogPrintf("Configured for header-trimming mode. This will reduce memory usage substantially, but we will be unable to serve as a full P2P peer, and certain header fields may be missing from JSON RPC output.\n");
+        LogPrintf("Configured for header-trimming mode. This will reduce memory usage substantially, but will increase IO usage when the headers need to be temporarily untrimmed.\n");
         node::fTrimHeaders = true;
         // This calculation is driven by GetValidFedpegScripts in pegins.cpp, which walks the chain
         //   back to current epoch start, and then an additional total_valid_epochs on top of that.
         //   We add one epoch here for the current partial epoch, and then another one for good luck.
 
-        node::nMustKeepFullHeaders = (chainparams.GetConsensus().total_valid_epochs + 2) * epoch_length;
+        node::nMustKeepFullHeaders = chainparams.GetConsensus().total_valid_epochs * epoch_length;
         // This is the number of headers we can have in flight downloading at a time, beyond the
         //   set of blocks we've already validated. Capping this is necessary to keep memory usage
         //   bounded during IBD.
@@ -1243,6 +1243,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     const ArgsManager& args = *Assert(node.args);
     const CChainParams& chainparams = Params();
 
+    CBlockIndex::SetNodeContext(&node);
     auto opt_max_upload = ParseByteUnits(args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET), ByteUnit::M);
     if (!opt_max_upload) {
         return InitError(strprintf(_("Unable to parse -maxuploadtarget: '%s'"), args.GetArg("-maxuploadtarget", "")));
@@ -1712,7 +1713,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
 
     // if pruning, unset the service bit and perform the initial blockstore prune
     // after any wallet rescanning has taken place.
-    if (fPruneMode || node::fTrimHeaders) {
+    if (fPruneMode) {
         LogPrintf("Unsetting NODE_NETWORK on prune mode\n");
         nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK);
         if (!fReindex) {
@@ -1724,11 +1725,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
     }
 
-    if (node::fTrimHeaders) {
-        LogPrintf("Unsetting NODE_NETWORK_LIMITED on header trim mode\n");
-        nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK_LIMITED);
-    }
-
     // ********************************************************* Step 11: import blocks
 
     if (!CheckDiskSpace(gArgs.GetDataDirNet())) {

src/net_processing.cpp

@@ -3305,12 +3305,13 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
         for (; pindex; pindex = m_chainman.ActiveChain().Next(pindex))
         {
             if (pindex->trimmed()) {
-                // For simplicity, if any of the headers they're asking for are trimmed,
-                //   just drop the request.
-                LogPrint(BCLog::NET, "%s: ignoring getheaders from peer=%i which would return at least one trimmed header\n", __func__, pfrom.GetId());
-                return;
+                // Header is trimmed, reload from disk before sending
+                CBlockIndex tmpBlockIndexFull;
+                const CBlockIndex* pindexfull = pindex->untrim_to(&tmpBlockIndexFull);
+                vHeaders.push_back(pindexfull->GetBlockHeader());
+            } else {
+                vHeaders.push_back(pindex->GetBlockHeader());
             }
-            vHeaders.push_back(pindex->GetBlockHeader());
             if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop)
                 break;
         }

src/node/blockstorage.cpp

@@ -223,15 +223,7 @@ bool BlockManager::LoadBlockIndex(
 {
     int trim_below_height = 0;
     if (fTrimHeaders) {
-        int max_height = 0;
-        if (!m_block_tree_db->WalkBlockIndexGutsForMaxHeight(&max_height)) {
-            LogPrintf("LoadBlockIndex: Failed to WalkBlockIndexGutsForMaxHeight.\n");
-            return false;
-        }
-
-        int must_keep_headers = (consensus_params.total_valid_epochs + 2) * consensus_params.dynamic_epoch_length;
-        int extra_headers_buffer = consensus_params.dynamic_epoch_length * 2; // XXX arbitrary
-        trim_below_height = max_height - must_keep_headers - extra_headers_buffer;
+        trim_below_height = std::numeric_limits<int>::max();
     }
     if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, trim_below_height)) {
         return false;
@@ -337,6 +329,9 @@ bool BlockManager::LoadBlockIndex(
         }
     }
 
+    if (pindexBestHeader) {
+        ForceUntrimHeader(pindexBestHeader);
+    }
     return true;
 }
 
@@ -803,23 +798,6 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus
     return true;
 }
 
-bool ReadBlockHeaderFromDisk(CBlockHeader& header, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
-{
-    // Not very efficient: read a block and throw away all but the header.
-    CBlock tmp;
-    if (!ReadBlockFromDisk(tmp, pindex, consensusParams)) {
-        return false;
-    }
-    const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())};
-
-    header = tmp.GetBlockHeader();
-    if (tmp.GetHash() != pindex->GetBlockHash()) {
-        return error("ReadBlockheaderFromDisk(CBlockHeader&, CBlockIndex*): GetHash() doesn't match index for %s at %s",
-                     pindex->ToString(), block_pos.ToString());
-    }
-    return true;
-}
-
 bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start)
 {
     FlatFilePos hpos = pos;

src/node/blockstorage.h

@@ -76,6 +76,7 @@ class BlockManager
 {
     friend CChainState;
     friend ChainstateManager;
+    friend CBlockIndex;
 
 private:
     void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false);
@@ -193,7 +194,6 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P
 bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
 bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start);
 // ELEMENTS:
-bool ReadBlockHeaderFromDisk(class CBlockHeader& header, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
 
 bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex);
 

src/pegins.cpp

@@ -26,6 +26,8 @@
 // ELEMENTS
 //
 
+#include <validation.h>
+
 namespace {
 static secp256k1_context* secp256k1_ctx_validation;
 
@@ -487,6 +489,10 @@ std::vector<std::pair<CScript, CScript>> GetValidFedpegScripts(const CBlockIndex
             break;
         }
 
+        if (node::fTrimHeaders) {
+            LOCK(cs_main);
+            ForceUntrimHeader(p_epoch_start);
+        }
         if (!p_epoch_start->dynafed_params().IsNull()) {
             fedpegscripts.push_back(std::make_pair(p_epoch_start->dynafed_params().m_current.m_fedpeg_program, p_epoch_start->dynafed_params().m_current.m_fedpegscript));
         } else {

src/qt/test/rpcnestedtests.cpp

@@ -48,6 +48,7 @@ void RPCNestedTests::rpcNestedTests()
 
     TestingSetup test;
     m_node.setContext(&test.m_node);
+    CBlockIndex::SetNodeContext(&test.m_node);
 
     if (RPCIsInWarmup(nullptr)) SetRPCWarmupFinished();
 

src/rest.cpp

@@ -232,13 +232,10 @@ static bool rest_headers(const std::any& context,
     case RetFormat::BINARY: {
         CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
         for (const CBlockIndex *pindex : headers) {
-            if (pindex->trimmed()) {
-                CBlockHeader tmp;
-                node::ReadBlockHeaderFromDisk(tmp, pindex, Params().GetConsensus());
-                ssHeader << tmp;
-            } else {
-                ssHeader << pindex->GetBlockHeader();
-            }
+            LOCK(cs_main);
+            CBlockIndex tmpBlockIndexFull;
+            const CBlockIndex* pindexfull=pindex->untrim_to(&tmpBlockIndexFull);
+            ssHeader << pindexfull->GetBlockHeader();
         }
 
         std::string binaryHeader = ssHeader.str();
@@ -250,14 +247,10 @@ static bool rest_headers(const std::any& context,
     case RetFormat::HEX: {
         CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
         for (const CBlockIndex *pindex : headers) {
-            if (pindex->trimmed()) {
-                CBlockHeader tmp;
-                node::ReadBlockHeaderFromDisk(tmp, pindex, Params().GetConsensus());
-                ssHeader << tmp;
-
-            } else {
-                ssHeader << pindex->GetBlockHeader();
-            }
+            LOCK(cs_main);
+            CBlockIndex tmpBlockIndexFull;
+            const CBlockIndex* pindexfull=pindex->untrim_to(&tmpBlockIndexFull);
+            ssHeader << pindexfull->GetBlockHeader();
         }
 
         std::string strHex = HexStr(ssHeader) + "\n";