diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1907e2fa78439..5f1f175b836cb 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error) throw dbwrapper_error("Unknown database error"); } -static leveldb::Options GetOptions(size_t nCacheSize) +static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles) { leveldb::Options options; options.block_cache = leveldb::NewLRUCache(nCacheSize / 2); options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously options.filter_policy = leveldb::NewBloomFilterPolicy(10); - options.compression = leveldb::kNoCompression; - options.max_open_files = 64; + options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression; + options.max_open_files = maxOpenFiles; if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. @@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles) { penv = NULL; readoptions.verify_checksums = true; iteroptions.verify_checksums = true; iteroptions.fill_cache = false; syncoptions.sync = true; - options = GetOptions(nCacheSize); + options = GetOptions(nCacheSize, compression, maxOpenFiles); options.create_if_missing = true; if (fMemory) { penv = leveldb::NewMemEnv(leveldb::Env::Default()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5e7313f7eb544..2acd5fc05e07d 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -169,14 +169,16 @@ class CDBWrapper public: /** - * @param[in] path Location in the filesystem where leveldb data will be stored. - * @param[in] nCacheSize Configures various leveldb cache settings. - * @param[in] fMemory If true, use leveldb's memory environment. - * @param[in] fWipe If true, remove all existing data. - * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR - * with a zero'd byte array. + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + * @param[in] compression Enable snappy compression for the database + * @param[in] maxOpenFiles The maximum number of open files for the database */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64); ~CDBWrapper(); template diff --git a/src/init.cpp b/src/init.cpp index 3893500926d6d..4d952ada06399 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1294,18 +1294,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // block tree db settings + int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES); + bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION); + + LogPrintf("Block index database configuration:\n"); + LogPrintf("* Using %d max open files\n", dbMaxOpenFiles); + LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled"); + // cache size calculations int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache int64_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) - nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) { + // enable 3/4 of the cache if addressindex and/or spentindex is enabled + nBlockTreeDBCache = nTotalCache * 3 / 4; + } else { + if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + } + } nTotalCache -= nBlockTreeDBCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache LogPrintf("Cache configuration:\n"); + LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); @@ -1326,7 +1341,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) delete pcoinscatcher; delete pblocktree; - pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); + pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); diff --git a/src/main.h b/src/main.h index 3f8a3fb32a219..1361a75f0e666 100644 --- a/src/main.h +++ b/src/main.h @@ -115,6 +115,8 @@ static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; static const bool DEFAULT_SPENTINDEX = false; +static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000; +static const bool DEFAULT_DB_COMPRESSION = true; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index ee39fbefd6254..64050694e5469 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -59,8 +59,10 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud out.push_back(Pair("addresses", a)); } -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +void TxToJSONExpanded(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, + int nHeight = 0, int nConfirmations = 0, int nBlockTime = 0) { + uint256 txid = tx.GetHash(); entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); @@ -121,6 +123,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) } entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { + entry.push_back(Pair("blockhash", hashBlock.GetHex())); + + if (nConfirmations > 0) { + entry.push_back(Pair("height", nHeight)); + entry.push_back(Pair("confirmations", nConfirmations)); + entry.push_back(Pair("time", nBlockTime)); + entry.push_back(Pair("blocktime", nBlockTime)); + } else { + entry.push_back(Pair("height", -1)); + entry.push_back(Pair("confirmations", 0)); + } + } + +} + +void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +{ + + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); + entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); + entry.push_back(Pair("version", tx.nVersion)); + entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); + + UniValue vin(UniValue::VARR); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + UniValue in(UniValue::VOBJ); + if (tx.IsCoinBase()) + in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + else { + in.push_back(Pair("txid", txin.prevout.hash.GetHex())); + in.push_back(Pair("vout", (int64_t)txin.prevout.n)); + UniValue o(UniValue::VOBJ); + o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); + o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + in.push_back(Pair("scriptSig", o)); + } + in.push_back(Pair("sequence", (int64_t)txin.nSequence)); + vin.push_back(in); + } + entry.push_back(Pair("vin", vin)); + + UniValue vout(UniValue::VARR); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + const CTxOut& txout = tx.vout[i]; + UniValue out(UniValue::VOBJ); + out.push_back(Pair("value", ValueFromAmount(txout.nValue))); + out.push_back(Pair("valueSat", txout.nValue)); + out.push_back(Pair("n", (int64_t)i)); + UniValue o(UniValue::VOBJ); + ScriptPubKeyToJSON(txout.scriptPubKey, o, true); + out.push_back(Pair("scriptPubKey", o)); + vout.push_back(out); + } + entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); BlockMap::iterator mi = mapBlockIndex.find(hashBlock); @@ -206,8 +265,6 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") ); - LOCK(cs_main); - uint256 hash = ParseHashV(params[0], "parameter 1"); bool fVerbose = false; @@ -215,9 +272,31 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) fVerbose = (params[1].get_int() != 0); CTransaction tx; + uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + int nHeight = 0; + int nConfirmations = 0; + int nBlockTime = 0; + + { + LOCK(cs_main); + if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + + BlockMap::iterator mi = mapBlockIndex.find(hashBlock); + if (mi != mapBlockIndex.end() && (*mi).second) { + CBlockIndex* pindex = (*mi).second; + if (chainActive.Contains(pindex)) { + nHeight = pindex->nHeight; + nConfirmations = 1 + chainActive.Height() - pindex->nHeight; + nBlockTime = pindex->GetBlockTime(); + } else { + nHeight = -1; + nConfirmations = 0; + nBlockTime = pindex->GetBlockTime(); + } + } + } string strHex = EncodeHexTx(tx); @@ -226,7 +305,8 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", strHex)); - TxToJSON(tx, hashBlock, result); + TxToJSONExpanded(tx, hashBlock, result, nHeight, nConfirmations, nBlockTime); + return result; } diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index e399315870e79..466a0f4de4ed3 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -47,6 +47,49 @@ BOOST_AUTO_TEST_CASE(dbwrapper) } } +BOOST_AUTO_TEST_CASE(dbwrapper_compression) +{ + // Perform tests both with compression and without + for (int i = 0; i < 2; i++) { + bool compression = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, compression); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_64) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 64); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_1000) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 1000); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + // Test batch operations BOOST_AUTO_TEST_CASE(dbwrapper_batch) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 48150198f3a34..f204989f12605 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -33,7 +33,7 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true, false, 64) { } @@ -75,7 +75,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, false, compression, maxOpenFiles) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { diff --git a/src/txdb.h b/src/txdb.h index 14d501278f23e..b2a99e0b3eeb3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -54,7 +54,7 @@ class CCoinsViewDB : public CCoinsView class CBlockTreeDB : public CDBWrapper { public: - CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000); private: CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&);