Merge bitpay's upstream

.gitignore

@@ -132,3 +132,4 @@ dash-cli
 dashd
 dash-qt
 make
+wallet-utility

Makefile.am

@@ -12,6 +12,7 @@ endif
 BITCOIND_BIN=$(top_builddir)/src/dashd$(EXEEXT)
 BITCOIN_QT_BIN=$(top_builddir)/src/qt/dash-qt$(EXEEXT)
 BITCOIN_CLI_BIN=$(top_builddir)/src/dash-cli$(EXEEXT)
+WALLET_UTILITY_BIN=$(top_builddir)/src/wallet-utility$(EXEEXT)
 BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT)
 
 OSX_APP=Dash-Qt.app
@@ -63,6 +64,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release
+	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(WALLET_UTILITY_BIN) $(top_builddir)/release
 	@test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \
 	  echo error: could not build $@
 	@echo built $@
@@ -145,6 +147,9 @@ $(BITCOIND_BIN): FORCE
 $(BITCOIN_CLI_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
+$(WALLET_UTILITY_BIN): FORCE
+	$(MAKE) -C src $(@F)
+
 if USE_LCOV
 
 baseline.info:

configure.ac

@@ -191,7 +191,7 @@ CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS"
 
 AC_ARG_WITH([utils],
   [AS_HELP_STRING([--with-utils],
-  [build dash-cli dash-tx (default=yes)])],
+  [build dash-cli dash-tx wallet-utility (default=yes)])],
   [build_bitcoin_utils=$withval],
   [build_bitcoin_utils=yes])
 
@@ -781,7 +781,7 @@ AC_MSG_CHECKING([whether to build dashd])
 AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes])
 AC_MSG_RESULT($build_bitcoind)
 
-AC_MSG_CHECKING([whether to build utils (dash-cli dash-tx)])
+AC_MSG_CHECKING([whether to build utils (dash-cli dash-tx wallet-utility)])
 AM_CONDITIONAL([BUILD_BITCOIN_UTILS], [test x$build_bitcoin_utils = xyes])
 AC_MSG_RESULT($build_bitcoin_utils)
 

qa/rpc-tests/addressindex.py

@@ -171,7 +171,7 @@ def run_test(self):
         assert_equal(balance2["balance"], change_amount)
 
         # Check that deltas are returned correctly
-        deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200})
+        deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 1, "end": 200})
         balance3 = 0
         for delta in deltas:
             balance3 += delta["satoshis"]
@@ -323,6 +323,27 @@ def run_test(self):
         mempool_deltas = self.nodes[2].getaddressmempool({"addresses": [address1]})
         assert_equal(len(mempool_deltas), 2)
 
+        # Include chaininfo in results
+        print "Testing results with chain info..."
+
+        deltas_with_info = self.nodes[1].getaddressdeltas({
+            "addresses": [address2],
+            "start": 1,
+            "end": 200,
+            "chainInfo": True
+        })
+        start_block_hash = self.nodes[1].getblockhash(1);
+        end_block_hash = self.nodes[1].getblockhash(200);
+        assert_equal(deltas_with_info["start"]["height"], 1)
+        assert_equal(deltas_with_info["start"]["hash"], start_block_hash)
+        assert_equal(deltas_with_info["end"]["height"], 200)
+        assert_equal(deltas_with_info["end"]["hash"], end_block_hash)
+
+        utxos_with_info = self.nodes[1].getaddressutxos({"addresses": [address2], "chainInfo": True})
+        expected_tip_block_hash = self.nodes[1].getblockhash(267);
+        assert_equal(utxos_with_info["height"], 267)
+        assert_equal(utxos_with_info["hash"], expected_tip_block_hash)
+
         print "Passed\n"
 
 

qa/rpc-tests/spentindex.py

@@ -104,14 +104,34 @@ def run_test(self):
         assert_equal(txVerbose3["vin"][0]["valueSat"], amount)
 
         # Check the database index
-        self.nodes[0].generate(1)
+        block_hash = self.nodes[0].generate(1)
         self.sync_all()
 
         txVerbose4 = self.nodes[3].getrawtransaction(txid2, 1)
         assert_equal(txVerbose4["vin"][0]["address"], address2)
         assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"]))
         assert_equal(txVerbose4["vin"][0]["valueSat"], amount)
 
+
+        # Check block deltas
+        print "Testing getblockdeltas..."
+
+        block = self.nodes[3].getblockdeltas(block_hash[0])
+        assert_equal(len(block["deltas"]), 2)
+        assert_equal(block["deltas"][0]["index"], 0)
+        assert_equal(len(block["deltas"][0]["inputs"]), 0)
+        assert_equal(len(block["deltas"][0]["outputs"]), 0)
+        assert_equal(block["deltas"][1]["index"], 1)
+        assert_equal(block["deltas"][1]["txid"], txid2)
+        assert_equal(block["deltas"][1]["inputs"][0]["index"], 0)
+        assert_equal(block["deltas"][1]["inputs"][0]["address"], "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3")
+        assert_equal(block["deltas"][1]["inputs"][0]["satoshis"], amount * -1)
+        assert_equal(block["deltas"][1]["inputs"][0]["prevtxid"], txid)
+        assert_equal(block["deltas"][1]["inputs"][0]["prevout"], 0)
+        assert_equal(block["deltas"][1]["outputs"][0]["index"], 0)
+        assert_equal(block["deltas"][1]["outputs"][0]["address"], "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3")
+        assert_equal(block["deltas"][1]["outputs"][0]["satoshis"], amount)
+
         print "Passed\n"
 
 

qa/rpc-tests/timestampindex.py

@@ -35,15 +35,25 @@ def setup_network(self):
         self.sync_all()
 
     def run_test(self):
-        print "Mining 5 blocks..."
-        blockhashes = self.nodes[0].generate(5)
-        low = self.nodes[0].getblock(blockhashes[0])["time"]
-        high = self.nodes[0].getblock(blockhashes[4])["time"]
+        print "Mining 25 blocks..."
+        blockhashes = self.nodes[0].generate(25)
+        time.sleep(3)
+        print "Mining 25 blocks..."
+        blockhashes.extend(self.nodes[0].generate(25))
+        time.sleep(3)
+        print "Mining 25 blocks..."
+        blockhashes.extend(self.nodes[0].generate(25))
         self.sync_all()
+        low = self.nodes[1].getblock(blockhashes[0])["time"]
+        high = low + 76
+
         print "Checking timestamp index..."
         hashes = self.nodes[1].getblockhashes(high, low)
-        assert_equal(len(hashes), 5)
-        assert_equal(sorted(blockhashes), sorted(hashes))
+
+        assert_equal(len(hashes), len(blockhashes))
+
+        assert_equal(hashes, blockhashes)
+
         print "Passed\n"
 
 

src/Makefile.am

@@ -60,6 +60,9 @@ endif
 
 if BUILD_BITCOIN_UTILS
   bin_PROGRAMS += dash-cli dash-tx
+if ENABLE_WALLET
+  bin_PROGRAMS += wallet-utility
+endif
 endif
 
 .PHONY: FORCE check-symbols check-security
@@ -420,6 +423,14 @@ dash_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS)
 dash_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 dash_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
+# wallet-utility binary #
+if ENABLE_WALLET
+wallet_utility_SOURCES = wallet-utility.cpp
+wallet_utility_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAG)
+wallet_utility_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+wallet_utility_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+endif
+
 if TARGET_WINDOWS
 dash_cli_SOURCES += dash-cli-res.rc
 endif
@@ -429,6 +440,11 @@ dash_cli_LDADD = \
   $(LIBUNIVALUE) \
   $(LIBBITCOIN_UTIL)
 dash_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS)
+
+if ENABLE_WALLET
+wallet_utility_LDADD = libbitcoin_wallet.a $(LIBBITCOIN_COMMON) $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1) $(LIBBITCOIN_UTIL) $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS)
+endif
+
 #
 
 # dash-tx binary #

src/Makefile.test.include

@@ -17,7 +17,9 @@ EXTRA_DIST += \
 	test/data/txcreate2.hex \
 	test/data/txcreatedata1.hex \
 	test/data/txcreatedata2.hex \
-	test/data/txcreatesign.hex
+	test/data/txcreatesign.hex \
+	test/wallet-utility.py \
+	test/data/wallet.dat
 
 JSON_TEST_FILES = \
   test/data/script_tests.json \
@@ -132,6 +134,10 @@ dash_test_clean : FORCE
 check-local:
 	@echo "Running test/bitcoin-util-test.py..."
 	$(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(srcdir)/test/bitcoin-util-test.py
+if ENABLE_WALLET
+	@echo "Running test/wallet-utility.py..."
+	$(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(srcdir)/test/wallet-utility.py
+endif
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check
 

src/dbwrapper.cpp

@@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error)
     throw dbwrapper_error("Unknown database error");
 }
 
-static leveldb::Options GetOptions(size_t nCacheSize)
+static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles)
 {
     leveldb::Options options;
     options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
     options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
     options.filter_policy = leveldb::NewBloomFilterPolicy(10);
-    options.compression = leveldb::kNoCompression;
-    options.max_open_files = 64;
+    options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression;
+    options.max_open_files = maxOpenFiles;
     if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
         // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
         // on corruption in later versions.
@@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize)
     return options;
 }
 
-CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
+CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles)
 {
     penv = NULL;
     readoptions.verify_checksums = true;
     iteroptions.verify_checksums = true;
     iteroptions.fill_cache = false;
     syncoptions.sync = true;
-    options = GetOptions(nCacheSize);
+    options = GetOptions(nCacheSize, compression, maxOpenFiles);
     options.create_if_missing = true;
     if (fMemory) {
         penv = leveldb::NewMemEnv(leveldb::Env::Default());

src/dbwrapper.h

@@ -169,14 +169,16 @@ class CDBWrapper
 
 public:
     /**
-     * @param[in] path        Location in the filesystem where leveldb data will be stored.
-     * @param[in] nCacheSize  Configures various leveldb cache settings.
-     * @param[in] fMemory     If true, use leveldb's memory environment.
-     * @param[in] fWipe       If true, remove all existing data.
-     * @param[in] obfuscate   If true, store data obfuscated via simple XOR. If false, XOR
-     *                        with a zero'd byte array.
+     * @param[in] path          Location in the filesystem where leveldb data will be stored.
+     * @param[in] nCacheSize    Configures various leveldb cache settings.
+     * @param[in] fMemory       If true, use leveldb's memory environment.
+     * @param[in] fWipe         If true, remove all existing data.
+     * @param[in] obfuscate     If true, store data obfuscated via simple XOR. If false, XOR
+     *                          with a zero'd byte array.
+     * @param[in] compression   Enable snappy compression for the database
+     * @param[in] maxOpenFiles  The maximum number of open files for the database
      */
-    CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
+    CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64);
     ~CDBWrapper();
 
     template <typename K, typename V>

src/init.cpp

@@ -1445,18 +1445,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
         }
     }
 
+    // block tree db settings
+    int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES);
+    bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION);
+
+    LogPrintf("Block index database configuration:\n");
+    LogPrintf("* Using %d max open files\n", dbMaxOpenFiles);
+    LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled");
+
     // cache size calculations
     int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20);
     nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
     nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache
     int64_t nBlockTreeDBCache = nTotalCache / 8;
-    if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX))
-        nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
+    if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) {
+        // enable 3/4 of the cache if addressindex and/or spentindex is enabled
+        nBlockTreeDBCache = nTotalCache * 3 / 4;
+    } else {
+        if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
+            nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
+        }
+    }
     nTotalCache -= nBlockTreeDBCache;
     int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
     nTotalCache -= nCoinDBCache;
     nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
     LogPrintf("Cache configuration:\n");
+    LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache);
     LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024));
@@ -1477,7 +1492,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
                 delete pcoinscatcher;
                 delete pblocktree;
 
-                pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
+                pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles);
                 pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
                 pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
                 pcoinsTip = new CCoinsViewCache(pcoinscatcher);

src/main.cpp

@@ -1509,12 +1509,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
     return res;
 }
 
-bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector<uint256> &hashes)
+bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, const bool fActiveOnly, std::vector<std::pair<uint256, unsigned int> > &hashes)
 {
     if (!fTimestampIndex)
         return error("Timestamp index not enabled");
 
-    if (!pblocktree->ReadTimestampIndex(high, low, hashes))
+    if (!pblocktree->ReadTimestampIndex(high, low, fActiveOnly, hashes))
         return error("Unable to get hashes for timestamps");
 
     return true;
@@ -2831,10 +2831,27 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
         if (!pblocktree->UpdateSpentIndex(spentIndex))
             return AbortNode(state, "Failed to write transaction index");
 
-    if (fTimestampIndex)
-        if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash())))
+    if (fTimestampIndex) {
+        unsigned int logicalTS = pindex->nTime;
+        unsigned int prevLogicalTS = 0;
+
+        // retrieve logical timestamp of the previous block
+        if (pindex->pprev)
+            if (!pblocktree->ReadTimestampBlockIndex(pindex->pprev->GetBlockHash(), prevLogicalTS))
+                LogPrintf("%s: Failed to read previous block's logical timestamp\n", __func__);
+
+        if (logicalTS <= prevLogicalTS) {
+            logicalTS = prevLogicalTS + 1;
+            LogPrintf("%s: Previous logical timestamp is newer Actual[%d] prevLogical[%d] Logical[%d]\n", __func__, pindex->nTime, prevLogicalTS, logicalTS);
+        }
+
+        if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(logicalTS, pindex->GetBlockHash())))
             return AbortNode(state, "Failed to write timestamp index");
 
+        if (!pblocktree->WriteTimestampBlockIndex(CTimestampBlockIndexKey(pindex->GetBlockHash()), CTimestampBlockIndexValue(logicalTS)))
+            return AbortNode(state, "Failed to write blockhash index");
+    }
+
     // add this block to the view's block chain
     view.SetBestBlock(pindex->GetBlockHash());