Skip to content

Commit 20a3f6e

Browse files
authored
Merge pull request #548 from bitpay/opt/txlist-cache
add txIdList cache
2 parents 11612e0 + ddda913 commit 20a3f6e

File tree

4 files changed

+287
-72
lines changed

4 files changed

+287
-72
lines changed

lib/services/address/index.js

Lines changed: 85 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
1313
var Transform = require('stream').Transform;
1414
var assert = require('assert');
1515
var utils = require('../../utils');
16+
var LRU = require('lru-cache');
17+
var XXHash = require('xxhash');
18+
19+
20+
21+
// See rationale about this cache at function getTxList(next)
22+
const TXID_LIST_CACHE_ITEMS = 250; // nr of items (this translates to: consecutive
23+
// clients downloading their tx history)
24+
const TXID_LIST_CACHE_EXPIRATION = 1000 * 30; // ms
25+
const TXID_LIST_CACHE_MIN = 100; // Min items to cache
26+
const TXID_LIST_CACHE_SEED = 0x3233DE; // Min items to cache
1627

1728
var AddressService = function(options) {
1829

@@ -24,6 +35,11 @@ var AddressService = function(options) {
2435
this._network = this.node.network;
2536
this._db = this.node.services.db;
2637
this._mempool = this.node.services.mempool;
38+
this._txIdListCache = new LRU({
39+
max: TXID_LIST_CACHE_ITEMS,
40+
maxAge: TXID_LIST_CACHE_EXPIRATION
41+
});
42+
2743

2844
if (this._network === 'livenet') {
2945
this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
4965
// for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
5066
// in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
5167
// then I would pass back [tx1, tx2] in that order
68+
//
69+
// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
70+
//
5271
AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
5372
var self = this;
73+
var cacheUsed = false;
5474

5575
options = options || {};
5676
options.from = options.from || 0;
@@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
6585
addresses = [addresses];
6686
}
6787

68-
async.eachLimit(addresses, 4, function(address, next) {
69-
self._getAddressTxidHistory(address, options, next);
7088

71-
}, function(err) {
89+
function getTxList(next) {
7290

73-
if(err) {
74-
return callback(err);
91+
92+
function hashAddresses(addresses) {
93+
94+
// Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
95+
// a 32 bits hash is secure enough
96+
97+
return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
98+
};
99+
100+
var calculatedCacheKey;
101+
102+
// We use the cache ONLY on from > 0 queries.
103+
//
104+
// Rationale: The a full history is downloaded, the client do
105+
// from =0, to=x
106+
// then from =x+1 to=y
107+
// then [...]
108+
// The objective of this cache is to speed up the from>0 queries, and also
109+
// "freeze" the txid list during download.
110+
//
111+
if (options.from >0 ) {
112+
113+
let cacheKey = options.cacheKey;
114+
if (!cacheKey) {
115+
calculatedCacheKey = hashAddresses(addresses);
116+
cacheKey = calculatedCacheKey;
117+
}
118+
119+
var txIdList = self._txIdListCache.get(cacheKey);
120+
if (txIdList) {
121+
options.txIdList = txIdList;
122+
cacheUsed = true;
123+
return next();
124+
}
75125
}
76126

77-
var list = lodash.uniqBy(options.txIdList, function(x) {
78-
return x.txid + x.height;
127+
// Get the list from the db
128+
async.eachLimit(addresses, 4, function(address, next) {
129+
self._getAddressTxidHistory(address, options, next);
130+
}, function(err) {
131+
if (err) return next(err);
132+
133+
var list = lodash.uniqBy(options.txIdList, function(x) {
134+
return x.txid + x.height;
135+
});
136+
137+
138+
options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
139+
140+
if (list.length > TXID_LIST_CACHE_MIN) {
141+
calculatedCacheKey = calculatedCacheKey || hashAddresses(addresses);
142+
143+
self._txIdListCache.set(calculatedCacheKey, options.txIdList);
144+
}
145+
146+
return next();
79147
});
80148

149+
};
150+
151+
152+
getTxList(function(err) {
153+
if(err) {
154+
return callback(err);
155+
}
81156

82-
options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
83157
self._getAddressTxHistory(options, function(err, txList) {
84158

85159
if (err) {
@@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
88162

89163
var results = {
90164
totalCount: options.txIdList.length || 0,
91-
items: txList
165+
items: txList,
92166
};
93167

94-
callback(null, results);
168+
// cacheUsed is returned for testing
169+
callback(null, results, cacheUsed);
95170

96171
});
97172
});

package-lock.json

Lines changed: 81 additions & 60 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,13 @@
4343
"levelup": "^2.0.0",
4444
"liftoff": "^2.2.0",
4545
"lodash": "^4.17.4",
46-
"lru-cache": "^4.0.2",
46+
"lru-cache": "^4.1.1",
4747
"memwatch-next": "^0.3.0",
4848
"mkdirp": "0.5.0",
4949
"path-is-absolute": "^1.0.0",
5050
"socket.io": "^1.4.5",
51-
"socket.io-client": "^1.4.5"
51+
"socket.io-client": "^1.4.5",
52+
"xxhash": "^0.2.4"
5253
},
5354
"devDependencies": {
5455
"chai": "^3.5.0",

0 commit comments

Comments
 (0)