Merge pull request #336 from matiu/opt/post-txs

Opt/post txs

Author: matiu

app/controllers/addresses.js

@@ -9,6 +9,9 @@ var Address = require('../models/Address');
 var common = require('./common');
 var async = require('async');
 
+var MAX_BATCH_SIZE = 100;
+var RPC_CONCURRENCY = 5;
+
 var tDb = require('../../lib/TransactionDb').default();
 
 var checkSync = function(req, res) {
@@ -50,7 +53,7 @@ var getAddrs = function(req, res, next) {
     }
   } catch (e) {
     common.handleErrors({
-      message: 'Invalid address:' + e.message,
+      message: 'Invalid addrs param:' + e.message,
       code: 1
     }, res, next);
     return null;
@@ -101,7 +104,7 @@ exports.multiutxo = function(req, res, next) {
   var as = getAddrs(req, res, next);
   if (as) {
     var utxos = [];
-    async.each(as, function(a, callback) {
+    async.eachLimit(as, RPC_CONCURRENCY, function(a, callback) {
       a.update(function(err) {
         if (err) callback(err);
         utxos = utxos.concat(a.unspent);
@@ -123,25 +126,39 @@ exports.multitxs = function(req, res, next) {
   function processTxs(txs, from, to, cb) {
     txs = _.uniq(_.flatten(txs), 'txid');
     var nbTxs = txs.length;
-    var paginated = !_.isUndefined(from) || !_.isUndefined(to);
 
-    if (paginated) {
-      txs.sort(function(a, b) {
-        return (b.ts || b.ts) - (a.ts || a.ts);
-      });
-      var start = Math.max(from || 0, 0);
-      var end = Math.min(to || txs.length, txs.length);
-      txs = txs.slice(start, end);
+    if (_.isUndefined(from) && _.isUndefined(to)) {
+      from = 0;
+      to = MAX_BATCH_SIZE;
     }
+    if (!_.isUndefined(from) && _.isUndefined(to))
+      to = from + MAX_BATCH_SIZE;
+
+    if (!_.isUndefined(from) && !_.isUndefined(to) && to - from > MAX_BATCH_SIZE)
+      to = from + MAX_BATCH_SIZE;
+
+    if (from < 0) from = 0;
+    if (to < 0) to = 0;
+    if (from > nbTxs) from = nbTxs;
+    if (to > nbTxs) to = nbTxs;
+
+    txs.sort(function(a, b) {
+      return (b.ts || b.ts) - (a.ts || a.ts);
+    });
+
+    txs = txs.slice(from, to);
 
     var txIndex = {};
     _.each(txs, function(tx) {
       txIndex[tx.txid] = tx;
     });
 
-    async.each(txs, function(tx, callback) {
+    async.eachLimit(txs, RPC_CONCURRENCY, function(tx, callback) {
       tDb.fromIdWithInfo(tx.txid, function(err, tx) {
-        if (err) console.log(err);
+        if (err) {
+          console.log(err);
+          return common.handleErrors(err, res);
+        }
         if (tx && tx.info) {
           txIndex[tx.txid].info = tx.info;
         }
@@ -151,14 +168,12 @@ exports.multitxs = function(req, res, next) {
       if (err) return cb(err);
 
       var transactions = _.pluck(txs, 'info');
-      if (paginated) {
-        transactions = {
-          totalItems: nbTxs,
-          from: +from,
-          to: +to,
-          items: transactions,
-        };
-      }
+      transactions = {
+        totalItems: nbTxs,
+        from: +from,
+        to: +to,
+        items: transactions,
+      };
       return cb(null, transactions);
     });
   };
@@ -169,17 +184,19 @@ exports.multitxs = function(req, res, next) {
   var as = getAddrs(req, res, next);
   if (as) {
     var txs = [];
-    async.eachLimit(as, 10, function(a, callback) {
+    async.eachLimit(as, RPC_CONCURRENCY, function(a, callback) {
       a.update(function(err) {
         if (err) callback(err);
         txs.push(a.transactions);
         callback();
       }, {
         ignoreCache: req.param('noCache'),
-        includeTxInfo: true
+        includeTxInfo: true,
+        dontFillSpent: true,
       });
     }, function(err) { // finished callback
       if (err) return common.handleErrors(err, res);
+
       processTxs(txs, from, to, function(err, transactions) {
         if (err) return common.handleErrors(err, res);
         res.jsonp(transactions);

config/express.js

@@ -28,7 +28,8 @@ module.exports = function(app, historicSync, peerSync) {
   app.enable('jsonp callback');
   app.use(config.apiPrefix, setHistoric);
   app.use(config.apiPrefix, setPeer);
-  app.use(express.logger('dev'));
+  app.use(require('morgan')(':remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent" '));
+  
   app.use(express.json());
   app.use(express.urlencoded());
   app.use(express.methodOverride());

lib/TransactionDb.js

@@ -17,7 +17,7 @@ var ADDR_PREFIX = 'txa2-'; //txa-<addr>-<tsr>-<txid>-<n>
 
 // TODO: use bitcore networks module
 var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
-var CONCURRENCY = 10;
+var CONCURRENCY = 5;
 var DEFAULT_SAFE_CONFIRMATIONS = 6;
 
 var MAX_OPEN_FILES = 500;
@@ -448,6 +448,9 @@ TransactionDb.prototype._parseAddrData = function(k, data, ignoreCache) {
   return item;
 };
 
+
+// opts.dontFillSpent
+
 TransactionDb.prototype.fromAddr = function(addr, opts, cb) {
   opts = opts || {};
   var self = this;
@@ -470,6 +473,10 @@ TransactionDb.prototype.fromAddr = function(addr, opts, cb) {
     })
     .on('error', cb)
     .on('end', function() {
+      if (opts.dontFillSpent) {
+        return cb(null, ret)
+      }
+
       async.eachLimit(ret.filter(function(x) {
           return !x.spentIsConfirmed;
         }), CONCURRENCY, function(o, e_c) {

package.json

@@ -51,7 +51,7 @@
     "async": "*",
     "base58-native": "0.1.2",
     "bignum": "*",
-    "bitauth": "^0.1.1",
+    "morgan": "*",
     "bitcore": "git://github.com/bitpay/bitcore.git#aa41c70cff2583d810664c073a324376c39c8b36",
     "bufferput": "git://github.com/bitpay/node-bufferput.git",
     "buffertools": "*",