From 3214390d4ca8bd543ece2efe23ea62325602537e Mon Sep 17 00:00:00 2001 From: Samuel Reed Date: Fri, 18 Dec 2015 09:56:34 -0600 Subject: [PATCH 001/299] Fix #377; db must contain hash type, not just hash. Prevents erroneous crediting of all transactions to both the p2pkh and the corresponding p2sh address. --- integration/regtest-node.js | 5 +- lib/services/address/index.js | 107 ++++++++++++++++--- test/services/address/index.unit.js | 159 +++++++++++++++++++++++----- 3 files changed, 226 insertions(+), 45 deletions(-) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index 8b55db7f..fb7ff6bf 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -726,8 +726,9 @@ describe('Node Functionality', function() { node.services.bitcoind.sendTransaction(tx.serialize()); setImmediate(function() { - var hashBuffer = bitcore.Address(address).hashBuffer; - node.services.address._getOutputsMempool(address, hashBuffer, function(err, outs) { + var addrObj = node.services.address._getAddressInfo(address); + node.services.address._getOutputsMempool(address, addrObj.hashBuffer, + addrObj.hashTypeBuffer, function(err, outs) { if (err) { throw err; } diff --git a/lib/services/address/index.js b/lib/services/address/index.js index a557ba24..ef7e3e2c 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -70,6 +70,27 @@ AddressService.MEMPREFIXES = { SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output }; +// To save space, we're only storing the PubKeyHash or ScriptHash in our index. +// To avoid intentional unspendable collisions, which have been seen on the blockchain, +// we must store the hash type (PK or Script) as well. +AddressService.HASH_TYPES = { + PUBKEY: new Buffer('01', 'hex'), + REDEEMSCRIPT: new Buffer('02', 'hex') +}; + +// Translates from our enum type back into the hash types returned by +// bitcore-lib/address. +AddressService.HASH_TYPES_READABLE = { + '01': 'pubkeyhash', + '02': 'scripthash' +}; + +// Trnaslates from address types to our enum type. +AddressService.HASH_TYPES_MAP = { + 'pubkeyhash': AddressService.HASH_TYPES.PUBKEY, + 'scripthash': AddressService.HASH_TYPES.REDEEMSCRIPT +}; + AddressService.SPACER_MIN = new Buffer('00', 'hex'); AddressService.SPACER_MAX = new Buffer('ff', 'hex'); @@ -303,6 +324,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var outKey = Buffer.concat([ AddressService.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, + addressInfo.hashTypeBuffer, txidBuffer, outputIndexBuffer ]); @@ -355,16 +377,20 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { // Update input index var inputHashBuffer; + var inputHashType; if (input.script.isPublicKeyHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); + inputHashType = AddressService.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); + inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; } else { continue; } var inputKey = Buffer.concat([ AddressService.MEMPREFIXES.SPENTS, inputHashBuffer, + inputHashType, input.prevTxId, inputOutputIndexBuffer ]); @@ -389,7 +415,6 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { } this.mempoolIndex.batch(operations, callback); - }; /** @@ -401,16 +426,20 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { AddressService.prototype._extractAddressInfoFromScript = function(script) { var hashBuffer; var addressType; + var hashTypeBuffer; if (script.isPublicKeyHashOut()) { hashBuffer = script.chunks[2].buf; + hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; addressType = Address.PayToPublicKeyHash; } else if (script.isScriptHashOut()) { hashBuffer = script.chunks[1].buf; + hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; addressType = Address.PayToScriptHash; } else if (script.isPublicKeyOut()) { var pubkey = script.chunks[0].buf; var address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); hashBuffer = address.hashBuffer; + hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; // pay-to-publickey doesn't have an address, however for compatibility // purposes, we can create an address addressType = Address.PayToPublicKeyHash; @@ -419,6 +448,7 @@ AddressService.prototype._extractAddressInfoFromScript = function(script) { } return { hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, addressType: addressType }; }; @@ -474,7 +504,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var key = this._encodeOutputKey(addressInfo.hashBuffer, height, txidBuffer, outputIndex); + var key = this._encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + height, txidBuffer, outputIndex); var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -514,11 +545,14 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var input = inputs[inputIndex]; var inputHash; + var inputHashType; if (input.script.isPublicKeyHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); + inputHashType = AddressService.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); + inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; } else { continue; } @@ -526,7 +560,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); // To be able to query inputs by address and spent height - var inputKey = this._encodeInputKey(inputHash, height, prevTxIdBuffer, input.outputIndex); + var inputKey = this._encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); var inputValue = this._encodeInputValue(txidBuffer, inputIndex); operations.push({ @@ -563,7 +597,7 @@ AddressService.prototype._encodeSpentIndexSyncKey = function(txidBuffer, outputI return key.toString('binary'); }; -AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txidBuffer, outputIndex) { +AddressService.prototype._encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); var outputIndexBuffer = new Buffer(4); @@ -571,6 +605,7 @@ AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txidBuf var key = Buffer.concat([ AddressService.PREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, heightBuffer, txidBuffer, @@ -583,6 +618,7 @@ AddressService.prototype._decodeOutputKey = function(buffer) { var reader = new BufferReader(buffer); var prefix = reader.read(1); var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); var spacer = reader.read(1); var height = reader.readUInt32BE(); var txid = reader.read(32); @@ -590,6 +626,7 @@ AddressService.prototype._decodeOutputKey = function(buffer) { return { prefix: prefix, hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, height: height, txid: txid, outputIndex: outputIndex @@ -611,7 +648,7 @@ AddressService.prototype._decodeOutputValue = function(buffer) { }; }; -AddressService.prototype._encodeInputKey = function(hashBuffer, height, prevTxIdBuffer, outputIndex) { +AddressService.prototype._encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); var outputIndexBuffer = new Buffer(4); @@ -619,6 +656,7 @@ AddressService.prototype._encodeInputKey = function(hashBuffer, height, prevTxId return Buffer.concat([ AddressService.PREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, heightBuffer, prevTxIdBuffer, @@ -630,6 +668,7 @@ AddressService.prototype._decodeInputKey = function(buffer) { var reader = new BufferReader(buffer); var prefix = reader.read(1); var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); var spacer = reader.read(1); var height = reader.readUInt32BE(); var prevTxId = reader.read(32); @@ -637,6 +676,7 @@ AddressService.prototype._decodeInputKey = function(buffer) { return { prefix: prefix, hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, height: height, prevTxId: prevTxId, outputIndex: outputIndex @@ -698,6 +738,17 @@ AddressService.prototype._decodeInputValueMap = function(buffer) { }; }; +AddressService.prototype._getAddressInfo = function(addressStr) { + var addrObj = bitcore.Address(addressStr); + var hashTypeBuffer = AddressService.HASH_TYPES_MAP[addrObj.type]; + + return { + hashBuffer: addrObj.hashBuffer, + hashTypeBuffer: hashTypeBuffer, + hashTypeReadable: addrObj.type + }; +}; + /** * This function is responsible for emitting events to any subscribers to the * `address/transaction` event. @@ -902,6 +953,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options /** * Will give inputs that spend previous outputs for an address as an object with: * address - The base58check encoded address + * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction input * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -921,7 +973,12 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var inputs = []; var stream; - var hashBuffer = bitcore.Address(addressStr).hashBuffer; + var addrObj = this._getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + if (!hashTypeBuffer) { + return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); + } if (options.start && options.end) { @@ -935,12 +992,14 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { gte: Buffer.concat([ AddressService.PREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, endBuffer ]), lte: Buffer.concat([ AddressService.PREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, startBuffer ]), @@ -948,7 +1007,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { keyEncoding: 'binary' }); } else { - var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer]); + var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), @@ -964,6 +1023,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var input = { address: addressStr, + hashType: addrObj.hashTypeReadable, txid: value.txid.toString('hex'), inputIndex: value.inputIndex, height: key.height, @@ -988,7 +1048,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } if(options.queryMempool) { - self._getInputsMempool(addressStr, hashBuffer, function(err, mempoolInputs) { + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { if (err) { return callback(err); } @@ -1005,7 +1065,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { }; -AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, callback) { +AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, hashTypeBuffer, callback) { var self = this; var mempoolInputs = []; @@ -1013,11 +1073,13 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ca gte: Buffer.concat([ AddressService.MEMPREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN ]), lte: Buffer.concat([ AddressService.MEMPREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MAX ]), valueEncoding: 'binary', @@ -1029,6 +1091,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ca var inputIndex = data.value.readUInt32BE(32); var output = { address: addressStr, + hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, height: -1, @@ -1082,6 +1145,7 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca /** * Will give outputs for an address as an object with: * address - The base58check encoded address + * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction output * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -1101,7 +1165,12 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - var hashBuffer = bitcore.Address(addressStr).hashBuffer; + var addrObj = this._getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + if (!hashTypeBuffer) { + return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); + } var outputs = []; var stream; @@ -1117,12 +1186,14 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { gte: Buffer.concat([ AddressService.PREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, endBuffer ]), lte: Buffer.concat([ AddressService.PREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN, startBuffer ]), @@ -1130,7 +1201,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { keyEncoding: 'binary' }); } else { - var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer]); + var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), @@ -1146,6 +1217,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { var output = { address: addressStr, + hashType: addrObj.hashTypeReadable, txid: key.txid.toString('hex'), //TODO use a buffer outputIndex: key.outputIndex, height: key.height, @@ -1172,7 +1244,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } if(options.queryMempool) { - self._getOutputsMempool(addressStr, hashBuffer, function(err, mempoolOutputs) { + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return callback(err); } @@ -1188,7 +1260,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { }; -AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, callback) { +AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, hashTypeBuffer, callback) { var self = this; var mempoolOutputs = []; @@ -1196,11 +1268,13 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, c gte: Buffer.concat([ AddressService.MEMPREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MIN ]), lte: Buffer.concat([ AddressService.MEMPREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, AddressService.SPACER_MAX ]), valueEncoding: 'binary', @@ -1208,12 +1282,13 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, c }); stream.on('data', function(data) { - // Format of data: prefix: 1, hashBuffer: 20, txid: 32, outputIndex: 4 - var txid = data.key.slice(21, 53); - var outputIndex = data.key.readUInt32BE(53); + // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 + var txid = data.key.slice(22, 54); + var outputIndex = data.key.readUInt32BE(54); var value = self._decodeOutputValue(data.value); var output = { address: addressStr, + hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e7660649..e7460a1d 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -343,7 +343,9 @@ describe('Address Service', function() { }); am.node.network = Networks.livenet; var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var hashHex = bitcore.Address(address).hashBuffer.toString('hex'); + var addrObj = bitcore.Address(address); + var hashHex = addrObj.hashBuffer.toString('hex'); + var hashType = addrObj.type; var messages = {}; am.transactionOutputHandler(messages, tx, 0, true); should.exist(messages[hashHex]); @@ -351,6 +353,7 @@ describe('Address Service', function() { message.tx.should.equal(tx); message.outputIndexes.should.deep.equal([0]); message.addressInfo.hashBuffer.toString('hex').should.equal(hashHex); + message.addressInfo.addressType.should.equal(hashType); message.addressInfo.hashHex.should.equal(hashHex); message.rejected.should.equal(true); }); @@ -446,16 +449,16 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(151); operations[0].type.should.equal('put'); - operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); + operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b0100000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('put'); - operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); + operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e0100000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); operations[4].type.should.equal('put'); operations[4].key.toString('hex').should.equal('053d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); operations[4].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); operations[121].type.should.equal('put'); - operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c75280100000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); @@ -472,13 +475,13 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(151); operations[0].type.should.equal('del'); - operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); + operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b0100000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('del'); - operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); + operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e0100000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); operations[121].type.should.equal('del'); - operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c75280100000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); @@ -818,6 +821,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -866,8 +870,9 @@ describe('Address Service', function() { end: 12, queryMempool: true }; - am._getInputsMempool = sinon.stub().callsArgWith(2, null, { + am._getInputsMempool = sinon.stub().callsArgWith(3, null, { address: address, + hashType: 'pubkeyhash', height: -1, confirmations: 0 }); @@ -890,9 +895,11 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('000000000c', 'hex')]); + var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, new Buffer('000000000c', 'hex')]); ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('0000000010', 'hex')]); + var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + hashTypeBuffer, new Buffer('0000000010', 'hex')]); ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; @@ -901,7 +908,7 @@ describe('Address Service', function() { am.node.services.bitcoind = { getMempoolInputs: sinon.stub().returns([]) }; - am._getInputsMempool = sinon.stub().callsArgWith(2, null, []); + am._getInputsMempool = sinon.stub().callsArgWith(3, null, []); am.getInputs(address, args, function(err, inputs) { should.not.exist(err); inputs.length.should.equal(1); @@ -913,7 +920,7 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); @@ -927,9 +934,9 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('00', 'hex')]); + var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('00', 'hex')]); ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('ff', 'hex')]); + var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ff', 'hex')]); ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; @@ -949,7 +956,7 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); @@ -979,6 +986,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1005,7 +1013,7 @@ describe('Address Service', function() { am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getInputsMempool(address, hashBuffer, function(err, outputs) { + am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); @@ -1021,11 +1029,13 @@ describe('Address Service', function() { am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getInputsMempool(address, hashBuffer, function(err, outputs) { + am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { should.not.exist(err); outputs.length.should.equal(1); outputs[0].address.should.equal(address); outputs[0].txid.should.equal(txid); + outputs[0].hashType.should.equal('pubkeyhash'); + outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].inputIndex.should.equal(5); outputs[0].height.should.equal(-1); outputs[0].confirmations.should.equal(0); @@ -1099,6 +1109,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1135,20 +1146,22 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, new Buffer('000000000c', 'hex')]); + var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, new Buffer('0000000010', 'hex')]); + var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; } }; - am._getOutputsMempool = sinon.stub().callsArgWith(2, null, []); + am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); am.getOutputs(address, args, function(err, outputs) { should.not.exist(err); outputs.length.should.equal(1); outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); + outputs[0].hashType.should.equal('pubkeyhash'); + outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); @@ -1157,7 +1170,7 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; testStream.emit('data', data); @@ -1170,10 +1183,11 @@ describe('Address Service', function() { createReadStream: sinon.stub().returns(readStream1) }; - am._getOutputsMempool = sinon.stub().callsArgWith(2, null, [ + am._getOutputsMempool = sinon.stub().callsArgWith(3, null, [ { address: address, height: -1, + hashType: 'pubkeyhash', confirmations: 0, txid: 'aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371', satoshis: 307627737, @@ -1186,18 +1200,21 @@ describe('Address Service', function() { should.not.exist(err); outputs.length.should.equal(3); outputs[0].address.should.equal(address); + outputs[0].hashType.should.equal('pubkeyhash'); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); outputs[0].height.should.equal(345000); outputs[1].address.should.equal(address); + outputs[1].hashType.should.equal('pubkeyhash'); outputs[1].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); outputs[1].outputIndex.should.equal(2); outputs[1].satoshis.should.equal(10000); outputs[1].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); outputs[1].height.should.equal(345004); outputs[2].address.should.equal(address); + outputs[2].hashType.should.equal('pubkeyhash'); outputs[2].txid.should.equal('aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371'); outputs[2].script.should.equal('76a914f6db95c81dea3d10f0ff8d890927751bf7b203c188ac'); outputs[2].height.should.equal(-1); @@ -1206,12 +1223,12 @@ describe('Address Service', function() { }); var data1 = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68700000543a8125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b6870100000543a8125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; var data2 = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68700000543ac3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000002', 'hex'), + key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b6870100000543ac3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000002', 'hex'), value: new Buffer('40c388000000000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; @@ -1237,12 +1254,98 @@ describe('Address Service', function() { readStream2.emit('close'); }); }); + + it('should print outputs for a p2sh address', function(done) { + // This address has the redeemScript 0x038a213afdfc551fc658e9a2a58a86e98d69b687, + // which is the same as the pkhash for the address 1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W. + // See https://github.com/bitpay/bitcore-node/issues/377 + var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; + var hashBuffer = bitcore.Address(address).hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; + var testStream = new EventEmitter(); + var args = { + start: 15, + end: 12, + queryMempool: true + }; + var createReadStreamCallCount = 0; + am.node.services.db.store = { + createReadStream: function(ops) { + var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gte.toString('hex').should.equal(gte.toString('hex')); + var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lte.toString('hex').should.equal(lte.toString('hex')); + createReadStreamCallCount++; + return testStream; + } + }; + am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); + am.getOutputs(address, args, function(err, outputs) { + should.not.exist(err); + outputs.length.should.equal(1); + outputs[0].address.should.equal(address); + outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); + outputs[0].hashType.should.equal('scripthash'); + outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].outputIndex.should.equal(1); + outputs[0].satoshis.should.equal(4527773864); + outputs[0].script.should.equal('a914038a213afdfc551fc658e9a2a58a86e98d69b68787'); + outputs[0].height.should.equal(15); + done(); + }); + createReadStreamCallCount.should.equal(1); + var data = { + // note '68702', '02' meaning p2sh redeemScript, not p2pkh + // value is also the p2sh script, not p2pkh + key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68702000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + value: new Buffer('41f0de058a800000a914038a213afdfc551fc658e9a2a58a86e98d69b68787', 'hex') + }; + testStream.emit('data', data); + testStream.emit('close'); + }); + + it('should not print outputs for a p2pkh address, if the output was sent to a p2sh redeemScript', function(done) { + // This address has the redeemScript 0x038a213afdfc551fc658e9a2a58a86e98d69b687, + // which is the same as the pkhash for the address 1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W. + // See https://github.com/bitpay/bitcore-node/issues/377 + var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; + var hashBuffer = bitcore.Address(address).hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; + var testStream = new EventEmitter(); + var args = { + start: 15, + end: 12, + queryMempool: true + }; + var createReadStreamCallCount = 0; + + // Verifying that the db query is looking for a redeemScript, *not* a p2pkh + am.node.services.db.store = { + createReadStream: function(ops) { + var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gte.toString('hex').should.equal(gte.toString('hex')); + var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lte.toString('hex').should.equal(lte.toString('hex')); + createReadStreamCallCount++; + return testStream; + } + }; + am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); + am.getOutputs(address, args, function(err, outputs) { + should.not.exist(err); + outputs.length.should.equal(0); + done(); + }); + createReadStreamCallCount.should.equal(1); + testStream.emit('close'); + }); }); describe('#_getOutputsMempool', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; + var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1268,7 +1371,7 @@ describe('Address Service', function() { var testStream = new EventEmitter(); am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getOutputsMempool(address, hashBuffer, function(err, outputs) { + am._getOutputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); @@ -1283,12 +1386,13 @@ describe('Address Service', function() { am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getOutputsMempool(address, hashBuffer, function(err, outputs) { + am._getOutputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { if (err) { throw err; } outputs.length.should.equal(1); outputs[0].address.should.equal(address); + outputs[0].hashType.should.equal('pubkeyhash'); outputs[0].txid.should.equal(txid); outputs[0].outputIndex.should.equal(outputIndex); outputs[0].height.should.equal(-1); @@ -1304,8 +1408,9 @@ describe('Address Service', function() { var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var keyData = Buffer.concat([ - new Buffer('01', 'hex'), + AddressService.MEMPREFIXES.OUTPUTS, hashBuffer, + hashTypeBuffer, txidBuffer, outputIndexBuffer ]); From cab25cf397f67f8da60c3af3e2f120cff7fd5729 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 28 Dec 2015 15:43:20 -0500 Subject: [PATCH 002/299] Address Service: Start to use streams for memory optimization with large queries --- lib/services/address/constants.js | 41 + lib/services/address/encoding.js | 211 +++++ lib/services/address/index.js | 746 +++++++----------- .../address/streams/inputs-transform.js | 40 + .../address/streams/outputs-transform.js | 42 + 5 files changed, 636 insertions(+), 444 deletions(-) create mode 100644 lib/services/address/constants.js create mode 100644 lib/services/address/encoding.js create mode 100644 lib/services/address/streams/inputs-transform.js create mode 100644 lib/services/address/streams/outputs-transform.js diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js new file mode 100644 index 00000000..7a0cf8dd --- /dev/null +++ b/lib/services/address/constants.js @@ -0,0 +1,41 @@ +'use strict'; + +var exports = {}; + +exports.PREFIXES = { + OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height + SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height + SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output +}; + +exports.MEMPREFIXES = { + OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address + SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address + SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output +}; + +// To save space, we're only storing the PubKeyHash or ScriptHash in our index. +// To avoid intentional unspendable collisions, which have been seen on the blockchain, +// we must store the hash type (PK or Script) as well. +exports.HASH_TYPES = { + PUBKEY: new Buffer('01', 'hex'), + REDEEMSCRIPT: new Buffer('02', 'hex') +}; + +// Translates from our enum type back into the hash types returned by +// bitcore-lib/address. +exports.HASH_TYPES_READABLE = { + '01': 'pubkeyhash', + '02': 'scripthash' +}; + +exports.HASH_TYPES_MAP = { + 'pubkeyhash': exports.HASH_TYPES.PUBKEY, + 'scripthash': exports.HASH_TYPES.REDEEMSCRIPT +}; + +exports.SPACER_MIN = new Buffer('00', 'hex'); +exports.SPACER_MAX = new Buffer('ff', 'hex'); + +module.exports = exports; + diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js new file mode 100644 index 00000000..68313754 --- /dev/null +++ b/lib/services/address/encoding.js @@ -0,0 +1,211 @@ +'use strict'; + +var bitcore = require('bitcore-lib'); +var BufferReader = bitcore.encoding.BufferReader; +var Address = bitcore.Address; +var PublicKey = bitcore.PublicKey; +var constants = require('./constants'); +var $ = bitcore.util.preconditions; + +var exports = {}; + +exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + txidBuffer, + outputIndexBuffer + ]); + return key.toString('binary'); +}; + +exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + txidBuffer, + outputIndexBuffer + ]); + return key; +}; + +exports.decodeOutputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var txid = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + txid: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeOutputValue = function(satoshis, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, scriptBuffer]); +}; + +exports.decodeOutputValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var scriptBuffer = buffer.slice(8, buffer.length); + return { + satoshis: satoshis, + scriptBuffer: scriptBuffer + }; +}; + +exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + prevTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var prevTxId = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + prevTxId: prevTxId, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValue = function(txidBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + txidBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValue = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + txid: txid, + inputIndex: inputIndex + }; +}; + +exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTSMAP, + outputTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKeyMap = function(buffer) { + var txid = buffer.slice(1, 33); + var outputIndex = buffer.readUInt32BE(33); + return { + outputTxId: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + inputTxIdBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValueMap = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + inputTxId: txid, + inputIndex: inputIndex + }; +}; + +exports.getAddressInfo = function(addressStr) { + var addrObj = bitcore.Address(addressStr); + var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; + + return { + hashBuffer: addrObj.hashBuffer, + hashTypeBuffer: hashTypeBuffer, + hashTypeReadable: addrObj.type + }; +}; + +/** + * This function is optimized to return address information about an output script + * without constructing a Bitcore Address instance. + * @param {Script} - An instance of a Bitcore Script + * @param {Network|String} - The network for the address + */ +exports.extractAddressInfoFromScript = function(script, network) { + $.checkArgument(network, 'Second argument is expected to be a network'); + var hashBuffer; + var addressType; + var hashTypeBuffer; + if (script.isPublicKeyHashOut()) { + hashBuffer = script.chunks[2].buf; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + addressType = Address.PayToPublicKeyHash; + } else if (script.isScriptHashOut()) { + hashBuffer = script.chunks[1].buf; + hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + addressType = Address.PayToScriptHash; + } else if (script.isPublicKeyOut()) { + var pubkey = script.chunks[0].buf; + var address = Address.fromPublicKey(new PublicKey(pubkey), network); + hashBuffer = address.hashBuffer; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + // pay-to-publickey doesn't have an address, however for compatibility + // purposes, we can create an address + addressType = Address.PayToPublicKeyHash; + } else { + return false; + } + return { + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + addressType: addressType + }; +}; + +module.exports = exports; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index ef7e3e2c..cf37e7e2 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -16,11 +16,14 @@ var memdown = require('memdown'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var Hash = bitcore.crypto.Hash; -var BufferReader = bitcore.encoding.BufferReader; var EventEmitter = require('events').EventEmitter; -var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; var AddressHistory = require('./history'); +var constants = require('./constants'); +var encoding = require('./encoding'); +var InputsTransformStream = require('./streams/inputs-transform'); +var OutputsTransformStream = require('./streams/outputs-transform'); + /** * The Address Service builds upon the Database Service and the Bitcoin Service to add additional @@ -58,42 +61,6 @@ AddressService.dependencies = [ 'db' ]; -AddressService.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height - SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height - SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output -}; - -AddressService.MEMPREFIXES = { - OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address - SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address - SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output -}; - -// To save space, we're only storing the PubKeyHash or ScriptHash in our index. -// To avoid intentional unspendable collisions, which have been seen on the blockchain, -// we must store the hash type (PK or Script) as well. -AddressService.HASH_TYPES = { - PUBKEY: new Buffer('01', 'hex'), - REDEEMSCRIPT: new Buffer('02', 'hex') -}; - -// Translates from our enum type back into the hash types returned by -// bitcore-lib/address. -AddressService.HASH_TYPES_READABLE = { - '01': 'pubkeyhash', - '02': 'scripthash' -}; - -// Trnaslates from address types to our enum type. -AddressService.HASH_TYPES_MAP = { - 'pubkeyhash': AddressService.HASH_TYPES.PUBKEY, - 'scripthash': AddressService.HASH_TYPES.REDEEMSCRIPT -}; - -AddressService.SPACER_MIN = new Buffer('00', 'hex'); -AddressService.SPACER_MAX = new Buffer('ff', 'hex'); - AddressService.prototype.start = function(callback) { var self = this; @@ -205,7 +172,7 @@ AddressService.prototype.transactionOutputHandler = function(messages, tx, outpu return; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { return; } @@ -312,7 +279,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { if (!output.script) { continue; } - var addressInfo = this._extractAddressInfoFromScript(output.script); + var addressInfo = encoding.extractAddressInfoFromScript(output.script, this.node.network); if (!addressInfo) { continue; } @@ -322,14 +289,14 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { outputIndexBuffer.writeUInt32BE(outputIndex); var outKey = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, txidBuffer, outputIndexBuffer ]); - var outValue = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var outValue = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -347,7 +314,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { inputOutputIndexBuffer.writeUInt32BE(input.outputIndex); // Add an additional small spent index for fast synchronous lookups - var spentIndexSyncKey = this._encodeSpentIndexSyncKey( + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( input.prevTxId, input.outputIndex ); @@ -359,7 +326,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { // Add a more detailed spent index with values var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, input.prevTxId, inputOutputIndexBuffer ]); @@ -380,15 +347,15 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var inputHashType; if (input.script.isPublicKeyHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } var inputKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, input.prevTxId, @@ -417,42 +384,6 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { this.mempoolIndex.batch(operations, callback); }; -/** - * This function is optimized to return address information about an output script - * without constructing a Bitcore Address instance. - * @param {Script} - An instance of a Bitcore Script - * @private - */ -AddressService.prototype._extractAddressInfoFromScript = function(script) { - var hashBuffer; - var addressType; - var hashTypeBuffer; - if (script.isPublicKeyHashOut()) { - hashBuffer = script.chunks[2].buf; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - addressType = Address.PayToPublicKeyHash; - } else if (script.isScriptHashOut()) { - hashBuffer = script.chunks[1].buf; - hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - addressType = Address.PayToScriptHash; - } else if (script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - var address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); - hashBuffer = address.hashBuffer; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - // pay-to-publickey doesn't have an address, however for compatibility - // purposes, we can create an address - addressType = Address.PayToPublicKeyHash; - } else { - return false; - } - return { - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - addressType: addressType - }; -}; - /** * The Database Service will run this function when blocks are connected and * disconnected to the chain during syncing and reorganizations. @@ -494,7 +425,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { continue; } @@ -504,9 +435,9 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var key = this._encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); - var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + height, txidBuffer, outputIndex); + var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, key: key, @@ -549,10 +480,10 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { if (input.script.isPublicKeyHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } @@ -560,8 +491,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); // To be able to query inputs by address and spent height - var inputKey = this._encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); - var inputValue = this._encodeInputValue(txidBuffer, inputIndex); + var inputKey = encoding.encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); + var inputValue = encoding.encodeInputValue(txidBuffer, inputIndex); operations.push({ type: action, @@ -570,8 +501,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); // To be able to search for an input spending an output - var inputKeyMap = this._encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); - var inputValueMap = this._encodeInputValueMap(txidBuffer, inputIndex); + var inputKeyMap = encoding.encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); + var inputValueMap = encoding.encodeInputValueMap(txidBuffer, inputIndex); operations.push({ type: action, @@ -587,168 +518,6 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); }; -AddressService.prototype._encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - txidBuffer, - outputIndexBuffer - ]); - return key.toString('binary'); -}; - -AddressService.prototype._encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - txidBuffer, - outputIndexBuffer - ]); - return key; -}; - -AddressService.prototype._decodeOutputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var txid = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - txid: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeOutputValue = function(satoshis, scriptBuffer) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([satoshisBuffer, scriptBuffer]); -}; - -AddressService.prototype._decodeOutputValue = function(buffer) { - var satoshis = buffer.readDoubleBE(0); - var scriptBuffer = buffer.slice(8, buffer.length); - return { - satoshis: satoshis, - scriptBuffer: scriptBuffer - }; -}; - -AddressService.prototype._encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - prevTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var prevTxId = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - prevTxId: prevTxId, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValue = function(txidBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - txidBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValue = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - txid: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTSMAP, - outputTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKeyMap = function(buffer) { - var txid = buffer.slice(1, 33); - var outputIndex = buffer.readUInt32BE(33); - return { - outputTxId: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - inputTxIdBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValueMap = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - inputTxId: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._getAddressInfo = function(addressStr) { - var addrObj = bitcore.Address(addressStr); - var hashTypeBuffer = AddressService.HASH_TYPES_MAP[addrObj.type]; - - return { - hashBuffer: addrObj.hashBuffer, - hashTypeBuffer: hashTypeBuffer, - hashTypeReadable: addrObj.type - }; -}; - /** * This function is responsible for emitting events to any subscribers to the * `address/transaction` event. @@ -926,12 +695,12 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options txidBuffer = new Buffer(txid, 'hex'); } if (options.queryMempool) { - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); if (this.mempoolSpentIndex[spentIndexSyncKey]) { return this._getSpentMempool(txidBuffer, outputIndex, callback); } } - var key = this._encodeInputKeyMap(txidBuffer, outputIndex); + var key = encoding.encodeInputKeyMap(txidBuffer, outputIndex); var dbOptions = { valueEncoding: 'binary', keyEncoding: 'binary' @@ -942,7 +711,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options } else if (err) { return callback(err); } - var value = self._decodeInputValueMap(buffer); + var value = encoding.decodeInputValueMap(buffer); callback(null, { inputTxId: value.inputTxId.toString('hex'), inputIndex: value.inputIndex @@ -951,34 +720,33 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options }; /** - * Will give inputs that spend previous outputs for an address as an object with: - * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' - * txid - A string of the transaction hash - * outputIndex - A number of corresponding transaction input - * height - The height of the block the transaction was included, will be -1 for mempool transactions - * confirmations - The number of confirmations, will equal 0 for mempool transactions + * A streaming equivalent to `getInputs`, and returns a transform stream with data + * emitted in the same format as `getInputs`. * * @param {String} addressStr - The relevant address * @param {Object} options - Additional options for query the outputs * @param {Number} [options.start] - The relevant start block height * @param {Number} [options.end] - The relevant end block height - * @param {Boolean} [options.queryMempool] - Include the mempool in the results * @param {Function} callback */ -AddressService.prototype.getInputs = function(addressStr, options, callback) { +AddressService.prototype.createInputsStream = function(addressStr, options, callback) { - var self = this; + var inputStream = new InputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); - var inputs = []; - var stream; + var stream = this.createInputsDBStream(addressStr, options).pipe(inputStream); + + return stream; + +}; - var addrObj = this._getAddressInfo(addressStr); +AddressService.prototype.createInputsDBStream = function(addressStr, options) { + var stream; + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (!hashTypeBuffer) { - return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); - } if (options.start && options.end) { @@ -990,48 +758,65 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { stream = this.node.services.db.store.createReadStream({ gte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, + constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN, + constants.SPACER_MIN, endBuffer ]), lte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, + constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN, + constants.SPACER_MIN, startBuffer ]), valueEncoding: 'binary', keyEncoding: 'binary' }); } else { - var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); + var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), + gte: Buffer.concat([allKey, constants.SPACER_MIN]), + lte: Buffer.concat([allKey, constants.SPACER_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); } - stream.on('data', function(data) { + return stream; +}; - var key = self._decodeInputKey(data.key); - var value = self._decodeInputValue(data.value); +/** + * Will give inputs that spend previous outputs for an address as an object with: + * address - The base58check encoded address + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * txid - A string of the transaction hash + * outputIndex - A number of corresponding transaction input + * height - The height of the block the transaction was included, will be -1 for mempool transactions + * confirmations - The number of confirmations, will equal 0 for mempool transactions + * + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Boolean} [options.queryMempool] - Include the mempool in the results + * @param {Function} callback + */ +AddressService.prototype.getInputs = function(addressStr, options, callback) { - var input = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: value.txid.toString('hex'), - inputIndex: value.inputIndex, - height: key.height, - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; + var self = this; - inputs.push(input); + var inputs = []; + + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + var stream = this.createInputsStream(addressStr, options); + + stream.on('data', function(input) { + inputs.push(input); }); var error; @@ -1042,7 +827,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('end', function() { if (error) { return callback(error); } @@ -1071,16 +856,16 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1091,7 +876,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha var inputIndex = data.value.readUInt32BE(32); var output = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, height: -1, @@ -1121,7 +906,7 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, txidBuffer, outputIndexBuffer ]); @@ -1142,37 +927,24 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca ); }; -/** - * Will give outputs for an address as an object with: - * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' - * txid - A string of the transaction hash - * outputIndex - A number of corresponding transaction output - * height - The height of the block the transaction was included, will be -1 for mempool transactions - * satoshis - The satoshis value of the output - * script - The script of the output as a hex string - * confirmations - The number of confirmations, will equal 0 for mempool transactions - * - * @param {String} addressStr - The relevant address - * @param {Object} options - Additional options for query the outputs - * @param {Number} [options.start] - The relevant start block height - * @param {Number} [options.end] - The relevant end block height - * @param {Boolean} [options.queryMempool] - Include the mempool in the results - * @param {Function} callback - */ -AddressService.prototype.getOutputs = function(addressStr, options, callback) { - var self = this; - $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); - $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); +AddressService.prototype.createOutputsStream = function(addressStr, options) { + + var outputStream = new OutputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); + + var stream = this.createOutputsDBStream(addressStr, options).pipe(outputStream); - var addrObj = this._getAddressInfo(addressStr); + return stream; + +}; + +AddressService.prototype.createOutputsDBStream = function(addressStr, options) { + + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (!hashTypeBuffer) { - return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); - } - - var outputs = []; var stream; if (options.start && options.end) { @@ -1184,50 +956,71 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { stream = this.node.services.db.store.createReadStream({ gte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, + constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN, + constants.SPACER_MIN, endBuffer ]), lte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, + constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN, + constants.SPACER_MIN, startBuffer ]), valueEncoding: 'binary', keyEncoding: 'binary' }); } else { - var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); + var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), + gte: Buffer.concat([allKey, constants.SPACER_MIN]), + lte: Buffer.concat([allKey, constants.SPACER_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); } - stream.on('data', function(data) { + return stream; - var key = self._decodeOutputKey(data.key); - var value = self._decodeOutputValue(data.value); +}; - var output = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: key.txid.toString('hex'), //TODO use a buffer - outputIndex: key.outputIndex, - height: key.height, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; +/** + * Will give outputs for an address as an object with: + * address - The base58check encoded address + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * txid - A string of the transaction hash + * outputIndex - A number of corresponding transaction output + * height - The height of the block the transaction was included, will be -1 for mempool transactions + * satoshis - The satoshis value of the output + * script - The script of the output as a hex string + * confirmations - The number of confirmations, will equal 0 for mempool transactions + * + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Boolean} [options.queryMempool] - Include the mempool in the results + * @param {Function} callback + */ +AddressService.prototype.getOutputs = function(addressStr, options, callback) { + var self = this; + $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); + $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); + + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + if (!hashTypeBuffer) { + return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); + } - outputs.push(output); + var outputs = []; + var stream = this.createOutputsStream(addressStr, options); + stream.on('data', function(data) { + outputs.push(data); }); var error; @@ -1238,7 +1031,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('end', function() { if (error) { return callback(error); } @@ -1266,16 +1059,16 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1285,10 +1078,10 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 var txid = data.key.slice(22, 54); var outputIndex = data.key.readUInt32BE(54); - var value = self._decodeOutputValue(data.value); + var value = encoding.decodeOutputValue(data.value); var output = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, @@ -1407,7 +1200,7 @@ AddressService.prototype.isSpent = function(output, options, callback) { var spent = self.node.services.bitcoind.isSpent(txid, output.outputIndex); if (!spent && queryMempool) { var txidBuffer = new Buffer(txid, 'hex'); - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); spent = self.mempoolSpentIndex[spentIndexSyncKey] ? true : false; } setImmediate(function() { @@ -1474,111 +1267,176 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba * @param {Boolean} [options.noTxList] - if set, txid array will not be included * @param {Function} callback */ -AddressService.prototype.getAddressSummary = function(address, options, callback) { +AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { + var self = this; + + var address = new Address(addressArg); + + async.waterfall([ + function(next) { + self._getAddressInputsSummary(address, options, next); + }, + function(result, next) { + self._getAddressOutputsSummary(address, options, result, next); + } + ], function(err, result) { + if (err) { + return callback(err); + } + + var confirmedTxids = Object.keys(result.appearanceIds); + var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + + var summary = { + totalReceived: result.totalReceived, + totalSpent: result.totalSpent, + balance: result.balance, + unconfirmedBalance: result.unconfirmedBalance, + appearances: confirmedTxids.length, + unconfirmedAppearances: unconfirmedTxids.length + }; + + if (!options.noTxList) { + var txids = confirmedTxids.concat(unconfirmedTxids); + + // sort by height + summary.txids = txids.sort(function(a, b) { + return a.height > b.height ? 1 : -1; + }).map(function(obj) { + return obj.txid; + }).filter(function(value, index, self) { + return self.indexOf(value) === index; + }); + } + + callback(null, summary); + + }); + +}; + +AddressService.prototype._getAddressInputsSummary = function(address, options, callback) { + $.checkArgument(address instanceof Address); var self = this; - var opt = { - queryMempool: true + var error = null; + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, }; - var outputs; - var inputs; - - async.parallel( - [ - function(next) { - self.getInputs(address, opt, function(err, ins) { - inputs = ins; - next(err); - }); - }, - function(next) { - self.getOutputs(address, opt, function(err, outs) { - outputs = outs; - next(err); - }); - } - ], - function(err) { - if(err) { + var inputsStream = self.createInputsStream(address, options); + inputsStream.on('data', function(input) { + var txid = input.txid; + result.appearanceIds[txid] = true; + }); + + inputsStream.on('error', function(err) { + error = err; + }); + + inputsStream.on('end', function() { + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { + if (err) { return callback(err); } + for(var i = 0; i < mempoolInputs.length; i++) { + var input = mempoolInputs[i]; + result.unconfirmedAppearanceIds[input.txid] = true; + } + callback(error, result); + }); + }); +}; - var totalReceived = 0; - var totalSpent = 0; - var balance = 0; - var unconfirmedBalance = 0; - var appearanceIds = {}; - var unconfirmedAppearanceIds = {}; - var txids = []; - - for(var i = 0; i < outputs.length; i++) { - // Bitcoind's isSpent only works for confirmed transactions - var spentDB = self.node.services.bitcoind.isSpent(outputs[i].txid, outputs[i].outputIndex); - var spentIndexSyncKey = self._encodeSpentIndexSyncKey( - new Buffer(outputs[i].txid, 'hex'), // TODO: get buffer directly - outputs[i].outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; +AddressService.prototype._getAddressOutputsSummary = function(address, options, result, callback) { + $.checkArgument(address instanceof Address); + $.checkArgument(!_.isUndefined(result) && + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); - txids.push(outputs[i]); + var self = this; - if(outputs[i].confirmations) { - totalReceived += outputs[i].satoshis; - balance += outputs[i].satoshis; - appearanceIds[outputs[i].txid] = true; - } else { - unconfirmedBalance += outputs[i].satoshis; - unconfirmedAppearanceIds[outputs[i].txid] = true; - } + var outputStream = self.createOutputsStream(address, options); - if(spentDB || spentMempool) { - if(spentDB) { - totalSpent += outputs[i].satoshis; - balance -= outputs[i].satoshis; - } else if(!outputs[i].confirmations) { - unconfirmedBalance -= outputs[i].satoshis; - } - } + result.totalReceived = 0; + result.totalSpent = 0; + result.balance = 0; + result.unconfirmedBalance = 0; + + outputStream.on('data', function(output) { + + var txid = output.txid; + var outputIndex = output.outputIndex; + + // Bitcoind's isSpent only works for confirmed transactions + var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = true; + + if (spentDB) { + result.totalSpent += output.satoshis; + } else { + result.balance += output.satoshis; + } + + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + result.unconfirmedBalance -= output.satoshis; + } + + }); + + var error = null; + + outputStream.on('error', function(err) { + error = err; + }); + + outputStream.on('end', function() { + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { + if (err) { + return callback(err); } - for(var j = 0; j < inputs.length; j++) { - if (inputs[j].confirmations) { - appearanceIds[inputs[j].txid] = true; - } else { - unconfirmedAppearanceIds[outputs[j].txid] = true; + for(var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; + + result.unconfirmedAppearanceIds[output.txid] = true; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + result.unconfirmedBalance += output.satoshis; } } - var summary = { - totalReceived: totalReceived, - totalSpent: totalSpent, - balance: balance, - unconfirmedBalance: unconfirmedBalance, - appearances: Object.keys(appearanceIds).length, - unconfirmedAppearances: Object.keys(unconfirmedAppearanceIds).length - }; - - if(!options.noTxList) { - for(var i = 0; i < inputs.length; i++) { - txids.push(inputs[i]); - } + callback(error, result); - // sort by height - txids = txids.sort(function(a, b) { - return a.height > b.height ? 1 : -1; - }).map(function(obj) { - return obj.txid; - }).filter(function(value, index, self) { - return self.indexOf(value) === index; - }); + }); - summary.txids = txids; - } + }); - callback(null, summary); - } - ); }; module.exports = AddressService; diff --git a/lib/services/address/streams/inputs-transform.js b/lib/services/address/streams/inputs-transform.js new file mode 100644 index 00000000..8b8f71d3 --- /dev/null +++ b/lib/services/address/streams/inputs-transform.js @@ -0,0 +1,40 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function InputsTransformStream(options) { + $.checkArgument(options.address instanceof bitcore.Address); + Transform.call(this, { + objectMode: true + }); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(InputsTransformStream, Transform); + +InputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeInputKey(chunk.key); + var value = encodingUtil.decodeInputValue(chunk.value); + + var input = { + address: this._addressStr, + hashType: this._address.type, + txid: value.txid.toString('hex'), + inputIndex: value.inputIndex, + height: key.height, + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(input); + callback(); + +}; + +module.exports = InputsTransformStream; diff --git a/lib/services/address/streams/outputs-transform.js b/lib/services/address/streams/outputs-transform.js new file mode 100644 index 00000000..b9c8e8d3 --- /dev/null +++ b/lib/services/address/streams/outputs-transform.js @@ -0,0 +1,42 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function OutputsTransformStream(options) { + Transform.call(this, { + objectMode: true + }); + $.checkArgument(options.address instanceof bitcore.Address); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(OutputsTransformStream, Transform); + +OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeOutputKey(chunk.key); + var value = encodingUtil.decodeOutputValue(chunk.value); + + var output = { + address: this._addressStr, + hashType: this._address.type, + txid: key.txid.toString('hex'), //TODO use a buffer + outputIndex: key.outputIndex, + height: key.height, + satoshis: value.satoshis, + script: value.scriptBuffer.toString('hex'), //TODO use a buffer + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(output); + callback(); + +}; + +module.exports = OutputsTransformStream; From 40eb4f50aec0d841daea783e3055ed16e570af7e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Dec 2015 00:16:22 -0500 Subject: [PATCH 003/299] Address Service: Start to cache `getAddressSummary` based on range of block heights --- lib/services/address/constants.js | 4 + lib/services/address/encoding.js | 59 +++++++++++ lib/services/address/index.js | 167 +++++++++++++++++++++++------- lib/services/db.js | 6 +- 4 files changed, 197 insertions(+), 39 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 7a0cf8dd..dc0b18b7 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -37,5 +37,9 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +// The total number of transactions that an address can receive before it will start +// to cache the summary to disk. +exports.SUMMARY_CACHE_THRESHOLD = 10000; + module.exports = exports; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 68313754..ca42fc7f 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -160,6 +160,65 @@ exports.decodeInputValueMap = function(buffer) { }; }; +exports.encodeSummaryCacheKey = function(address) { + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_BUFFER[address.type]]); +}; + +exports.decodeSummaryCacheKey = function(buffer, network) { + var hashBuffer = buffer.read(20); + var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')]; + var address = new Address({ + hashBuffer: hashBuffer, + type: type, + network: network + }); + return address; +}; + +exports.encodeSummaryCacheValue = function(cache, tipHeight) { + var buffer = new Buffer(new Array(20)); + buffer.writeUInt32BE(tipHeight); + buffer.writeDoubleBE(cache.result.totalReceived, 4); + buffer.writeDoubleBE(cache.result.balance, 12); + var txidBuffers = []; + for (var key in cache.result.appearanceIds) { + txidBuffers.push(new Buffer(key, 'hex')); + } + var txidsBuffer = Buffer.concat(txidBuffers); + var value = Buffer.concat([buffer, txidsBuffer]); + + return value; +}; + +exports.decodeSummaryCacheValue = function(buffer) { + + var height = buffer.readUInt32BE(); + var totalReceived = buffer.readDoubleBE(4); + var balance = buffer.readDoubleBE(12); + + // read 32 byte chunks until exhausted + var appearanceIds = {}; + var pos = 16; + while(pos < buffer.length) { + var txid = buffer.slice(pos, pos + 32).toString('hex'); + appearanceIds[txid] = true; + pos += 32; + } + + var cache = { + height: height, + result: { + appearanceIds: appearanceIds, + totalReceived: totalReceived, + balance: balance, + unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache + unconfirmedBalance: 0 + } + }; + + return cache; +}; + exports.getAddressInfo = function(addressStr) { var addrObj = bitcore.Address(addressStr); var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index cf37e7e2..665a32dc 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,7 +44,10 @@ var AddressService = function(options) { this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; + this._setMempoolIndexPath(); + this._setSummaryCachePath(); if (options.mempoolMemoryIndex) { this.levelupStore = memdown; } else { @@ -74,6 +77,7 @@ AddressService.prototype.start = function(callback) { } }, function(next) { + // Setup new mempool index if (!fs.existsSync(self.mempoolIndexPath)) { mkdirp(self.mempoolIndexPath, next); } else { @@ -87,7 +91,21 @@ AddressService.prototype.start = function(callback) { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', - fillCache: false + fillCache: false, + maxOpenFiles: 200 + }, + next + ); + }, + function(next) { + self.summaryCache = levelup( + self.summaryCachePath, + { + db: self.levelupStore, + keyEncoding: 'binary', + valueEncoding: 'binary', + fillCache: false, + maxOpenFiles: 200 }, next ); @@ -102,21 +120,35 @@ AddressService.prototype.stop = function(callback) { }; /** - * This function will set `this.dataPath` based on `this.node.network`. + * This function will set `this.summaryCachePath` based on `this.node.network`. + * @private + */ +AddressService.prototype._setSummaryCachePath = function() { + this.summaryCachePath = this._getDBPathFor('bitcore-addresssummary.db'); +}; + +/** + * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private */ AddressService.prototype._setMempoolIndexPath = function() { + this.mempoolIndexPath = this._getDBPathFor('bitcore-addressmempool.db'); +}; + +AddressService.prototype._getDBPathFor = function(dbname) { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); + var path; var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { - this.mempoolIndexPath = this.node.datadir + '/bitcore-addressmempool.db'; + path = this.node.datadir + '/' + dbname; } else if (this.node.network === Networks.testnet) { - this.mempoolIndexPath = this.node.datadir + '/testnet3/bitcore-addressmempool.db'; + path = this.node.datadir + '/testnet3/' + dbname; } else if (this.node.network === regtest) { - this.mempoolIndexPath = this.node.datadir + '/regtest/bitcore-addressmempool.db'; + path = this.node.datadir + '/regtest/' + dbname; } else { throw new Error('Unknown network: ' + this.network); } + return path; }; /** @@ -1270,32 +1302,49 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; + var startTime = new Date(); + var address = new Address(addressArg); + var tipHeight = this.node.services.db.tip.__height; async.waterfall([ function(next) { - self._getAddressInputsSummary(address, options, next); + self._getAddressSummaryCache(address, next); }, - function(result, next) { - self._getAddressOutputsSummary(address, options, result, next); + function(cache, next) { + self._getAddressInputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._getAddressOutputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._saveAddressSummaryCache(address, cache, tipHeight, next); } - ], function(err, result) { + ], function(err, cache) { if (err) { return callback(err); } + var result = cache.result; var confirmedTxids = Object.keys(result.appearanceIds); var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { totalReceived: result.totalReceived, - totalSpent: result.totalSpent, + totalSpent: result.totalReceived - result.balance, balance: result.balance, - unconfirmedBalance: result.unconfirmedBalance, appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, unconfirmedAppearances: unconfirmedTxids.length }; + var timeDelta = new Date() - startTime; + if (timeDelta > 5000) { + var seconds = Math.round(timeDelta / 1000); + log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); + log.warn('Address Summary:', summary); + } + if (!options.noTxList) { var txids = confirmedTxids.concat(unconfirmedTxids); @@ -1315,20 +1364,64 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }; -AddressService.prototype._getAddressInputsSummary = function(address, options, callback) { +AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { + var transactionLength = Object.keys(cache.result.appearanceIds).length; + var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); + if (exceedsCacheThreshold) { + log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); + var key = encoding.encodeSummaryCacheKey(address); + var value = encoding.encodeSummaryCacheValue(cache, tipHeight); + this.summaryCache.put(key, value, function(err) { + if (err) { + return callback(err); + } + callback(null, cache); + }); + } else { + callback(null, cache); + } +}; + +AddressService.prototype._getAddressSummaryCache = function(address, callback) { + var baseCache = { + result: { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + } + }; + var key = encoding.encodeSummaryCacheKey(address); + this.summaryCache.get(key, { + valueEncoding: 'binary', + keyEncoding: 'binary' + }, function(err, buffer) { + if (err instanceof levelup.errors.NotFoundError) { + return callback(null, baseCache); + } else if (err) { + return callback(err); + } + var cache = encoding.decodeSummaryCacheValue(buffer); + callback(null, cache); + }); +}; + +AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); var self = this; var error = null; - var result = { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, + + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight }; - var inputsStream = self.createInputsStream(address, options); + var inputsStream = self.createInputsStream(address, opts); inputsStream.on('data', function(input) { var txid = input.txid; - result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = true; }); inputsStream.on('error', function(err) { @@ -1347,27 +1440,27 @@ AddressService.prototype._getAddressInputsSummary = function(address, options, c } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = true; } - callback(error, result); + callback(error, cache); }); }); }; -AddressService.prototype._getAddressOutputsSummary = function(address, options, result, callback) { +AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); + $.checkArgument(!_.isUndefined(cache.result) && + !_.isUndefined(cache.result.appearanceIds) && + !_.isUndefined(cache.result.unconfirmedAppearanceIds)); var self = this; - var outputStream = self.createOutputsStream(address, options); + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight + }; - result.totalReceived = 0; - result.totalSpent = 0; - result.balance = 0; - result.unconfirmedBalance = 0; + var outputStream = self.createOutputsStream(address, opts); outputStream.on('data', function(output) { @@ -1376,13 +1469,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = true; + cache.result.totalReceived += output.satoshis; + cache.result.appearanceIds[txid] = true; - if (spentDB) { - result.totalSpent += output.satoshis; - } else { - result.balance += output.satoshis; + if (!spentDB) { + cache.result.balance += output.satoshis; } // Check to see if this output is spent in the mempool and if so @@ -1393,7 +1484,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; if (spentMempool) { - result.unconfirmedBalance -= output.satoshis; + cache.result.unconfirmedBalance -= output.satoshis; } }); @@ -1418,7 +1509,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = true; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly @@ -1427,11 +1518,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already if (!spentMempool) { - result.unconfirmedBalance += output.satoshis; + cache.result.unconfirmedBalance += output.satoshis; } } - callback(error, result); + callback(error, cache); }); diff --git a/lib/services/db.js b/lib/services/db.js index 0c655051..c3c3dfd4 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -46,6 +46,8 @@ function DB(options) { this._setDataPath(); + this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; + this.levelupStore = leveldown; if (options.store) { this.levelupStore = options.store; @@ -68,6 +70,8 @@ DB.PREFIXES = { TIP: new Buffer('04', 'hex') }; +DB.DEFAULT_MAX_OPEN_FILES = 200; + /** * This function will set `this.dataPath` based on `this.node.network`. * @private @@ -98,7 +102,7 @@ DB.prototype.start = function(callback) { } this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore }); + this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles }); this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.once('ready', function() { From cef2f7686de33b7bbda126b853cf65498b0df882 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Jan 2016 16:47:29 -0500 Subject: [PATCH 004/299] Address Service: Limit the length of outputs that can be queried at a time --- lib/services/address/constants.js | 6 ++++++ lib/services/address/index.js | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index dc0b18b7..f11b3fe7 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -41,5 +41,11 @@ exports.SPACER_MAX = new Buffer('ff', 'hex'); // to cache the summary to disk. exports.SUMMARY_CACHE_THRESHOLD = 10000; + +// The default maximum length queries +exports.MAX_INPUTS_QUERY_LENGTH = 50000; +exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; + + module.exports = exports; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 665a32dc..1eeb0404 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -45,6 +45,8 @@ var AddressService = function(options) { this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; + this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; + this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this._setMempoolIndexPath(); this._setSummaryCachePath(); @@ -849,6 +851,12 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { stream.on('data', function(input) { inputs.push(input); + if (inputs.length > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + stream.pause(); + stream.end(); + } }); var error; @@ -859,7 +867,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } }); - stream.on('end', function() { + stream.on('finish', function() { if (error) { return callback(error); } @@ -1053,6 +1061,12 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { stream.on('data', function(data) { outputs.push(data); + if (outputs.length > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + stream.pause(); + stream.end(); + } }); var error; @@ -1063,7 +1077,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } }); - stream.on('end', function() { + stream.on('finish', function() { if (error) { return callback(error); } From 8298e380ed195699bb9e2f1a9cd98224d92563a0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Jan 2016 19:14:45 -0500 Subject: [PATCH 005/299] Address Service: Use streams to combine inputs and outputs --- lib/services/address/history.js | 148 +++++--------------- lib/services/address/streams/combined.js | 166 +++++++++++++++++++++++ 2 files changed, 199 insertions(+), 115 deletions(-) create mode 100644 lib/services/address/streams/combined.js diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 61e77158..1795fede 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -2,6 +2,7 @@ var bitcore = require('bitcore-lib'); var async = require('async'); +var CombinedStream = require('./streams/combined'); var _ = bitcore.deps._; /** @@ -19,7 +20,6 @@ function AddressHistory(args) { } else { this.addresses = [args.addresses]; } - this.transactionInfo = []; this.combinedArray = []; this.detailedArray = []; } @@ -35,129 +35,47 @@ AddressHistory.prototype.get = function(callback) { var self = this; var totalCount; - async.eachLimit( - self.addresses, - AddressHistory.MAX_ADDRESS_QUERIES, - function(address, next) { - self.getTransactionInfo(address, next); - }, - function(err) { - if (err) { - return callback(err); - } + // TODO: handle multiple addresses (restore previous functionality) + if (self.addresses.length > 1) { + return callback('Only single address queries supported currently'); + } - self.combineTransactionInfo(); - totalCount = Number(self.combinedArray.length); - self.sortAndPaginateCombinedArray(); + var address = self.addresses[0]; - async.eachSeries( - self.combinedArray, - function(txInfo, next) { - self.getDetailedInfo(txInfo, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); - } - ); -}; + var combinedStream = new CombinedStream({ + inputStream: this.node.services.address.createInputsStream(address, this.options), + outputStream: this.node.services.address.createOutputsStream(address, this.options) + }); -/** - * This function will retrieve input and output information for an address - * and set the property `this.transactionInfo`. - * @param {String} address - A base58check encoded address - * @param {Function} next - */ -AddressHistory.prototype.getTransactionInfo = function(address, next) { - var self = this; + // Results from the transaction info stream are grouped into + // sets based on block height + combinedStream.on('data', function(block) { + self.combinedArray = self.combinedArray.concat(block); + }); - var args = { - start: self.options.start, - end: self.options.end, - queryMempool: _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool - }; + combinedStream.on('end', function() { + totalCount = Number(self.combinedArray.length); - var outputs; - var inputs; + self.sortAndPaginateCombinedArray(); - async.parallel([ - function(done) { - self.node.services.address.getOutputs(address, args, function(err, result) { - if (err) { - return done(err); - } - outputs = result; - done(); - }); - }, - function(done) { - self.node.services.address.getInputs(address, args, function(err, result) { + // TODO: Add the mempool transactions + + async.eachSeries( + self.combinedArray, + function(txInfo, next) { + self.getDetailedInfo(txInfo, next); + }, + function(err) { if (err) { - return done(err); + return callback(err); } - inputs = result; - done(); - }); - } - ], function(err) { - if (err) { - return next(err); - } - self.transactionInfo = self.transactionInfo.concat(outputs, inputs); - next(); - }); -}; - -/** - * This function combines results from getInputs and getOutputs at - * `this.transactionInfo` to be "txid" unique at `this.combinedArray`. - */ -AddressHistory.prototype.combineTransactionInfo = function() { - var combinedArrayMap = {}; - this.combinedArray = []; - var l = this.transactionInfo.length; - for(var i = 0; i < l; i++) { - var item = this.transactionInfo[i]; - var mapKey = item.txid; - if (combinedArrayMap[mapKey] >= 0) { - var combined = this.combinedArray[combinedArrayMap[mapKey]]; - if (!combined.addresses[item.address]) { - combined.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - } - if (item.outputIndex >= 0) { - combined.satoshis += item.satoshis; - combined.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - combined.addresses[item.address].inputIndexes.push(item.inputIndex); - } - } else { - item.addresses = {}; - item.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - if (item.outputIndex >= 0) { - item.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - item.addresses[item.address].inputIndexes.push(item.inputIndex); + callback(null, { + totalCount: totalCount, + items: self.detailedArray + }); } - delete item.outputIndex; - delete item.inputIndex; - delete item.address; - this.combinedArray.push(item); - combinedArrayMap[mapKey] = this.combinedArray.length - 1; - } - } + ); + }); }; /** diff --git a/lib/services/address/streams/combined.js b/lib/services/address/streams/combined.js new file mode 100644 index 00000000..050d8474 --- /dev/null +++ b/lib/services/address/streams/combined.js @@ -0,0 +1,166 @@ +'use strict'; + +var ReadableStream = require('stream').Readable; +var inherits = require('util').inherits; + +function TransactionInfoStream(options) { + ReadableStream.call(this, { + objectMode: true + }); + + // TODO: Be able to specify multiple input and output streams + // so that it's possible to query multiple addresses at the same time. + this._inputStream = options.inputStream; + this._outputStream = options.outputStream; + + // This holds a collection of combined inputs and outputs + // grouped into the matching block heights. + this._blocks = {}; + + this._inputCurrentHeight = 0; + this._outputCurrentHeight = 0; + this._inputFinishedHeights = []; + this._outputFinishedHeights = []; + this._inputEnded = false; + this._outputEnded = false; + + this._listenStreamEvents(); +} + +inherits(TransactionInfoStream, ReadableStream); + +TransactionInfoStream.prototype._listenStreamEvents = function() { + var self = this; + + self._inputStream.on('data', function(input) { + self._addToBlock(input); + if (input.height > self._inputCurrentHeight) { + self._inputFinishedHeights.push(input.height); + } + self._inputCurrentHeight = input.height; + self._maybePushBlock(); + }); + + self._outputStream.on('data', function(output) { + self._addToBlock(output); + if (output.height > self._outputCurrentHeight) { + self._outputFinishedHeights.push(output.height); + } + self._outputCurrentHeight = output.height; + self._maybePushBlock(); + }); + + self._inputStream.on('end', function() { + self._inputFinishedHeights.push(self._inputCurrentHeight); + self._inputEnded = true; + self._maybeEndStream(); + }); + + self._outputStream.on('end', function() { + self._outputFinishedHeights.push(self._outputCurrentHeight); + self._outputEnded = true; + self._maybeEndStream(); + }); + +}; + +TransactionInfoStream.prototype._read = function() { + this._inputStream.resume(); + this._outputStream.resume(); +}; + +TransactionInfoStream.prototype._addToBlock = function(data) { + if (!this._blocks[data.height]) { + this._blocks[data.height] = []; + } + this._blocks[data.height].push(data); +}; + +TransactionInfoStream.prototype._maybeEndStream = function() { + if (this._inputEnded && this._outputEnded) { + this._pushRemainingBlocks(); + this.push(null); + } +}; + +TransactionInfoStream.prototype._pushRemainingBlocks = function() { + var keys = Object.keys(this._blocks); + for (var i = 0; i < keys.length; i++) { + this.push(this._blocks[keys[i]]); + delete this._blocks[keys[i]]; + } +}; + +TransactionInfoStream.prototype._combineTransactionInfo = function(transactionInfo) { + var combinedArrayMap = {}; + var combinedArray = []; + var l = transactionInfo.length; + for(var i = 0; i < l; i++) { + var item = transactionInfo[i]; + var mapKey = item.txid; + if (combinedArrayMap[mapKey] >= 0) { + var combined = combinedArray[combinedArrayMap[mapKey]]; + if (!combined.addresses[item.address]) { + combined.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; + } + if (item.outputIndex >= 0) { + combined.satoshis += item.satoshis; + combined.addresses[item.address].outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + combined.addresses[item.address].inputIndexes.push(item.inputIndex); + } + } else { + item.addresses = {}; + item.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; + if (item.outputIndex >= 0) { + item.addresses[item.address].outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + item.addresses[item.address].inputIndexes.push(item.inputIndex); + } + delete item.outputIndex; + delete item.inputIndex; + delete item.address; + combinedArray.push(item); + combinedArrayMap[mapKey] = combinedArray.length - 1; + } + } + return combinedArray; +}; + +TransactionInfoStream.prototype._maybePushBlock = function() { + if (!this._inputFinishedHeights[0] && !this._outputFinishedHeights[0]) { + return; + } + + var inputFinished = this._inputFinishedHeights[0]; + var outputFinished = this._outputFinishedHeights[0]; + var bothFinished; + + if (inputFinished === outputFinished) { + bothFinished = inputFinished; + this._inputFinishedHeights.shift(); + this._outputFinishedHeights.shift(); + } else if (inputFinished <= outputFinished) { + bothFinished = inputFinished; + this._inputFinishedHeights.shift(); + } else if (outputFinished <= inputFinished) { + bothFinished = outputFinished; + this._outputFinishedHeights.shift(); + } + + if (bothFinished) { + var block = this._combineTransactionInfo(this._blocks[bothFinished]); + this.push(block); + delete this._blocks[bothFinished]; + //this._inputStream.pause(); + //this._outputStream.pause(); + } +}; + +module.exports = TransactionInfoStream; From 5c4f3c4453f89b4e7c7ce160c88460c980b2498f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Jan 2016 21:20:10 -0500 Subject: [PATCH 006/299] Address Service: Use address summary cache for pagination --- lib/services/address/constants.js | 2 +- lib/services/address/encoding.js | 18 ++- lib/services/address/history.js | 125 ++++++++++++----- lib/services/address/index.js | 44 ++++-- lib/services/address/streams/combined.js | 166 ----------------------- 5 files changed, 133 insertions(+), 222 deletions(-) delete mode 100644 lib/services/address/streams/combined.js diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index f11b3fe7..6ab3aba1 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -45,7 +45,7 @@ exports.SUMMARY_CACHE_THRESHOLD = 10000; // The default maximum length queries exports.MAX_INPUTS_QUERY_LENGTH = 50000; exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; - +exports.MAX_HISTORY_QUERY_LENGTH = 1000; module.exports = exports; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index ca42fc7f..2f83faab 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -181,8 +181,12 @@ exports.encodeSummaryCacheValue = function(cache, tipHeight) { buffer.writeDoubleBE(cache.result.totalReceived, 4); buffer.writeDoubleBE(cache.result.balance, 12); var txidBuffers = []; - for (var key in cache.result.appearanceIds) { - txidBuffers.push(new Buffer(key, 'hex')); + for (var i = 0; i < cache.result.txids.length; i++) { + var buf = new Buffer(new Array(36)); + var txid = cache.result.txids[i]; + buf.write(txid, 'hex'); + buf.writeUInt32BE(cache.result.appearanceIds[txid], 32); + txidBuffers.push(buf); } var txidsBuffer = Buffer.concat(txidBuffers); var value = Buffer.concat([buffer, txidsBuffer]); @@ -198,17 +202,21 @@ exports.decodeSummaryCacheValue = function(buffer) { // read 32 byte chunks until exhausted var appearanceIds = {}; - var pos = 16; + var txids = []; + var pos = 20; while(pos < buffer.length) { var txid = buffer.slice(pos, pos + 32).toString('hex'); - appearanceIds[txid] = true; - pos += 32; + var txidHeight = buffer.readUInt32BE(pos + 32); + txids.push(txid); + appearanceIds[txid] = txidHeight; + pos += 36; } var cache = { height: height, result: { appearanceIds: appearanceIds, + txids: txids, totalReceived: totalReceived, balance: balance, unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 1795fede..dc15766c 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -2,9 +2,10 @@ var bitcore = require('bitcore-lib'); var async = require('async'); -var CombinedStream = require('./streams/combined'); var _ = bitcore.deps._; +var constants = require('./constants'); + /** * This represents an instance that keeps track of data over a series of * asynchronous I/O calls to get the transaction history for a group of @@ -20,7 +21,21 @@ function AddressHistory(args) { } else { this.addresses = [args.addresses]; } - this.combinedArray = []; + + this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH; + + this.addressStrings = []; + for (var i = 0; i < this.addresses.length; i++) { + var address = this.addresses[i]; + if (address instanceof bitcore.Address) { + this.addressStrings.push(address.toString()); + } else if (_.isString(address)) { + this.addressStrings.push(address); + } else { + throw new TypeError('Addresses are expected to be strings'); + } + } + this.detailedArray = []; } @@ -42,28 +57,33 @@ AddressHistory.prototype.get = function(callback) { var address = self.addresses[0]; - var combinedStream = new CombinedStream({ - inputStream: this.node.services.address.createInputsStream(address, this.options), - outputStream: this.node.services.address.createOutputsStream(address, this.options) - }); + this.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } - // Results from the transaction info stream are grouped into - // sets based on block height - combinedStream.on('data', function(block) { - self.combinedArray = self.combinedArray.concat(block); - }); + totalCount = summary.txids.length; - combinedStream.on('end', function() { - totalCount = Number(self.combinedArray.length); + // TODO: Make sure txids are sorted by height and time + var fromOffset = summary.txids.length - self.options.from; + var toOffset = summary.txids.length - self.options.to; + var txids = summary.txids.slice(toOffset, fromOffset); - self.sortAndPaginateCombinedArray(); + // Verify that this query isn't too long + if (txids.length > self.maxHistoryQueryLength) { + return callback(new Error( + 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + + this.address.join(',') + )); + } - // TODO: Add the mempool transactions + // Reverse to include most recent at the top + txids.reverse(); async.eachSeries( - self.combinedArray, - function(txInfo, next) { - self.getDetailedInfo(txInfo, next); + txids, + function(txid, next) { + self.getDetailedInfo(txid, next); }, function(err) { if (err) { @@ -75,12 +95,15 @@ AddressHistory.prototype.get = function(callback) { }); } ); + }); + }; /** * A helper function to sort and slice/paginate the `combinedArray` */ +// TODO: Remove once txids summary results are verified to be sorted AddressHistory.prototype.sortAndPaginateCombinedArray = function() { this.combinedArray.sort(AddressHistory.sortByHeight); if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { @@ -94,6 +117,7 @@ AddressHistory.prototype.sortAndPaginateCombinedArray = function() { * @param {Object} a - An item from the `combinedArray` * @param {Object} b */ +// TODO: Remove once txids summary results are verified to be sorted AddressHistory.sortByHeight = function(a, b) { if (a.height < 0 && b.height < 0) { // Both are from the mempool, compare timestamps @@ -123,12 +147,12 @@ AddressHistory.sortByHeight = function(a, b) { * @param {Object} txInfo - An item from the `combinedArray` * @param {Function} next */ -AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { +AddressHistory.prototype.getDetailedInfo = function(txid, next) { var self = this; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; self.node.services.db.getTransactionWithBlockInfo( - txInfo.txid, + txid, queryMempool, function(err, transaction) { if (err) { @@ -136,13 +160,15 @@ AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { } transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { + if (err) { return next(err); } + var addressDetails = self.getAddressDetailsForTransaction(transaction); + self.detailedArray.push({ - addresses: txInfo.addresses, - satoshis: self.getSatoshisDetail(transaction, txInfo), + addresses: addressDetails.addresses, + satoshis: addressDetails.satoshis, height: transaction.__height, confirmations: self.getConfirmationsDetail(transaction), timestamp: transaction.__timestamp, @@ -169,23 +195,52 @@ AddressHistory.prototype.getConfirmationsDetail = function(transaction) { return confirmations; }; -/** - * A helper function for `getDetailedInfo` for getting the satoshis. - * @param {Transaction} transaction - A transaction populated with previous outputs - * @param {Object} txInfo - An item from `combinedArray` - */ -AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { - var satoshis = txInfo.satoshis || 0; +AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) { + var result = { + addresses: {}, + satoshis: 0 + }; + + for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { + var input = transaction.inputs[inputIndex]; + if (!input.script) { + continue; + } + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress && this.addressStrings.indexOf(inputAddress.toString()) > 0) { + if (!result.addresses[inputAddress]) { + result.addresses[inputAddress] = { + inputIndexes: [], + outputIndexes: [] + }; + } else { + result.addresses[inputAddress].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; + } + } - for(var address in txInfo.addresses) { - if (txInfo.addresses[address].inputIndexes.length >= 0) { - for(var j = 0; j < txInfo.addresses[address].inputIndexes.length; j++) { - satoshis -= transaction.inputs[txInfo.addresses[address].inputIndexes[j]].output.satoshis; + for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { + var output = transaction.outputs[outputIndex]; + if (!output.script) { + continue; + } + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress && this.addressStrings.indexOf(outputAddress.toString()) > 0) { + if (!result.addresses[outputAddress]) { + result.addresses[outputAddress] = { + inputIndexes: [], + outputIndexes: [] + }; + } else { + result.addresses[outputAddress].inputIndexes.push(outputIndex); } + result.satoshis += output.satoshis; } } - return satoshis; + return result; + }; module.exports = AddressHistory; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 1eeb0404..0d3a5576 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -763,7 +763,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options * @param {Number} [options.end] - The relevant end block height * @param {Function} callback */ -AddressService.prototype.createInputsStream = function(addressStr, options, callback) { +AddressService.prototype.createInputsStream = function(addressStr, options) { var inputStream = new InputsTransformStream({ address: new Address(addressStr, this.node.network), @@ -1331,6 +1331,9 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb function(cache, next) { self._getAddressOutputsSummary(address, cache, tipHeight, next); }, + function(cache, next) { + self._sortTxids(cache, tipHeight, next); + }, function(cache, next) { self._saveAddressSummaryCache(address, cache, tipHeight, next); } @@ -1340,7 +1343,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } var result = cache.result; - var confirmedTxids = Object.keys(result.appearanceIds); + var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { @@ -1360,16 +1363,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } if (!options.noTxList) { - var txids = confirmedTxids.concat(unconfirmedTxids); - - // sort by height - summary.txids = txids.sort(function(a, b) { - return a.height > b.height ? 1 : -1; - }).map(function(obj) { - return obj.txid; - }).filter(function(value, index, self) { - return self.indexOf(value) === index; - }); + summary.txids = confirmedTxids.concat(unconfirmedTxids); } callback(null, summary); @@ -1378,8 +1372,22 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }; +AddressService.prototype._sortTxids = function(cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } + cache.result.txids = Object.keys(cache.result.appearanceIds); + cache.result.txids.sort(function(a, b) { + return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; + }); + callback(null, cache); +}; + AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { - var transactionLength = Object.keys(cache.result.appearanceIds).length; + if (cache.height === tipHeight) { + return callback(null, cache); + } + var transactionLength = cache.result.txids.length; var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); if (exceedsCacheThreshold) { log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); @@ -1422,6 +1430,9 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { }; AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } $.checkArgument(address instanceof Address); var self = this; @@ -1435,7 +1446,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip var inputsStream = self.createInputsStream(address, opts); inputsStream.on('data', function(input) { var txid = input.txid; - cache.result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = input.height; }); inputsStream.on('error', function(err) { @@ -1462,6 +1473,9 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip }; AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { + if (cache.height === tipHeight) { + return callback(null, cache); + } $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(cache.result) && !_.isUndefined(cache.result.appearanceIds) && @@ -1484,7 +1498,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); cache.result.totalReceived += output.satoshis; - cache.result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = output.height; if (!spentDB) { cache.result.balance += output.satoshis; diff --git a/lib/services/address/streams/combined.js b/lib/services/address/streams/combined.js deleted file mode 100644 index 050d8474..00000000 --- a/lib/services/address/streams/combined.js +++ /dev/null @@ -1,166 +0,0 @@ -'use strict'; - -var ReadableStream = require('stream').Readable; -var inherits = require('util').inherits; - -function TransactionInfoStream(options) { - ReadableStream.call(this, { - objectMode: true - }); - - // TODO: Be able to specify multiple input and output streams - // so that it's possible to query multiple addresses at the same time. - this._inputStream = options.inputStream; - this._outputStream = options.outputStream; - - // This holds a collection of combined inputs and outputs - // grouped into the matching block heights. - this._blocks = {}; - - this._inputCurrentHeight = 0; - this._outputCurrentHeight = 0; - this._inputFinishedHeights = []; - this._outputFinishedHeights = []; - this._inputEnded = false; - this._outputEnded = false; - - this._listenStreamEvents(); -} - -inherits(TransactionInfoStream, ReadableStream); - -TransactionInfoStream.prototype._listenStreamEvents = function() { - var self = this; - - self._inputStream.on('data', function(input) { - self._addToBlock(input); - if (input.height > self._inputCurrentHeight) { - self._inputFinishedHeights.push(input.height); - } - self._inputCurrentHeight = input.height; - self._maybePushBlock(); - }); - - self._outputStream.on('data', function(output) { - self._addToBlock(output); - if (output.height > self._outputCurrentHeight) { - self._outputFinishedHeights.push(output.height); - } - self._outputCurrentHeight = output.height; - self._maybePushBlock(); - }); - - self._inputStream.on('end', function() { - self._inputFinishedHeights.push(self._inputCurrentHeight); - self._inputEnded = true; - self._maybeEndStream(); - }); - - self._outputStream.on('end', function() { - self._outputFinishedHeights.push(self._outputCurrentHeight); - self._outputEnded = true; - self._maybeEndStream(); - }); - -}; - -TransactionInfoStream.prototype._read = function() { - this._inputStream.resume(); - this._outputStream.resume(); -}; - -TransactionInfoStream.prototype._addToBlock = function(data) { - if (!this._blocks[data.height]) { - this._blocks[data.height] = []; - } - this._blocks[data.height].push(data); -}; - -TransactionInfoStream.prototype._maybeEndStream = function() { - if (this._inputEnded && this._outputEnded) { - this._pushRemainingBlocks(); - this.push(null); - } -}; - -TransactionInfoStream.prototype._pushRemainingBlocks = function() { - var keys = Object.keys(this._blocks); - for (var i = 0; i < keys.length; i++) { - this.push(this._blocks[keys[i]]); - delete this._blocks[keys[i]]; - } -}; - -TransactionInfoStream.prototype._combineTransactionInfo = function(transactionInfo) { - var combinedArrayMap = {}; - var combinedArray = []; - var l = transactionInfo.length; - for(var i = 0; i < l; i++) { - var item = transactionInfo[i]; - var mapKey = item.txid; - if (combinedArrayMap[mapKey] >= 0) { - var combined = combinedArray[combinedArrayMap[mapKey]]; - if (!combined.addresses[item.address]) { - combined.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - } - if (item.outputIndex >= 0) { - combined.satoshis += item.satoshis; - combined.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - combined.addresses[item.address].inputIndexes.push(item.inputIndex); - } - } else { - item.addresses = {}; - item.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - if (item.outputIndex >= 0) { - item.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - item.addresses[item.address].inputIndexes.push(item.inputIndex); - } - delete item.outputIndex; - delete item.inputIndex; - delete item.address; - combinedArray.push(item); - combinedArrayMap[mapKey] = combinedArray.length - 1; - } - } - return combinedArray; -}; - -TransactionInfoStream.prototype._maybePushBlock = function() { - if (!this._inputFinishedHeights[0] && !this._outputFinishedHeights[0]) { - return; - } - - var inputFinished = this._inputFinishedHeights[0]; - var outputFinished = this._outputFinishedHeights[0]; - var bothFinished; - - if (inputFinished === outputFinished) { - bothFinished = inputFinished; - this._inputFinishedHeights.shift(); - this._outputFinishedHeights.shift(); - } else if (inputFinished <= outputFinished) { - bothFinished = inputFinished; - this._inputFinishedHeights.shift(); - } else if (outputFinished <= inputFinished) { - bothFinished = outputFinished; - this._outputFinishedHeights.shift(); - } - - if (bothFinished) { - var block = this._combineTransactionInfo(this._blocks[bothFinished]); - this.push(block); - delete this._blocks[bothFinished]; - //this._inputStream.pause(); - //this._outputStream.pause(); - } -}; - -module.exports = TransactionInfoStream; From 8d2f69c5fddd0cbf3504e2833b8c771995c261dc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 14:34:04 -0500 Subject: [PATCH 007/299] Address Service: Restored multi-address history queries - Restored functionality to be able to query the history of multiple addresses in one query - Sorted mempool transactions by timestamp in txid lists --- lib/services/address/constants.js | 11 ++- lib/services/address/history.js | 121 ++++++++++++++++-------------- lib/services/address/index.js | 35 ++++++--- 3 files changed, 94 insertions(+), 73 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 6ab3aba1..00fbafbe 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -36,16 +36,21 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); +exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); // The total number of transactions that an address can receive before it will start // to cache the summary to disk. exports.SUMMARY_CACHE_THRESHOLD = 10000; - -// The default maximum length queries +// The maximum number of inputs that can be queried at once exports.MAX_INPUTS_QUERY_LENGTH = 50000; +// The maximum number of outputs that can be queried at once exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; -exports.MAX_HISTORY_QUERY_LENGTH = 1000; +// The maximum number of transactions that can be queried at once +exports.MAX_HISTORY_QUERY_LENGTH = 100; +// The maximum number of addresses that can be queried at once +exports.MAX_ADDRESSES_QUERY = 100; module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js index dc15766c..f1b42cd5 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -22,7 +22,8 @@ function AddressHistory(args) { this.addresses = [args.addresses]; } - this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH; + this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; + this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; this.addressStrings = []; for (var i = 0; i < this.addresses.length; i++) { @@ -39,7 +40,33 @@ function AddressHistory(args) { this.detailedArray = []; } -AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.prototype._mergeAndSortTxids = function(summaries) { + var appearanceIds = {}; + var unconfirmedAppearanceIds = {}; + for (var i = 0; i < summaries.length; i++) { + var summary = summaries[i]; + for (var key in summary.appearanceIds) { + appearanceIds[key] = summary.appearanceIds[key]; + delete summary.appearanceIds[key]; + } + for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key]; + delete summary.unconfirmedAppearanceIds[key]; + } + } + var confirmedTxids = Object.keys(appearanceIds); + confirmedTxids.sort(function(a, b) { + // Confirmed are sorted by height + return appearanceIds[a] - appearanceIds[b]; + }); + var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); + unconfirmedTxids.sort(function(a, b) { + // Unconfirmed are sorted by timestamp + return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; + }); + var txids = confirmedTxids.concat(unconfirmedTxids); + return txids; +}; /** * This function will give detailed history for the configured @@ -50,30 +77,49 @@ AddressHistory.prototype.get = function(callback) { var self = this; var totalCount; - // TODO: handle multiple addresses (restore previous functionality) - if (self.addresses.length > 1) { - return callback('Only single address queries supported currently'); + if (this.addresses.length > this.maxAddressesQuery) { + return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } - var address = self.addresses[0]; - - this.node.services.address.getAddressSummary(address, this.options, function(err, summary) { - if (err) { - return callback(err); - } + if (this.addresses.length === 0) { + var address = this.addresses[0]; + self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } + return finish(summary.txids); + }); + } else { + var opts = _.clone(this.options); + opts.fullTxList = true; + async.map( + self.addresses, + function(address, next) { + self.node.services.address.getAddressSummary(address, opts, next); + }, + function(err, summaries) { + if (err) { + return callback(err); + } + var txids = self._mergeAndSortTxids(summaries); + return finish(txids); + } + ); + } - totalCount = summary.txids.length; + function finish(allTxids) { + totalCount = allTxids.length; - // TODO: Make sure txids are sorted by height and time - var fromOffset = summary.txids.length - self.options.from; - var toOffset = summary.txids.length - self.options.to; - var txids = summary.txids.slice(toOffset, fromOffset); + // Slice the page starting with the most recent + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + var txids = allTxids.slice(toOffset, fromOffset); // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { return callback(new Error( 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - this.address.join(',') + self.address.join(',') )); } @@ -96,49 +142,8 @@ AddressHistory.prototype.get = function(callback) { } ); - }); - -}; - -/** - * A helper function to sort and slice/paginate the `combinedArray` - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.prototype.sortAndPaginateCombinedArray = function() { - this.combinedArray.sort(AddressHistory.sortByHeight); - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); } -}; -/** - * A helper sort function to order by height and then by date - * for transactions that are in the mempool. - * @param {Object} a - An item from the `combinedArray` - * @param {Object} b - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.sortByHeight = function(a, b) { - if (a.height < 0 && b.height < 0) { - // Both are from the mempool, compare timestamps - if (a.timestamp === b.timestamp) { - return 0; - } else { - return a.timestamp < b.timestamp ? 1 : -1; - } - } else if (a.height < 0 && b.height > 0) { - // A is from the mempool and B is in a block - return -1; - } else if (a.height > 0 && b.height < 0) { - // A is in a block and B is in the mempool - return 1; - } else if (a.height === b.height) { - // The heights are equal - return 0; - } else { - // Otherwise compare heights - return a.height < b.height ? 1 : -1; - } }; /** diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 0d3a5576..0a0a8bac 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -298,6 +298,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { /* jshint maxstatements: 100 */ var operations = []; + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(new Date().getTime()); var action = 'put'; if (!add) { @@ -326,6 +328,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + timestampBuffer, txidBuffer, outputIndexBuffer ]); @@ -392,6 +395,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, + timestampBuffer, input.prevTxId, inputOutputIndexBuffer ]); @@ -899,22 +903,23 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { + var timestamp = data.key.readDoubleBE(22); var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); - var output = { + var input = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer @@ -922,7 +927,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha height: -1, confirmations: 0 }; - mempoolInputs.push(output); + mempoolInputs.push(input); }); var error; @@ -1108,22 +1113,24 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 - var txid = data.key.slice(22, 54); - var outputIndex = data.key.readUInt32BE(54); + // Format of data: + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4 + var timestamp = data.key.readDoubleBE(22); + var txid = data.key.slice(30, 62); + var outputIndex = data.key.readUInt32BE(62); var value = encoding.decodeOutputValue(data.value); var output = { address: addressStr, @@ -1131,6 +1138,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, + timestamp: timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1362,7 +1370,10 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb log.warn('Address Summary:', summary); } - if (!options.noTxList) { + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { summary.txids = confirmedTxids.concat(unconfirmedTxids); } @@ -1465,7 +1476,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } callback(error, cache); }); @@ -1537,7 +1548,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - cache.result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly From 188ff28ec786660179f8f156cf511b2c056562f8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 16:51:00 -0500 Subject: [PATCH 008/299] Address Service: Fixed HASH_TYPES_MAP naming issue --- lib/services/address/encoding.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 2f83faab..1f0933ad 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -161,7 +161,7 @@ exports.decodeInputValueMap = function(buffer) { }; exports.encodeSummaryCacheKey = function(address) { - return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_BUFFER[address.type]]); + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_MAP[address.type]]); }; exports.decodeSummaryCacheKey = function(buffer, network) { From 4fcec8755c3007bdbf78d07486182034ec0bdb7c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 18:45:51 -0500 Subject: [PATCH 009/299] Address Service: Fixed many bugs from tests - Refactored getAddressSummary and added several tests - Fixed bugs revealed from the integration regtests - Updated many unit tests --- integration/regtest-node.js | 36 +- lib/services/address/constants.js | 2 + lib/services/address/encoding.js | 32 +- lib/services/address/history.js | 58 +- lib/services/address/index.js | 408 ++++--- package.json | 4 +- test/services/address/encoding.unit.js | 103 ++ test/services/address/history.unit.js | 309 +----- test/services/address/index.unit.js | 1346 +++++++++++++++++++----- 9 files changed, 1517 insertions(+), 781 deletions(-) create mode 100644 test/services/address/encoding.unit.js diff --git a/integration/regtest-node.js b/integration/regtest-node.js index fb7ff6bf..e05e3039 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -28,6 +28,7 @@ var Transaction = index.Transaction; var BitcoreNode = index.Node; var AddressService = index.services.Address; var BitcoinService = index.services.Bitcoin; +var encoding = require('../lib/services/address/encoding'); var DBService = index.services.DB; var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG'; var testKey; @@ -43,22 +44,6 @@ describe('Node Functionality', function() { before(function(done) { this.timeout(30000); - // Add the regtest network - bitcore.Networks.remove(bitcore.Networks.testnet); - bitcore.Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); - regtest = bitcore.Networks.get('regtest'); - var datadir = __dirname + '/data'; testKey = bitcore.PrivateKey(testWIF); @@ -93,6 +78,9 @@ describe('Node Functionality', function() { node = new BitcoreNode(configuration); + regtest = bitcore.Networks.get('regtest'); + should.exist(regtest); + node.on('error', function(err) { log.error(err); }); @@ -208,7 +196,7 @@ describe('Node Functionality', function() { // We need to add a transaction to the mempool so that the next block will // have a different hash as the hash has been invalidated. - client.sendToAddress(testKey.toAddress().toString(), 10, function(err) { + client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { if (err) { throw err; } @@ -250,7 +238,7 @@ describe('Node Functionality', function() { var address; var unspentOutput; before(function() { - address = testKey.toAddress().toString(); + address = testKey.toAddress(regtest).toString(); }); it('should be able to get the balance of the test address', function(done) { node.services.address.getBalance(address, false, function(err, balance) { @@ -333,19 +321,19 @@ describe('Node Functionality', function() { /* jshint maxstatements: 50 */ testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x'); - address2 = testKey2.toAddress().toString(); + address2 = testKey2.toAddress(regtest).toString(); testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt'); - address3 = testKey3.toAddress().toString(); + address3 = testKey3.toAddress(regtest).toString(); testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw'); - address4 = testKey4.toAddress().toString(); + address4 = testKey4.toAddress(regtest).toString(); testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF'); - address5 = testKey5.toAddress().toString(); + address5 = testKey5.toAddress(regtest).toString(); testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF'); - address6 = testKey6.toAddress().toString(); + address6 = testKey6.toAddress(regtest).toString(); var tx = new Transaction(); tx.from(unspentOutput); @@ -726,7 +714,7 @@ describe('Node Functionality', function() { node.services.bitcoind.sendTransaction(tx.serialize()); setImmediate(function() { - var addrObj = node.services.address._getAddressInfo(address); + var addrObj = encoding.getAddressInfo(address); node.services.address._getOutputsMempool(address, addrObj.hashBuffer, addrObj.hashTypeBuffer, function(err, outs) { if (err) { diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 00fbafbe..7cc8ef58 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -36,6 +36,8 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex'); +exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 1f0933ad..81421a18 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -61,6 +61,12 @@ exports.encodeOutputValue = function(satoshis, scriptBuffer) { return Buffer.concat([satoshisBuffer, scriptBuffer]); }; +exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]); +}; + exports.decodeOutputValue = function(buffer) { var satoshis = buffer.readDoubleBE(0); var scriptBuffer = buffer.slice(8, buffer.length); @@ -70,6 +76,17 @@ exports.decodeOutputValue = function(buffer) { }; }; +exports.decodeOutputMempoolValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var timestamp = buffer.readDoubleBE(8); + var scriptBuffer = buffer.slice(16, buffer.length); + return { + satoshis: satoshis, + timestamp: timestamp, + scriptBuffer: scriptBuffer + }; +}; + exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); @@ -175,7 +192,8 @@ exports.decodeSummaryCacheKey = function(buffer, network) { return address; }; -exports.encodeSummaryCacheValue = function(cache, tipHeight) { +exports.encodeSummaryCacheValue = function(cache, tipHeight, tipHash) { + var tipHashBuffer = new Buffer(tipHash, 'hex'); var buffer = new Buffer(new Array(20)); buffer.writeUInt32BE(tipHeight); buffer.writeDoubleBE(cache.result.totalReceived, 4); @@ -189,21 +207,22 @@ exports.encodeSummaryCacheValue = function(cache, tipHeight) { txidBuffers.push(buf); } var txidsBuffer = Buffer.concat(txidBuffers); - var value = Buffer.concat([buffer, txidsBuffer]); + var value = Buffer.concat([tipHashBuffer, buffer, txidsBuffer]); return value; }; exports.decodeSummaryCacheValue = function(buffer) { - var height = buffer.readUInt32BE(); - var totalReceived = buffer.readDoubleBE(4); - var balance = buffer.readDoubleBE(12); + var hash = buffer.slice(0, 32).toString('hex'); + var height = buffer.readUInt32BE(32); + var totalReceived = buffer.readDoubleBE(36); + var balance = buffer.readDoubleBE(44); // read 32 byte chunks until exhausted var appearanceIds = {}; var txids = []; - var pos = 20; + var pos = 52; while(pos < buffer.length) { var txid = buffer.slice(pos, pos + 32).toString('hex'); var txidHeight = buffer.readUInt32BE(pos + 32); @@ -214,6 +233,7 @@ exports.decodeSummaryCacheValue = function(buffer) { var cache = { height: height, + hash: hash, result: { appearanceIds: appearanceIds, txids: txids, diff --git a/lib/services/address/history.js b/lib/services/address/history.js index f1b42cd5..eb689828 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -64,8 +64,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { // Unconfirmed are sorted by timestamp return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; }); - var txids = confirmedTxids.concat(unconfirmedTxids); - return txids; + return confirmedTxids.concat(unconfirmedTxids); }; /** @@ -81,7 +80,7 @@ AddressHistory.prototype.get = function(callback) { return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } - if (this.addresses.length === 0) { + if (this.addresses.length === 1) { var address = this.addresses[0]; self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { if (err) { @@ -111,9 +110,14 @@ AddressHistory.prototype.get = function(callback) { totalCount = allTxids.length; // Slice the page starting with the most recent - var fromOffset = totalCount - self.options.from; - var toOffset = totalCount - self.options.to; - var txids = allTxids.slice(toOffset, fromOffset); + var txids; + if (self.options.from >= 0 && self.options.to >= 0) { + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + txids = allTxids.slice(toOffset, fromOffset); + } else { + txids = allTxids; + } // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { @@ -212,16 +216,19 @@ AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) continue; } var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress && this.addressStrings.indexOf(inputAddress.toString()) > 0) { - if (!result.addresses[inputAddress]) { - result.addresses[inputAddress] = { - inputIndexes: [], - outputIndexes: [] - }; - } else { - result.addresses[inputAddress].inputIndexes.push(inputIndex); + if (inputAddress) { + var inputAddressString = inputAddress.toString(); + if (this.addressStrings.indexOf(inputAddressString) >= 0) { + if (!result.addresses[inputAddressString]) { + result.addresses[inputAddressString] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[inputAddressString].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; } - result.satoshis -= input.output.satoshis; } } @@ -231,16 +238,19 @@ AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) continue; } var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress && this.addressStrings.indexOf(outputAddress.toString()) > 0) { - if (!result.addresses[outputAddress]) { - result.addresses[outputAddress] = { - inputIndexes: [], - outputIndexes: [] - }; - } else { - result.addresses[outputAddress].inputIndexes.push(outputIndex); + if (outputAddress) { + var outputAddressString = outputAddress.toString(); + if (this.addressStrings.indexOf(outputAddressString) >= 0) { + if (!result.addresses[outputAddressString]) { + result.addresses[outputAddressString] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[outputAddressString].outputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; } - result.satoshis += output.satoshis; } } diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 0a0a8bac..22c2420f 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -328,12 +328,15 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - timestampBuffer, txidBuffer, outputIndexBuffer ]); - var outValue = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); + var outValue = encoding.encodeOutputMempoolValue( + output.satoshis, + timestampBuffer, + output._scriptBuffer + ); operations.push({ type: action, @@ -395,13 +398,13 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, - timestampBuffer, input.prevTxId, inputOutputIndexBuffer ]); var inputValue = Buffer.concat([ txidBuffer, - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); operations.push({ type: action, @@ -768,13 +771,17 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options * @param {Function} callback */ AddressService.prototype.createInputsStream = function(addressStr, options) { - var inputStream = new InputsTransformStream({ address: new Address(addressStr, this.node.network), tipHeight: this.node.services.db.tip.__height }); - var stream = this.createInputsDBStream(addressStr, options).pipe(inputStream); + var stream = this.createInputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + inputStream.emit('error', err); + inputStream.end(); + }).pipe(inputStream); return stream; @@ -786,23 +793,27 @@ AddressService.prototype.createInputsDBStream = function(addressStr, options) { var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (options.start && options.end) { + if (options.start >= 0 && options.end >= 0) { var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); + endBuffer.writeUInt32BE(options.end, 0); var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var adjustedStart = options.start + 1; + startBuffer.writeUInt32BE(adjustedStart, 0); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ + gt: Buffer.concat([ constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, constants.SPACER_MIN, endBuffer ]), - lte: Buffer.concat([ + lt: Buffer.concat([ constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, @@ -815,8 +826,8 @@ AddressService.prototype.createInputsDBStream = function(addressStr, options) { } else { var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, constants.SPACER_MIN]), - lte: Buffer.concat([allKey, constants.SPACER_MAX]), + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); @@ -903,27 +914,28 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - var timestamp = data.key.readDoubleBE(22); var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); + var timestamp = data.value.readDoubleBE(36); var input = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, + timestamp: timestamp, height: -1, confirmations: 0 }; @@ -979,7 +991,13 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { tipHeight: this.node.services.db.tip.__height }); - var stream = this.createOutputsDBStream(addressStr, options).pipe(outputStream); + var stream = this.createOutputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + outputStream.emit('error', err); + outputStream.end(); + }) + .pipe(outputStream); return stream; @@ -992,22 +1010,27 @@ AddressService.prototype.createOutputsDBStream = function(addressStr, options) { var hashTypeBuffer = addrObj.hashTypeBuffer; var stream; - if (options.start && options.end) { + if (options.start >= 0 && options.end >= 0) { - var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); + endBuffer.writeUInt32BE(options.end, 0); + + var startBuffer = new Buffer(4); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var startAdjusted = options.start + 1; + startBuffer.writeUInt32BE(startAdjusted, 0); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ + gt: Buffer.concat([ constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, constants.SPACER_MIN, endBuffer ]), - lte: Buffer.concat([ + lt: Buffer.concat([ constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, @@ -1020,8 +1043,8 @@ AddressService.prototype.createOutputsDBStream = function(addressStr, options) { } else { var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, constants.SPACER_MIN]), - lte: Buffer.concat([allKey, constants.SPACER_MAX]), + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), valueEncoding: 'binary', keyEncoding: 'binary' }); @@ -1113,13 +1136,13 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.TIMESTAMP_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1127,18 +1150,17 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h stream.on('data', function(data) { // Format of data: - // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4 - var timestamp = data.key.readDoubleBE(22); - var txid = data.key.slice(30, 62); - var outputIndex = data.key.readUInt32BE(62); - var value = encoding.decodeOutputValue(data.value); + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 + var txid = data.key.slice(22, 54); + var outputIndex = data.key.readUInt32BE(54); + var value = encoding.decodeOutputMempoolValue(data.value); var output = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, - timestamp: timestamp, + timestamp: value.timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1325,43 +1347,25 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb var self = this; var startTime = new Date(); - var address = new Address(addressArg); - var tipHeight = this.node.services.db.tip.__height; + + if (_.isUndefined(options.queryMempool)) { + options.queryMempool = true; + } async.waterfall([ function(next) { - self._getAddressSummaryCache(address, next); - }, - function(cache, next) { - self._getAddressInputsSummary(address, cache, tipHeight, next); - }, - function(cache, next) { - self._getAddressOutputsSummary(address, cache, tipHeight, next); + self._getAddressConfirmedSummary(address, options, next); }, function(cache, next) { - self._sortTxids(cache, tipHeight, next); - }, - function(cache, next) { - self._saveAddressSummaryCache(address, cache, tipHeight, next); + self._getAddressMempoolSummary(address, options, cache, next); } ], function(err, cache) { if (err) { return callback(err); } - var result = cache.result; - var confirmedTxids = result.txids; - var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); - - var summary = { - totalReceived: result.totalReceived, - totalSpent: result.totalReceived - result.balance, - balance: result.balance, - appearances: confirmedTxids.length, - unconfirmedBalance: result.unconfirmedBalance, - unconfirmedAppearances: unconfirmedTxids.length - }; + var summary = self._transformAddressSummaryFromCache(cache, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { @@ -1370,52 +1374,30 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb log.warn('Address Summary:', summary); } - if (options.fullTxList) { - summary.appearanceIds = result.appearanceIds; - summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; - } else if (!options.noTxList) { - summary.txids = confirmedTxids.concat(unconfirmedTxids); - } - callback(null, summary); }); }; -AddressService.prototype._sortTxids = function(cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } - cache.result.txids = Object.keys(cache.result.appearanceIds); - cache.result.txids.sort(function(a, b) { - return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; - }); - callback(null, cache); -}; +AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { + var self = this; + var tipHeight = this.node.services.db.tip.__height; -AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } - var transactionLength = cache.result.txids.length; - var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); - if (exceedsCacheThreshold) { - log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); - var key = encoding.encodeSummaryCacheKey(address); - var value = encoding.encodeSummaryCacheValue(cache, tipHeight); - this.summaryCache.put(key, value, function(err) { - if (err) { - return callback(err); - } - callback(null, cache); - }); - } else { - callback(null, cache); - } + self._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return callback(err); + } + // Immediately give cache is already current, otherwise update + if (cache && cache.height === tipHeight) { + return callback(null, cache); + } + self._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, callback); + }); }; -AddressService.prototype._getAddressSummaryCache = function(address, callback) { +AddressService.prototype._getAddressConfirmedSummaryCache = function(address, options, callback) { + var self = this; var baseCache = { result: { appearanceIds: {}, @@ -1425,6 +1407,11 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { unconfirmedBalance: 0 } }; + // Use the base cache if the "start" and "end" options have been used + // We only save and retrieve a cache for the summary of all history + if (options.start >= 0 || options.end >= 0) { + return callback(null, baseCache); + } var key = encoding.encodeSummaryCacheKey(address); this.summaryCache.get(key, { valueEncoding: 'binary', @@ -1436,25 +1423,64 @@ AddressService.prototype._getAddressSummaryCache = function(address, callback) { return callback(err); } var cache = encoding.decodeSummaryCacheValue(buffer); + + // Use base cache if the cached tip/height doesn't match (e.g. there has been a reorg) + var blockIndex = self.node.services.bitcoind.getBlockIndex(cache.height); + if (cache.hash !== blockIndex.hash) { + return callback(null, baseCache); + } + callback(null, cache); }); }; -AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); +AddressService.prototype._updateAddressConfirmedSummaryCache = function(address, options, cache, tipHeight, callback) { + var self = this; + + var optionsPartial = _.clone(options); + var isHeightQuery = (options.start >= 0 || options.end >= 0); + if (!isHeightQuery) { + // We will pick up from the last point cached and query for all blocks + // proceeding the cache + var cacheHeight = _.isUndefined(cache.height) ? 0 : cache.height + 1; + optionsPartial.start = tipHeight; + optionsPartial.end = cacheHeight; + } else { + $.checkState(_.isUndefined(cache.height)); } + + async.waterfall([ + function(next) { + self._getAddressConfirmedInputsSummary(address, cache, optionsPartial, next); + }, + function(cache, next) { + self._getAddressConfirmedOutputsSummary(address, cache, optionsPartial, next); + }, + function(cache, next) { + self._setAndSortTxidsFromAppearanceIds(cache, next); + } + ], function(err, cache) { + + // Skip saving the cache if the "start" or "end" options have been used, or + // if the transaction length does not exceed the caching threshold. + // We only want to cache full history results for addresses that have a large + // number of transactions. + var exceedsCacheThreshold = (cache.result.txids.length > self.summaryCacheThreshold); + if (exceedsCacheThreshold && !isHeightQuery) { + self._saveAddressConfirmedSummaryCache(address, cache, tipHeight, callback); + } else { + callback(null, cache); + } + + }); +}; + +AddressService.prototype._getAddressConfirmedInputsSummary = function(address, cache, options, callback) { $.checkArgument(address instanceof Address); var self = this; - var error = null; - var opts = { - start: _.isUndefined(cache.height) ? 0 : cache.height + 1, - end: tipHeight - }; - - var inputsStream = self.createInputsStream(address, opts); + var inputsStream = self.createInputsStream(address, options); inputsStream.on('data', function(input) { var txid = input.txid; cache.result.appearanceIds[txid] = input.height; @@ -1465,28 +1491,14 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip }); inputsStream.on('end', function() { - - var addressStr = address.toString(); - var hashBuffer = address.hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - - self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { - if (err) { - return callback(err); - } - for(var i = 0; i < mempoolInputs.length; i++) { - var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; - } - callback(error, cache); - }); + if (error) { + return callback(error); + } + callback(null, cache); }); }; -AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { - if (cache.height === tipHeight) { - return callback(null, cache); - } +AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, cache, options, callback) { $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(cache.result) && !_.isUndefined(cache.result.appearanceIds) && @@ -1494,12 +1506,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti var self = this; - var opts = { - start: _.isUndefined(cache.height) ? 0 : cache.height + 1, - end: tipHeight - }; - - var outputStream = self.createOutputsStream(address, opts); + var outputStream = self.createOutputsStream(address, options); outputStream.on('data', function(output) { @@ -1514,16 +1521,19 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti if (!spentDB) { cache.result.balance += output.satoshis; } - - // Check to see if this output is spent in the mempool and if so - // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(txid, 'hex'), // TODO: get buffer directly - outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - if (spentMempool) { - cache.result.unconfirmedBalance -= output.satoshis; + // TODO: subtract if spent (because of cache)? + + if (options.queryMempool) { + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + cache.result.unconfirmedBalance -= output.satoshis; + } } }); @@ -1535,37 +1545,111 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti }); outputStream.on('end', function() { + if (error) { + return callback(error); + } + callback(null, cache); + }); - var addressStr = address.toString(); - var hashBuffer = address.hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; +}; - self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { - if (err) { - return callback(err); - } +AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(cache, callback) { + cache.result.txids = Object.keys(cache.result.appearanceIds); + cache.result.txids.sort(function(a, b) { + return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; + }); + callback(null, cache); +}; + +AddressService.prototype._saveAddressConfirmedSummaryCache = function(address, cache, tipHeight, callback) { - for(var i = 0; i < mempoolOutputs.length; i++) { - var output = mempoolOutputs[i]; + log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); + var key = encoding.encodeSummaryCacheKey(address); + var tipBlockIndex = this.node.services.bitcoind.getBlockIndex(tipHeight); + var value = encoding.encodeSummaryCacheValue(cache, tipHeight, tipBlockIndex.hash); + this.summaryCache.put(key, value, function(err) { + if (err) { + return callback(err); + } + callback(null, cache); + }); - cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; +}; - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(output.txid, 'hex'), // TODO: get buffer directly - output.outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - // Only add this to the balance if it's not spent in the mempool already - if (!spentMempool) { - cache.result.unconfirmedBalance += output.satoshis; +AddressService.prototype._getAddressMempoolSummary = function(address, options, cache, callback) { + var self = this; + + // Skip if the options do not want to include the mempool + if (!options.queryMempool) { + return callback(null, cache); + } + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + async.waterfall([ + function(next) { + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { + if (err) { + return next(err); } - } + for(var i = 0; i < mempoolInputs.length; i++) { + var input = mempoolInputs[i]; + cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; + } + next(null, cache); + }); + + }, function(cache, next) { + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { + if (err) { + return next(err); + } + for(var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; + + cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + cache.result.unconfirmedBalance += output.satoshis; + } + } + next(null, cache); + }); + } + ], callback); +}; - callback(error, cache); +AddressService.prototype._transformAddressSummaryFromCache = function(cache, options) { - }); + var result = cache.result; + var confirmedTxids = cache.result.txids; + var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); - }); + var summary = { + totalReceived: result.totalReceived, + totalSpent: result.totalReceived - result.balance, + balance: result.balance, + appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, + unconfirmedAppearances: unconfirmedTxids.length + }; + + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { + summary.txids = confirmedTxids.concat(unconfirmedTxids); + } + + return summary; }; diff --git a/package.json b/package.json index 5e0a4478..deb421cf 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,8 @@ "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", - "leveldown": "^1.4.2", - "levelup": "^1.2.1", + "leveldown": "^1.4.3", + "levelup": "^1.3.1", "liftoff": "^2.2.0", "memdown": "^1.0.0", "mkdirp": "0.5.0", diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js new file mode 100644 index 00000000..e5ba7376 --- /dev/null +++ b/test/services/address/encoding.unit.js @@ -0,0 +1,103 @@ +'use strict'; + +var chai = require('chai'); +var should = chai.should(); +var sinon = require('sinon'); +var bitcorenode = require('../../../'); +var bitcore = require('bitcore-lib'); +var Address = bitcore.Address; +var Script = bitcore.Script; +var AddressService = bitcorenode.services.Address; +var Networks = bitcore.Networks; +var encoding = require('../../../lib/services/address/encoding'); + +var mockdb = { +}; + +var mocknode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub() + } + } +}; + +describe('Address Service Encoding', function() { + + describe('#encodeSpentIndexSyncKey', function() { + it('will encode to 36 bytes (string)', function() { + var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + key.length.should.equal(36); + }); + it('will be able to decode encoded value', function() { + var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; + var txidBuffer = new Buffer(txid, 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + var keyBuffer = new Buffer(key, 'binary'); + keyBuffer.slice(0, 32).toString('hex').should.equal(txid); + var outputIndex = keyBuffer.readUInt32BE(32); + outputIndex.should.equal(12); + }); + }); + + describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { + var encoded; + var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputKeyMap(outputTxIdBuffer, 13); + }); + it('decode key', function() { + var key = encoding.decodeInputKeyMap(encoded); + key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); + key.outputIndex.should.equal(13); + }); + }); + + describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { + var encoded; + var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputValueMap(inputTxIdBuffer, 7); + }); + it('decode key', function() { + var key = encoding.decodeInputValueMap(encoded); + key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); + key.inputIndex.should.equal(7); + }); + }); + + + describe('#extractAddressInfoFromScript', function() { + it('pay-to-publickey', function() { + var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); + var script = Script.buildPublicKeyOut(pubkey); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); + }); + it('pay-to-publickeyhash', function() { + var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('pay-to-scripthash', function() { + var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToScriptHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('non-address script type', function() { + var buf = new Buffer(40); + buf.fill(0); + var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.should.equal(false); + }); + }); + +}); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 8092a2f0..4745c624 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -23,8 +23,6 @@ describe('Address Service History', function() { history.node.should.equal(node); history.options.should.equal(options); history.addresses.should.equal(addresses); - history.transactionInfo.should.deep.equal([]); - history.combinedArray.should.deep.equal([]); history.detailedArray.should.deep.equal([]); }); it('will set addresses an array if only sent a string', function() { @@ -40,27 +38,29 @@ describe('Address Service History', function() { describe('#get', function() { it('will complete the async each limit series', function(done) { var addresses = [address]; + var summary = { + txids: [] + }; var history = new AddressHistory({ - node: {}, + node: { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }, options: {}, addresses: addresses }); var expected = [{}]; history.detailedArray = expected; - history.combinedArray = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.combineTransactionInfo = sinon.stub(); - history.sortAndPaginateCombinedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArg(1); - history.sortTransactionsIntoArray = sinon.stub(); history.get(function(err, results) { if (err) { throw err; } - history.getTransactionInfo.callCount.should.equal(1); history.getDetailedInfo.callCount.should.equal(1); history.combineTransactionInfo.callCount.should.equal(1); - history.sortAndPaginateCombinedArray.callCount.should.equal(1); results.should.deep.equal({ totalCount: 1, items: expected @@ -78,149 +78,15 @@ describe('Address Service History', function() { var expected = [{}]; history.sortedArray = expected; history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.paginateSortedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); history.get(function(err) { err.message.should.equal('test'); done(); }); }); - it('handle an error from getTransactionInfo', function(done) { - var addresses = [address]; - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); }); - describe('#getTransactionInfo', function() { - it('will handle an error from getInputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, new Error('test')) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will handle an error from getOutputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, new Error('test')), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will call getOutputs and getInputs with the correct options', function() { - var startTimestamp = 1438289011844; - var endTimestamp = 1438289012412; - var expectedArgs = { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true - }; - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true - }, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.node.services.address.getOutputs.args[0][1].should.deep.equal(expectedArgs); - history.node.services.address.getInputs.args[0][1].should.deep.equal(expectedArgs); - }); - }); - it('will handle empty results from getOutputs and getInputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.transactionInfo.length.should.equal(1); - history.node.services.address.getOutputs.args[0][0].should.equal(address); - }); - }); - it('will concatenate outputs and inputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, [{}]), - getInputs: sinon.stub().callsArgWith(2, null, [{}]) - } - } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.transactionInfo.length.should.equal(3); - history.node.services.address.getOutputs.args[0][0].should.equal(address); - }); - }); - }); - - describe('@sortByHeight', function() { + describe('#_mergeAndSortTxids', function() { it('will sort latest to oldest using height', function() { var transactionInfo = [ { @@ -386,131 +252,6 @@ describe('Address Service History', function() { }); }); - describe('#sortAndPaginateCombinedArray', function() { - it('from 0 to 2', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 2 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(2); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - }); - it('from 0 to 4 (exceeds length)', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 4 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(3); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - history.combinedArray[2].height.should.equal(12); - }); - it('from 0 to 1', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 1 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(14); - }); - it('from 2 to 3', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 2, - to: 3 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(12); - }); - it('from 10 to 20 (out of range)', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 10, - to: 20 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(0); - }); - }); - describe('#getDetailedInfo', function() { it('will add additional information to existing this.transactions', function() { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; @@ -602,7 +343,7 @@ describe('Address Service History', function() { } }, options: {}, - addresses: [] + addresses: [txAddress] }); var transactionInfo = { addresses: {}, @@ -614,7 +355,7 @@ describe('Address Service History', function() { transactionInfo.addresses[txAddress] = {}; transactionInfo.addresses[txAddress].outputIndexes = [1]; transactionInfo.addresses[txAddress].inputIndexes = []; - history.getDetailedInfo(transactionInfo, function(err) { + history.getDetailedInfo(txid, function(err) { if (err) { throw err; } @@ -653,28 +394,4 @@ describe('Address Service History', function() { history.getConfirmationsDetail(transaction).should.equal(1); }); }); - describe('#getSatoshisDetail', function() { - it('subtract inputIndexes satoshis without outputIndexes', function() { - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: [] - }); - var transaction = { - inputs: [ - { - output: { - satoshis: 10000 - } - } - ] - }; - var txInfo = { - addresses: {} - }; - txInfo.addresses[address] = {}; - txInfo.addresses[address].inputIndexes = [0]; - history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); - }); - }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e7460a1d..193a9f8a 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2,6 +2,8 @@ var should = require('chai').should(); var sinon = require('sinon'); +var stream = require('stream'); +var levelup = require('levelup'); var proxyquire = require('proxyquire'); var bitcorenode = require('../../../'); var AddressService = bitcorenode.services.Address; @@ -9,13 +11,15 @@ var blockData = require('../../data/livenet-345003.json'); var bitcore = require('bitcore-lib'); var memdown = require('memdown'); var leveldown = require('leveldown'); -var Script = bitcore.Script; -var Address = bitcore.Address; var Networks = bitcore.Networks; var EventEmitter = require('events').EventEmitter; var errors = bitcorenode.errors; var Transaction = require('../../../lib/transaction'); var txData = require('../../data/transaction.json'); +var index = require('../../../lib'); +var log = index.log; +var constants = require('../../../lib/services/address/constants'); +var encoding = require('../../../lib/services/address/encoding'); var mockdb = { }; @@ -96,7 +100,8 @@ describe('Address Service', function() { done(); }); }); - it('start levelup db for mempool index', function(done) { + it('start levelup db for mempool and summary index', function(done) { + var levelupStub = sinon.stub().callsArg(2); var TestAddressService = proxyquire('../../../lib/services/address', { 'fs': { existsSync: sinon.stub().returns(true) @@ -104,14 +109,7 @@ describe('Address Service', function() { 'leveldown': { destroy: sinon.stub().callsArgWith(1, null) }, - 'levelup': function(dbPath, options, callback) { - dbPath.should.equal('testdir/testnet3/bitcore-addressmempool.db'); - options.db.should.equal(memdown); - options.keyEncoding.should.equal('binary'); - options.valueEncoding.should.equal('binary'); - options.fillCache.should.equal(false); - setImmediate(callback); - }, + 'levelup': levelupStub, 'mkdirp': sinon.stub().callsArgWith(1, null) }); var am = new TestAddressService({ @@ -119,6 +117,16 @@ describe('Address Service', function() { node: mocknode }); am.start(function() { + levelupStub.callCount.should.equal(2); + var dbPath1 = levelupStub.args[0][0]; + dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); + var options = levelupStub.args[0][1]; + options.db.should.equal(memdown); + options.keyEncoding.should.equal('binary'); + options.valueEncoding.should.equal('binary'); + options.fillCache.should.equal(false); + var dbPath2 = levelupStub.args[1][0]; + dbPath2.should.equal('testdir/testnet3/bitcore-addresssummary.db'); done(); }); }); @@ -253,7 +261,7 @@ describe('Address Service', function() { }); it('should load the db with regtest', function() { // Switch to use regtest - // Networks.remove(Networks.testnet); + Networks.remove(Networks.testnet); Networks.add({ name: 'regtest', alias: 'regtest', @@ -387,43 +395,6 @@ describe('Address Service', function() { }); }); - describe('#_extractAddressInfoFromScript', function() { - var am; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.node.network = Networks.livenet; - }); - it('pay-to-publickey', function() { - var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); - var script = Script.buildPublicKeyOut(pubkey); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); - }); - it('pay-to-publickeyhash', function() { - var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('pay-to-scripthash', function() { - var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToScriptHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('non-address script type', function() { - var buf = new Buffer(40); - buf.fill(0); - var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); - var info = am._extractAddressInfoFromScript(script); - info.should.equal(false); - }); - }); - describe('#blockHandler', function() { var am; var testBlock = bitcore.Block.fromString(blockData); @@ -524,6 +495,7 @@ describe('Address Service', function() { var testnode = { datadir: 'testdir', db: db, + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -559,73 +531,6 @@ describe('Address Service', function() { }); }); - describe('#_encodeSpentIndexSyncKey', function() { - it('will encode to 36 bytes (string)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - key.length.should.equal(36); - }); - it('will be able to decode encoded value', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var txidBuffer = new Buffer(txid, 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - var keyBuffer = new Buffer(key, 'binary'); - keyBuffer.slice(0, 32).toString('hex').should.equal(txid); - var outputIndex = keyBuffer.readUInt32BE(32); - outputIndex.should.equal(12); - }); - }); - - describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { - var encoded; - var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputKeyMap(outputTxIdBuffer, 13); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputKeyMap(encoded); - key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); - key.outputIndex.should.equal(13); - }); - }); - - describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { - var encoded; - var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputValueMap(inputTxIdBuffer, 7); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputValueMap(encoded); - key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); - key.inputIndex.should.equal(7); - }); - }); - describe('#transactionEventHandler', function() { it('will emit a transaction if there is a subscriber', function(done) { var am = new AddressService({ @@ -817,18 +722,127 @@ describe('Address Service', function() { }); + describe('#createInputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createInputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df72'); + data.inputIndex.should.equal(1); + data.height.should.equal(157); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('030b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d786cfeae288d74aaf9f51f215f9882e7bd7bc18af7a550683c4d7c6962f6372900000004', 'hex'), + value: new Buffer('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df7200000001', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createInputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getInputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -845,14 +859,18 @@ describe('Address Service', function() { }); it('will add mempool inputs on close', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var db = { store: { createReadStream: sinon.stub().returns(testStream) + }, + tip: { + __height: 10 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -883,10 +901,11 @@ describe('Address Service', function() { inputs[0].height.should.equal(-1); done(); }); - testStream.emit('close'); + testStream.push(null); }); it('will get inputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -895,12 +914,12 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -924,20 +943,21 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get inputs for address', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { queryMempool: true }; var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('00', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ff', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000000', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ffffffffff', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -960,15 +980,16 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should give an error if the readstream has an error', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(testStream) }; - am.getOutputs(address, {}, function(err, outputs) { + am.getInputs(address, {}, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); @@ -976,7 +997,7 @@ describe('Address Service', function() { testStream.emit('error', new Error('readstreamerror')); setImmediate(function() { - testStream.emit('close'); + testStream.push(null); }); }); @@ -986,7 +1007,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1025,39 +1046,44 @@ describe('Address Service', function() { }); }); it('it will parse data', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { + var nowTime = new Date().getTime(); + + am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, inputs) { should.not.exist(err); - outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].txid.should.equal(txid); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - outputs[0].inputIndex.should.equal(5); - outputs[0].height.should.equal(-1); - outputs[0].confirmations.should.equal(0); + inputs.length.should.equal(1); + var input = inputs[0]; + input.address.should.equal(address); + input.txid.should.equal(txid); + input.hashType.should.equal('pubkeyhash'); + input.hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + input.inputIndex.should.equal(5); + input.height.should.equal(-1); + input.confirmations.should.equal(0); + input.timestamp.should.equal(nowTime); done(); }); var txid = '5d32f0fff6871c377e00c16f48ebb5e89c723d0b9dd25f68fdda70c3392bee61'; var inputIndex = 5; var inputIndexBuffer = new Buffer(4); + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(nowTime); inputIndexBuffer.writeUInt32BE(inputIndex); var valueData = Buffer.concat([ new Buffer(txid, 'hex'), - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); - // Note: key is not used currently testStream.emit('data', { value: valueData }); - setImmediate(function() { - testStream.emit('close'); - }); + testStream.emit('close'); }); }); @@ -1105,18 +1131,129 @@ describe('Address Service', function() { }); }); + describe('#createOutputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createOutputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f9'); + data.outputIndex.should.equal(2); + data.height.should.equal(157); + data.satoshis.should.equal(10000); + data.script.toString('hex').should.equal('76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('020b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f900000002', 'hex'), + value: new Buffer('40c388000000000076a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createOutputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -1137,7 +1274,8 @@ describe('Address Service', function() { }); it('will get outputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1146,10 +1284,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1161,7 +1299,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); @@ -1174,11 +1312,12 @@ describe('Address Service', function() { value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get outputs for an address', function(done) { - var readStream1 = new EventEmitter(); + var readStream1 = new stream.Readable(); + readStream1._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream1) }; @@ -1234,11 +1373,12 @@ describe('Address Service', function() { readStream1.emit('data', data1); readStream1.emit('data', data2); - readStream1.emit('close'); + readStream1.push(null); }); it('should give an error if the readstream has an error', function(done) { - var readStream2 = new EventEmitter(); + var readStream2 = new stream.Readable(); + readStream2._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream2) }; @@ -1251,7 +1391,7 @@ describe('Address Service', function() { readStream2.emit('error', new Error('readstreamerror')); setImmediate(function() { - readStream2.emit('close'); + readStream2.push(null); }); }); @@ -1261,8 +1401,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1271,10 +1412,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1286,7 +1427,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('scripthash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('a914038a213afdfc551fc658e9a2a58a86e98d69b68787'); @@ -1301,7 +1442,7 @@ describe('Address Service', function() { value: new Buffer('41f0de058a800000a914038a213afdfc551fc658e9a2a58a86e98d69b68787', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should not print outputs for a p2pkh address, if the output was sent to a p2sh redeemScript', function(done) { @@ -1310,8 +1451,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1322,10 +1464,10 @@ describe('Address Service', function() { // Verifying that the db query is looking for a redeemScript, *not* a p2pkh am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1337,7 +1479,7 @@ describe('Address Service', function() { done(); }); createReadStreamCallCount.should.equal(1); - testStream.emit('close'); + testStream.push(null); }); }); @@ -1345,7 +1487,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1391,14 +1533,16 @@ describe('Address Service', function() { throw err; } outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].txid.should.equal(txid); - outputs[0].outputIndex.should.equal(outputIndex); - outputs[0].height.should.equal(-1); - outputs[0].satoshis.should.equal(3); - outputs[0].script.should.equal('ac'); - outputs[0].confirmations.should.equal(0); + var output = outputs[0]; + output.address.should.equal(address); + output.hashType.should.equal('pubkeyhash'); + output.txid.should.equal(txid); + output.outputIndex.should.equal(outputIndex); + output.height.should.equal(-1); + output.satoshis.should.equal(3); + output.script.should.equal('ac'); + output.timestamp.should.equal(1452696715750); + output.confirmations.should.equal(0); done(); }); @@ -1408,7 +1552,7 @@ describe('Address Service', function() { var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var keyData = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, txidBuffer, @@ -1417,6 +1561,7 @@ describe('Address Service', function() { var valueData = Buffer.concat([ new Buffer('4008000000000000', 'hex'), + new Buffer('427523b78c1e6000', 'hex'), new Buffer('ac', 'hex') ]); @@ -1803,12 +1948,18 @@ describe('Address Service', function() { describe('#updateMempoolIndex/#removeMempoolIndex', function() { var am; var tx = Transaction().fromBuffer(txBuf); + var clock; - before(function() { + beforeEach(function() { am = new AddressService({ mempoolMemoryIndex: true, node: mocknode }); + clock = sinon.useFakeTimers(); + }); + + afterEach(function() { + clock.restore(); }); it('will update the input and output indexes', function() { @@ -1819,12 +1970,18 @@ describe('Address Service', function() { for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('put'); } - var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002'; + var nowTime = new Date().getTime(); + var nowTimeBuffer = new Buffer(8); + nowTimeBuffer.writeDoubleBE(nowTime); + var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002' + + nowTimeBuffer.toString('hex'); operations[7].value.toString('hex').should.equal(expectedValue); var matches = 0; + + for (var j = 0; j < operations.length; j++) { var match = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, bitcore.Address('1JT7KDYwT9JY9o2vyqcKNSJgTWeKfV3ui8').hashBuffer ]).toString('hex'); @@ -1850,88 +2007,743 @@ describe('Address Service', function() { }); }); + describe('#getAddressSummary', function() { - var node = { - datadir: 'testdir', - network: Networks.testnet, - services: { - bitcoind: { - isSpent: sinon.stub().returns(false), - on: sinon.spy() - } - } - }; - var inputs = [ - { - 'txid': '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - 'height': -1, - 'confirmations': 0, - 'addresses': { - 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2': { - 'outputIndexes': [], - 'inputIndexes': [ - 3 - ] + var clock; + beforeEach(function() { + clock = sinon.useFakeTimers(); + sinon.stub(log, 'warn'); + }); + afterEach(function() { + clock.restore(); + log.warn.restore(); + }); + it('will handle error from _getAddressConfirmedSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() } }, - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2' - } - ]; - - var outputs = [ - { - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', - 'txid': '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', - 'outputIndex': 0, - 'height': 556351, - 'satoshis': 3487110, - 'script': '76a914653b58493c2208481e0902a8ffb97b8112b13fe188ac', - 'confirmations': 13190 - } - ]; - - var as = new AddressService({ - mempoolMemoryIndex: true, - node: node - }); - as.getInputs = sinon.stub().callsArgWith(2, null, inputs); - as.getOutputs = sinon.stub().callsArgWith(2, null, outputs); - var key = Buffer.concat([ - new Buffer('689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', 'hex'), - new Buffer(Array(4)) - ]).toString('binary'); - as.mempoolSpentIndex = {}; - as.mempoolSpentIndex[key] = true; - it('should handle unconfirmed and confirmed outputs and inputs', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - summary.txids.should.deep.equal( - [ - '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5' - ] - ); + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, new Error('test')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); done(); }); }); - it('noTxList should not include txids array', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {noTxList: true}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - should.not.exist(summary.txids); + it('will handle error from _getAddressMempoolSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArg(2); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(2, new Error('test2')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); + }); + }); + it('will pass cache and summary between functions correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function(err, sum) { + addressService._getAddressConfirmedSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); + addressService._transformAddressSummaryFromCache.callCount.should.equal(1); + addressService._transformAddressSummaryFromCache.args[0][0].should.equal(cache); + sum.should.equal(summary); + done(); + }); + }); + it('will log if there is a slow query', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function() { + log.warn.callCount.should.equal(2); done(); }); + clock.tick(6000); }); }); + + describe('#_getAddressConfirmedSummary', function() { + it('handle error from _getAddressConfirmedSummaryCache', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 10 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, new Error('test')); + addressService._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + + it('will NOT update cache if matches current tip', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 10 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = { + height: 10 + }; + addressService._updateAddressConfirmedSummaryCache = sinon.stub(); + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(0); + done(); + }); + }); + + it('will call _updateAddressConfirmedSummaryCache with correct arguments', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 11 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = { + height: 10 + }; + addressService._updateAddressConfirmedSummaryCache = sinon.stub().callsArgWith(4, null, cache); + addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(1); + var args = addressService._updateAddressConfirmedSummaryCache.args[0]; + args[0].should.equal(address); + args[1].should.equal(options); + args[2].should.equal(cache); + args[3].should.equal(11); + done(); + }); + }); + }); + + describe('#_getAddressConfirmedSummaryCache', function() { + function shouldExistBasecache(cache) { + should.exist(cache); + should.not.exist(cache.height); + should.exist(cache.result); + cache.result.appearanceIds.should.deep.equal({}); + cache.result.totalReceived.should.equal(0); + cache.result.balance.should.equal(0); + cache.result.unconfirmedAppearanceIds.should.deep.equal({}); + cache.result.unconfirmedBalance.should.equal(0); + } + it('give base cache if "start" or "end" options are used (e.g. >= 0)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + start: 0, + end: 0 + }; + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + it('give base cache if "start" or "end" options are used (e.g. 10, 9)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + start: 10, + end: 9 + }; + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('give base cache if cache does NOT exist', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('give base cache if cached tip hash differs (e.g. reorg)', function(done) { + var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({ + hash: '00000000700e92a916b46b8b91a14d1303d5d91ef0b09eecc3151fb958fd9a2e' + }) + }, + db: { + tip: { + hash: hash + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; + var options = {}; + var cache = { + height: 10, + hash: hash, + result: { + totalReceived: 100, + balance: 10, + txids: [txid], + appearanceIds: { + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + } + } + }; + var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + shouldExistBasecache(cache); + done(); + }); + }); + + it('handle error from levelup', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, new Error('test')); + addressService._getAddressConfirmedSummaryCache(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + + it('call encode and decode with args and result', function(done) { + var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({ + hash: hash + }) + }, + db: { + tip: { + hash: hash + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; + var options = {}; + var cache = { + height: 10, + hash: hash, + result: { + totalReceived: 100, + balance: 10, + txids: [txid], + appearanceIds: { + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + } + } + }; + var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); + addressService.summaryCache = {}; + addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); + addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { + if (err) { + return done(err); + } + should.exist(cache); + cache.height.should.equal(10); + cache.hash.should.equal(hash); + should.exist(cache.result); + cache.result.totalReceived.should.equal(100); + cache.result.balance.should.equal(10); + cache.result.txids.should.deep.equal([txid]); + cache.result.appearanceIds.should.deep.equal({ + '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 + }); + done(); + }); + }); + }); + + describe('#_updateAddressConfirmedSummaryCache', function() { + it('will pass partial options to input/output summary query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + height: 10, + result: { + txids: [] + } + }; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err, cache) { + if (err) { + return done(err); + } + as._getAddressConfirmedInputsSummary.callCount.should.equal(1); + as._getAddressConfirmedOutputsSummary.callCount.should.equal(1); + + as._getAddressConfirmedInputsSummary.args[0][2].start.should.equal(12); + as._getAddressConfirmedInputsSummary.args[0][2].end.should.equal(11); + + as._getAddressConfirmedOutputsSummary.args[0][2].start.should.equal(12); + as._getAddressConfirmedOutputsSummary.args[0][2].end.should.equal(11); + done(); + }); + }); + + it('will save cache if exceeds threshold and is NOT height query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + height: 10, + result: { + txids: [ + '9a816264c50910cbf57aa4637dde5f7fec03df642b822661e8bc9710475986b6', + '05c6b9ccf3fc4026391bf8f8a64b4784a95b930851359b8f85a4be7bb6bf6f1e' + ] + } + }; + as.summaryCacheThreshold = 1; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { + if (err) { + return done(err); + } + as._saveAddressConfirmedSummaryCache.callCount.should.equal(1); + done(); + }); + }); + + it('will NOT save cache if exceeds threshold and IS height query', function(done) { + var tipHeight = 12; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var cache = { + result: { + txids: [] + } + }; + as.summaryCacheThreshold = 1; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); + as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { + if (err) { + return done(err); + } + as._saveAddressConfirmedSummaryCache.callCount.should.equal(0); + done(); + }); + }); + + }); + + describe('#_getAddressConfirmedInputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {} + } + }; + var options = {}; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + if (err) { + return done(err); + } + cache.result.appearanceIds[txid].should.equal(10); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = {}; + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe('#_getAddressConfirmedOutputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + isSpent: sinon.stub().returns(false) + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + } + }; + var options = { + queryMempool: true + }; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + + as.createOutputsStream = sinon.stub().returns(streamStub); + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + + as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + if (err) { + return done(err); + } + cache.result.appearanceIds[txid].should.equal(10); + cache.result.balance.should.equal(1000); + cache.result.totalReceived.should.equal(1000); + cache.result.unconfirmedBalance.should.equal(-1000); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10, + outputIndex: 2, + satoshis: 1000 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = { + height: 10, + result: { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + } + }; + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createOutputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { + }); + + describe.skip('#_saveAddressConfirmedSummaryCache', function() { + }); + + describe.skip('#_getAddressMempoolSummary', function() { + }); + + describe.skip('#_transformAddressSummaryFromCache', function() { + }); + }); From ead6c2f45fc768ca7792a64c5d7456edc4591745 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 14 Jan 2016 17:07:44 -0500 Subject: [PATCH 010/299] Address Service: Removed caching and added max query limits Querying addresses that have millions of transactions is supported however takes hundreds of seconds to fully calculate the balance. Creating a cache of previous results wasn't currently working because the `isSpent` query is always based on the current bitcoind tip. Thus the balance of the outputs would be included however wouldn't be removed when spent as the output wouldn't be checked again when querying for blocks past the last checkpoint. Including the satoshis in the inputs address index would make it possible to subtract the spent amount, however this degrades optimizations elsewhere. The syncing times or querying for addresses with 10,000 transactions per address. It may preferrable to have an additional address service that handles high-volume addresses be on an opt-in basis so that a custom running client could select high volume addresses to create optimizations for querying balances and history. The strategies for creating indexes differs on these use cases. --- lib/services/address/index.js | 219 ++++-------- test/services/address/index.unit.js | 511 ++-------------------------- 2 files changed, 92 insertions(+), 638 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 22c2420f..829d8414 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,12 +44,10 @@ var AddressService = function(options) { this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); - this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this._setMempoolIndexPath(); - this._setSummaryCachePath(); if (options.mempoolMemoryIndex) { this.levelupStore = memdown; } else { @@ -98,19 +96,6 @@ AddressService.prototype.start = function(callback) { }, next ); - }, - function(next) { - self.summaryCache = levelup( - self.summaryCachePath, - { - db: self.levelupStore, - keyEncoding: 'binary', - valueEncoding: 'binary', - fillCache: false, - maxOpenFiles: 200 - }, - next - ); } ], callback); @@ -121,14 +106,6 @@ AddressService.prototype.stop = function(callback) { this.mempoolIndex.close(callback); }; -/** - * This function will set `this.summaryCachePath` based on `this.node.network`. - * @private - */ -AddressService.prototype._setSummaryCachePath = function() { - this.summaryCachePath = this._getDBPathFor('bitcore-addresssummary.db'); -}; - /** * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private @@ -1357,21 +1334,20 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb function(next) { self._getAddressConfirmedSummary(address, options, next); }, - function(cache, next) { - self._getAddressMempoolSummary(address, options, cache, next); + function(result, next) { + self._getAddressMempoolSummary(address, options, result, next); } - ], function(err, cache) { + ], function(err, result) { if (err) { return callback(err); } - var summary = self._transformAddressSummaryFromCache(cache, options); + var summary = self._transformAddressSummaryFromCache(result, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { var seconds = Math.round(timeDelta / 1000); log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); - log.warn('Address Summary:', summary); } callback(null, summary); @@ -1382,108 +1358,48 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { var self = this; - var tipHeight = this.node.services.db.tip.__height; - - self._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return callback(err); - } - // Immediately give cache is already current, otherwise update - if (cache && cache.height === tipHeight) { - return callback(null, cache); - } - self._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, callback); - }); -}; - -AddressService.prototype._getAddressConfirmedSummaryCache = function(address, options, callback) { - var self = this; - var baseCache = { - result: { - appearanceIds: {}, - totalReceived: 0, - balance: 0, - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - } + var baseResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 }; - // Use the base cache if the "start" and "end" options have been used - // We only save and retrieve a cache for the summary of all history - if (options.start >= 0 || options.end >= 0) { - return callback(null, baseCache); - } - var key = encoding.encodeSummaryCacheKey(address); - this.summaryCache.get(key, { - valueEncoding: 'binary', - keyEncoding: 'binary' - }, function(err, buffer) { - if (err instanceof levelup.errors.NotFoundError) { - return callback(null, baseCache); - } else if (err) { - return callback(err); - } - var cache = encoding.decodeSummaryCacheValue(buffer); - - // Use base cache if the cached tip/height doesn't match (e.g. there has been a reorg) - var blockIndex = self.node.services.bitcoind.getBlockIndex(cache.height); - if (cache.hash !== blockIndex.hash) { - return callback(null, baseCache); - } - - callback(null, cache); - }); -}; - -AddressService.prototype._updateAddressConfirmedSummaryCache = function(address, options, cache, tipHeight, callback) { - var self = this; - - var optionsPartial = _.clone(options); - var isHeightQuery = (options.start >= 0 || options.end >= 0); - if (!isHeightQuery) { - // We will pick up from the last point cached and query for all blocks - // proceeding the cache - var cacheHeight = _.isUndefined(cache.height) ? 0 : cache.height + 1; - optionsPartial.start = tipHeight; - optionsPartial.end = cacheHeight; - } else { - $.checkState(_.isUndefined(cache.height)); - } async.waterfall([ function(next) { - self._getAddressConfirmedInputsSummary(address, cache, optionsPartial, next); + self._getAddressConfirmedInputsSummary(address, baseResult, options, next); }, - function(cache, next) { - self._getAddressConfirmedOutputsSummary(address, cache, optionsPartial, next); + function(result, next) { + self._getAddressConfirmedOutputsSummary(address, result, options, next); }, - function(cache, next) { - self._setAndSortTxidsFromAppearanceIds(cache, next); - } - ], function(err, cache) { - - // Skip saving the cache if the "start" or "end" options have been used, or - // if the transaction length does not exceed the caching threshold. - // We only want to cache full history results for addresses that have a large - // number of transactions. - var exceedsCacheThreshold = (cache.result.txids.length > self.summaryCacheThreshold); - if (exceedsCacheThreshold && !isHeightQuery) { - self._saveAddressConfirmedSummaryCache(address, cache, tipHeight, callback); - } else { - callback(null, cache); + function(result, next) { + self._setAndSortTxidsFromAppearanceIds(result, next); } + ], callback); - }); }; -AddressService.prototype._getAddressConfirmedInputsSummary = function(address, cache, options, callback) { +AddressService.prototype._getAddressConfirmedInputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); var self = this; var error = null; + var count = 0; var inputsStream = self.createInputsStream(address, options); inputsStream.on('data', function(input) { var txid = input.txid; - cache.result.appearanceIds[txid] = input.height; + result.appearanceIds[txid] = input.height; + + count++; + + if (count > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + inputsStream.pause(); + inputsStream.end(); + } + }); inputsStream.on('error', function(err) { @@ -1494,17 +1410,18 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, c if (error) { return callback(error); } - callback(null, cache); + callback(null, result); }); }; -AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, cache, options, callback) { +AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(cache.result) && - !_.isUndefined(cache.result.appearanceIds) && - !_.isUndefined(cache.result.unconfirmedAppearanceIds)); + $.checkArgument(!_.isUndefined(result) && + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); var self = this; + var count = 0; var outputStream = self.createOutputsStream(address, options); @@ -1515,13 +1432,12 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - cache.result.totalReceived += output.satoshis; - cache.result.appearanceIds[txid] = output.height; + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = output.height; if (!spentDB) { - cache.result.balance += output.satoshis; + result.balance += output.satoshis; } - // TODO: subtract if spent (because of cache)? if (options.queryMempool) { // Check to see if this output is spent in the mempool and if so @@ -1532,10 +1448,19 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; if (spentMempool) { - cache.result.unconfirmedBalance -= output.satoshis; + result.unconfirmedBalance -= output.satoshis; } } + count++; + + if (count > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + outputStream.pause(); + outputStream.end(); + } + }); var error = null; @@ -1548,40 +1473,25 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, if (error) { return callback(error); } - callback(null, cache); + callback(null, result); }); }; -AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(cache, callback) { - cache.result.txids = Object.keys(cache.result.appearanceIds); - cache.result.txids.sort(function(a, b) { - return cache.result.appearanceIds[a] - cache.result.appearanceIds[b]; +AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, callback) { + result.txids = Object.keys(result.appearanceIds); + result.txids.sort(function(a, b) { + return result.appearanceIds[a] - result.appearanceIds[b]; }); - callback(null, cache); -}; - -AddressService.prototype._saveAddressConfirmedSummaryCache = function(address, cache, tipHeight, callback) { - - log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); - var key = encoding.encodeSummaryCacheKey(address); - var tipBlockIndex = this.node.services.bitcoind.getBlockIndex(tipHeight); - var value = encoding.encodeSummaryCacheValue(cache, tipHeight, tipBlockIndex.hash); - this.summaryCache.put(key, value, function(err) { - if (err) { - return callback(err); - } - callback(null, cache); - }); - + callback(null, result); }; -AddressService.prototype._getAddressMempoolSummary = function(address, options, cache, callback) { +AddressService.prototype._getAddressMempoolSummary = function(address, options, result, callback) { var self = this; // Skip if the options do not want to include the mempool if (!options.queryMempool) { - return callback(null, cache); + return callback(null, result); } var addressStr = address.toString(); @@ -1596,12 +1506,12 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; + result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } - next(null, cache); + next(null, result); }); - }, function(cache, next) { + }, function(result, next) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return next(err); @@ -1609,7 +1519,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + result.unconfirmedAppearanceIds[output.txid] = output.timestamp; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly @@ -1618,19 +1528,18 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already if (!spentMempool) { - cache.result.unconfirmedBalance += output.satoshis; + result.unconfirmedBalance += output.satoshis; } } - next(null, cache); + next(null, result); }); } ], callback); }; -AddressService.prototype._transformAddressSummaryFromCache = function(cache, options) { +AddressService.prototype._transformAddressSummaryFromCache = function(result, options) { - var result = cache.result; - var confirmedTxids = cache.result.txids; + var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 193a9f8a..a50f121b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -100,7 +100,7 @@ describe('Address Service', function() { done(); }); }); - it('start levelup db for mempool and summary index', function(done) { + it('start levelup db for mempool', function(done) { var levelupStub = sinon.stub().callsArg(2); var TestAddressService = proxyquire('../../../lib/services/address', { 'fs': { @@ -117,7 +117,7 @@ describe('Address Service', function() { node: mocknode }); am.start(function() { - levelupStub.callCount.should.equal(2); + levelupStub.callCount.should.equal(1); var dbPath1 = levelupStub.args[0][0]; dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); var options = levelupStub.args[0][1]; @@ -125,8 +125,6 @@ describe('Address Service', function() { options.keyEncoding.should.equal('binary'); options.valueEncoding.should.equal('binary'); options.fillCache.should.equal(false); - var dbPath2 = levelupStub.args[1][0]; - dbPath2.should.equal('testdir/testnet3/bitcore-addresssummary.db'); done(); }); }); @@ -2115,457 +2113,14 @@ describe('Address Service', function() { addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { - log.warn.callCount.should.equal(2); + log.warn.callCount.should.equal(1); done(); }); clock.tick(6000); }); }); - describe('#_getAddressConfirmedSummary', function() { - it('handle error from _getAddressConfirmedSummaryCache', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 10 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, new Error('test')); - addressService._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('will NOT update cache if matches current tip', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 10 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = { - height: 10 - }; - addressService._updateAddressConfirmedSummaryCache = sinon.stub(); - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressConfirmedSummary(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(0); - done(); - }); - }); - - it('will call _updateAddressConfirmedSummaryCache with correct arguments', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 11 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = { - height: 10 - }; - addressService._updateAddressConfirmedSummaryCache = sinon.stub().callsArgWith(4, null, cache); - addressService._getAddressConfirmedSummaryCache = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressConfirmedSummary(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - addressService._updateAddressConfirmedSummaryCache.callCount.should.equal(1); - var args = addressService._updateAddressConfirmedSummaryCache.args[0]; - args[0].should.equal(address); - args[1].should.equal(options); - args[2].should.equal(cache); - args[3].should.equal(11); - done(); - }); - }); - }); - - describe('#_getAddressConfirmedSummaryCache', function() { - function shouldExistBasecache(cache) { - should.exist(cache); - should.not.exist(cache.height); - should.exist(cache.result); - cache.result.appearanceIds.should.deep.equal({}); - cache.result.totalReceived.should.equal(0); - cache.result.balance.should.equal(0); - cache.result.unconfirmedAppearanceIds.should.deep.equal({}); - cache.result.unconfirmedBalance.should.equal(0); - } - it('give base cache if "start" or "end" options are used (e.g. >= 0)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = { - start: 0, - end: 0 - }; - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - it('give base cache if "start" or "end" options are used (e.g. 10, 9)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = { - start: 10, - end: 9 - }; - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('give base cache if cache does NOT exist', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('give base cache if cached tip hash differs (e.g. reorg)', function(done) { - var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({ - hash: '00000000700e92a916b46b8b91a14d1303d5d91ef0b09eecc3151fb958fd9a2e' - }) - }, - db: { - tip: { - hash: hash - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; - var options = {}; - var cache = { - height: 10, - hash: hash, - result: { - totalReceived: 100, - balance: 10, - txids: [txid], - appearanceIds: { - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - } - } - }; - var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - shouldExistBasecache(cache); - done(); - }); - }); - - it('handle error from levelup', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, new Error('test')); - addressService._getAddressConfirmedSummaryCache(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('call encode and decode with args and result', function(done) { - var hash = '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9'; - var testnode = { - services: { - bitcoind: { - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({ - hash: hash - }) - }, - db: { - tip: { - hash: hash - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var txid = '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e'; - var options = {}; - var cache = { - height: 10, - hash: hash, - result: { - totalReceived: 100, - balance: 10, - txids: [txid], - appearanceIds: { - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - } - } - }; - var cacheBuffer = encoding.encodeSummaryCacheValue(cache, 10, hash); - addressService.summaryCache = {}; - addressService.summaryCache.get = sinon.stub().callsArgWith(2, null, cacheBuffer); - addressService._getAddressConfirmedSummaryCache(address, options, function(err, cache) { - if (err) { - return done(err); - } - should.exist(cache); - cache.height.should.equal(10); - cache.hash.should.equal(hash); - should.exist(cache.result); - cache.result.totalReceived.should.equal(100); - cache.result.balance.should.equal(10); - cache.result.txids.should.deep.equal([txid]); - cache.result.appearanceIds.should.deep.equal({ - '5464b1c3f25160f0183fad68a406838d2d1ac0aee05990072ece49326c26c22e': 9 - }); - done(); - }); - }); - }); - - describe('#_updateAddressConfirmedSummaryCache', function() { - it('will pass partial options to input/output summary query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - height: 10, - result: { - txids: [] - } - }; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err, cache) { - if (err) { - return done(err); - } - as._getAddressConfirmedInputsSummary.callCount.should.equal(1); - as._getAddressConfirmedOutputsSummary.callCount.should.equal(1); - - as._getAddressConfirmedInputsSummary.args[0][2].start.should.equal(12); - as._getAddressConfirmedInputsSummary.args[0][2].end.should.equal(11); - - as._getAddressConfirmedOutputsSummary.args[0][2].start.should.equal(12); - as._getAddressConfirmedOutputsSummary.args[0][2].end.should.equal(11); - done(); - }); - }); - - it('will save cache if exceeds threshold and is NOT height query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - height: 10, - result: { - txids: [ - '9a816264c50910cbf57aa4637dde5f7fec03df642b822661e8bc9710475986b6', - '05c6b9ccf3fc4026391bf8f8a64b4784a95b930851359b8f85a4be7bb6bf6f1e' - ] - } - }; - as.summaryCacheThreshold = 1; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { - if (err) { - return done(err); - } - as._saveAddressConfirmedSummaryCache.callCount.should.equal(1); - done(); - }); - }); - - it('will NOT save cache if exceeds threshold and IS height query', function(done) { - var tipHeight = 12; - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var cache = { - result: { - txids: [] - } - }; - as.summaryCacheThreshold = 1; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, cache); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - as._saveAddressConfirmedSummaryCache = sinon.stub().callsArg(3, null, cache); - as._updateAddressConfirmedSummaryCache(address, options, cache, tipHeight, function(err) { - if (err) { - return done(err); - } - as._saveAddressConfirmedSummaryCache.callCount.should.equal(0); - done(); - }); - }); - + describe.skip('#_getAddressConfirmedSummary', function() { }); describe('#_getAddressConfirmedInputsSummary', function() { @@ -2584,21 +2139,18 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {} - } + var result = { + appearanceIds: {} }; var options = {}; var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); as.createInputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedInputsSummary(address, result, options, function(err, result) { if (err) { return done(err); } - cache.result.appearanceIds[txid].should.equal(10); + result.appearanceIds[txid].should.equal(10); done(); }); @@ -2655,16 +2207,14 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - } + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 }; + var options = { queryMempool: true }; @@ -2676,14 +2226,14 @@ describe('Address Service', function() { var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); as.mempoolSpentIndex[spentIndexSyncKey] = true; - as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { if (err) { return done(err); } - cache.result.appearanceIds[txid].should.equal(10); - cache.result.balance.should.equal(1000); - cache.result.totalReceived.should.equal(1000); - cache.result.unconfirmedBalance.should.equal(-1000); + result.appearanceIds[txid].should.equal(10); + result.balance.should.equal(1000); + result.totalReceived.should.equal(1000); + result.unconfirmedBalance.should.equal(-1000); done(); }); @@ -2710,20 +2260,18 @@ describe('Address Service', function() { mempoolMemoryIndex: true, node: testnode }); - var cache = { - height: 10, - result: { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - } + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 }; + var options = {}; var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); as.createOutputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedOutputsSummary(address, cache, options, function(err, cache) { + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { should.exist(err); err.message.should.equal('test'); done(); @@ -2737,9 +2285,6 @@ describe('Address Service', function() { describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { }); - describe.skip('#_saveAddressConfirmedSummaryCache', function() { - }); - describe.skip('#_getAddressMempoolSummary', function() { }); From e79c00db105ab8dd77308e28393be3453d73543a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Jan 2016 12:42:10 -0500 Subject: [PATCH 011/299] Address Service: Updated tests and fixed various bugs --- lib/services/address/history.js | 81 ++++--- lib/services/address/index.js | 13 +- test/services/address/history.unit.js | 259 ++++---------------- test/services/address/index.unit.js | 327 +++++++++++++++++++++++++- 4 files changed, 422 insertions(+), 258 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index eb689828..7c0cfc00 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -50,7 +50,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { delete summary.appearanceIds[key]; } for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { - unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key]; + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey]; delete summary.unconfirmedAppearanceIds[key]; } } @@ -74,8 +74,6 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { */ AddressHistory.prototype.get = function(callback) { var self = this; - var totalCount; - if (this.addresses.length > this.maxAddressesQuery) { return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } @@ -86,7 +84,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - return finish(summary.txids); + return self.finish.call(self, summary.txids, callback); }); } else { var opts = _.clone(this.options); @@ -101,52 +99,53 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } var txids = self._mergeAndSortTxids(summaries); - return finish(txids); + return self.finish.call(self, txids, callback); } ); } - function finish(allTxids) { - totalCount = allTxids.length; +}; - // Slice the page starting with the most recent - var txids; - if (self.options.from >= 0 && self.options.to >= 0) { - var fromOffset = totalCount - self.options.from; - var toOffset = totalCount - self.options.to; - txids = allTxids.slice(toOffset, fromOffset); - } else { - txids = allTxids; - } +AddressHistory.prototype.finish = function(allTxids, callback) { + var self = this; + var totalCount = allTxids.length; - // Verify that this query isn't too long - if (txids.length > self.maxHistoryQueryLength) { - return callback(new Error( - 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - self.address.join(',') - )); - } + // Slice the page starting with the most recent + var txids; + if (self.options.from >= 0 && self.options.to >= 0) { + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + txids = allTxids.slice(toOffset, fromOffset); + } else { + txids = allTxids; + } - // Reverse to include most recent at the top - txids.reverse(); + // Verify that this query isn't too long + if (txids.length > self.maxHistoryQueryLength) { + return callback(new Error( + 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + + self.address.join(',') + )); + } - async.eachSeries( - txids, - function(txid, next) { - self.getDetailedInfo(txid, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); + // Reverse to include most recent at the top + txids.reverse(); - } + async.eachSeries( + txids, + function(txid, next) { + self.getDetailedInfo(txid, next); + }, + function(err) { + if (err) { + return callback(err); + } + callback(null, { + totalCount: totalCount, + items: self.detailedArray + }); + } + ); }; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 829d8414..5dfebf57 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -760,6 +760,11 @@ AddressService.prototype.createInputsStream = function(addressStr, options) { inputStream.end(); }).pipe(inputStream); + + inputStream.on('end', function() { + stream.end(); + }); + return stream; }; @@ -976,6 +981,10 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { }) .pipe(outputStream); + outputStream.on('end', function() { + stream.end(); + }); + return stream; }; @@ -1342,7 +1351,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb return callback(err); } - var summary = self._transformAddressSummaryFromCache(result, options); + var summary = self._transformAddressSummaryFromResult(result, options); var timeDelta = new Date() - startTime; if (timeDelta > 5000) { @@ -1537,7 +1546,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, ], callback); }; -AddressService.prototype._transformAddressSummaryFromCache = function(result, options) { +AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { var confirmedTxids = result.txids; var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 4745c624..37c3d697 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -35,220 +35,65 @@ describe('Address Service History', function() { }); }); - describe('#get', function() { - it('will complete the async each limit series', function(done) { - var addresses = [address]; - var summary = { - txids: [] - }; - var history = new AddressHistory({ - node: { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, null, summary) - } + describe('#_mergeAndSortTxids', function() { + it('will merge and sort multiple summaries', function() { + var summaries = [ + { + totalReceived: 10000000, + totalSpent: 0, + balance: 10000000, + appearances: 2, + unconfirmedBalance: 20000000, + unconfirmedAppearances: 2, + appearanceIds: { + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579': 154, + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce': 120 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964 } }, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.detailedArray = expected; - history.getDetailedInfo = sinon.stub().callsArg(1); - history.get(function(err, results) { - if (err) { - throw err; + { + totalReceived: 59990000, + totalSpent: 0, + balance: 49990000, + appearances: 3, + unconfirmedBalance: 1000000, + unconfirmedAppearances: 3, + appearanceIds: { + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2': 156, + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47': 152, + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f': 151 + }, + unconfirmedAppearanceIds: { + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 + } } - history.getDetailedInfo.callCount.should.equal(1); - history.combineTransactionInfo.callCount.should.equal(1); - results.should.deep.equal({ - totalCount: 1, - items: expected - }); - done(); - }); - }); - it('handle an error from getDetailedInfo', function(done) { + ]; + var node = {}; + var options = {}; var addresses = [address]; var history = new AddressHistory({ - node: {}, - options: {}, + node: node, + options: options, addresses: addresses }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); - }); - - describe('#_mergeAndSortTxids', function() { - it('will sort latest to oldest using height', function() { - var transactionInfo = [ - { - height: 276328 - }, - { - height: 273845, - }, - { - height: 555655 - }, - { - height: 325496 - }, - { - height: 329186 - }, - { - height: 534195 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(555655); - transactionInfo[1].height.should.equal(534195); - transactionInfo[2].height.should.equal(329186); - transactionInfo[3].height.should.equal(325496); - transactionInfo[4].height.should.equal(276328); - transactionInfo[5].height.should.equal(273845); - }); - it('mempool and tip with time in the future', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050424429, - height: -1 - }, - { - timestamp: 1442050425439, - height: 15 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[0].timestamp.should.equal(1442050424429); - transactionInfo[1].height.should.equal(-1); - transactionInfo[1].timestamp.should.equal(1442050424328); - transactionInfo[2].height.should.equal(15); - transactionInfo[3].height.should.equal(14); - }); - it('tip with time in the future and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[1].height.should.equal(14); - }); - it('many transactions in the mempool', function() { - var transactionInfo = [ - { - timestamp: 1442259670462, - height: -1 - }, - { - timestamp: 1442259785114, - height: -1 - }, - { - timestamp: 1442259759896, - height: -1 - }, - { - timestamp: 1442259692601, - height: -1 - }, - { - timestamp: 1442259692601, - height: 100 - }, - { - timestamp: 1442259749463, - height: -1 - }, - { - timestamp: 1442259737719, - height: -1 - }, - { - timestamp: 1442259773138, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442259785114); - transactionInfo[1].timestamp.should.equal(1442259773138); - transactionInfo[2].timestamp.should.equal(1442259759896); - transactionInfo[3].timestamp.should.equal(1442259749463); - transactionInfo[4].timestamp.should.equal(1442259737719); - transactionInfo[5].timestamp.should.equal(1442259692601); - transactionInfo[6].timestamp.should.equal(1442259670462); - transactionInfo[7].height.should.equal(100); - }); - it('mempool and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050425439, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442050425439); - transactionInfo[1].timestamp.should.equal(1442050424328); - }); - it('mempool and mempool with the same timestamp', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: -1, - txid: '1', - }, - { - timestamp: 1442050425439, - height: -1, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); - }); - it('matching block heights', function() { - var transactionInfo = [ - { - height: 325496, - txid: '1', - }, - { - height: 325496, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); + var txids = history._mergeAndSortTxids(summaries); + txids.should.deep.equal([ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47', + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579', + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a', + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693', + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681' + ]); }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index a50f121b..c0635e4b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2080,13 +2080,13 @@ describe('Address Service', function() { var summary = {}; addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function(err, sum) { addressService._getAddressConfirmedSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); - addressService._transformAddressSummaryFromCache.callCount.should.equal(1); - addressService._transformAddressSummaryFromCache.args[0][0].should.equal(cache); + addressService._transformAddressSummaryFromResult.callCount.should.equal(1); + addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); sum.should.equal(summary); done(); }); @@ -2111,7 +2111,7 @@ describe('Address Service', function() { addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._transformAddressSummaryFromCache = sinon.stub().returns(summary); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { log.warn.callCount.should.equal(1); done(); @@ -2120,7 +2120,119 @@ describe('Address Service', function() { }); }); - describe.skip('#_getAddressConfirmedSummary', function() { + describe('#_getAddressConfirmedSummary', function() { + it('will pass arguments correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, result); + as._getAddressConfirmedSummary(address, options, function(err) { + if (err) { + return done(err); + } + var expectedResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + as._getAddressConfirmedInputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedInputsSummary.args[0][1].should.deep.equal(expectedResult); + as._getAddressConfirmedInputsSummary.args[0][2].should.deep.equal(options); + as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); + as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); + as._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(result); + done(); + }); + }); + it('will pass error correctly (inputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will pass error correctly (outputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will pass error correctly (sort)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); + as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); }); describe('#_getAddressConfirmedInputsSummary', function() { @@ -2282,13 +2394,212 @@ describe('Address Service', function() { }); }); - describe.skip('#_setAndSortTxidsFromAppearanceIds', function() { + describe('#_setAndSortTxidsFromAppearanceIds', function() { + it('will sort correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: { + '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, + '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, + '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 + } + }; + as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { + if (err) { + return done(err); + } + should.exist(result.txids); + result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); + result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); + result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); + done(); + }); + }); }); - describe.skip('#_getAddressMempoolSummary', function() { + describe('#_getAddressMempoolSummary', function() { + it('skip if options not enabled', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + Object.keys(result.unconfirmedAppearanceIds).length.should.equal(0); + result.unconfirmedBalance.should.equal(0); + done(); + }); + }); + it('include all txids and balance from inputs and outputs', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + queryMempool: true + }; + var mempoolInputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + inputIndex: 0, + timestamp: 1452874536321, + height: -1, + confirmations: 0 + } + ]; + var mempoolOutputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + }, + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + } + ]; + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(mempoolOutputs[1].txid, 'hex'), + 0 + ); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); + as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + var txid1 = '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8'; + var txid2 = '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d'; + var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; + result.unconfirmedAppearanceIds[txid1].should.equal(1452874536321); + result.unconfirmedAppearanceIds[txid2].should.equal(1452874521466); + result.unconfirmedAppearanceIds[txid3].should.equal(1452874521466); + result.unconfirmedBalance.should.equal(131368318); + done(); + }); + }); }); - describe.skip('#_transformAddressSummaryFromCache', function() { + describe('#_transformAddressSummaryFromResult', function() { + var result = { + totalReceived: 1000000, + balance: 500000, + txids: [ + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92' + ], + appearanceIds: { + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92': 100000, + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8': 200000 + }, + unconfirmedAppearanceIds: { + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 + }, + unconfirmedBalance: 500000 + }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + it('will transform result into summary', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var summary = as._transformAddressSummaryFromResult(result, options); + summary.totalReceived.should.equal(1000000); + summary.totalSpent.should.equal(500000); + summary.balance.should.equal(500000); + summary.appearances.should.equal(2); + summary.unconfirmedAppearances.should.equal(2); + summary.unconfirmedBalance.should.equal(500000); + summary.txids.length.should.equal(4); + }); + it('will omit txlist', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + noTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.not.exist(summary.txids); + }); + it('will include full appearance ids', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + fullTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.exist(summary.appearanceIds); + should.exist(summary.unconfirmedAppearanceIds); + }); }); }); From 3d9b6d5532b814bffa9d1a52bd992ca60a8630c0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 12:59:49 -0500 Subject: [PATCH 012/299] Address Service: More tests for history --- lib/services/address/history.js | 12 +- lib/services/address/index.js | 2 + test/services/address/history.unit.js | 253 +++++++++++++++++++++++++- test/services/address/index.unit.js | 2 + 4 files changed, 256 insertions(+), 13 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 7c0cfc00..1d707279 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -75,7 +75,7 @@ AddressHistory.prototype._mergeAndSortTxids = function(summaries) { AddressHistory.prototype.get = function(callback) { var self = this; if (this.addresses.length > this.maxAddressesQuery) { - return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); + return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } if (this.addresses.length === 1) { @@ -84,7 +84,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - return self.finish.call(self, summary.txids, callback); + return self._paginateWithDetails.call(self, summary.txids, callback); }); } else { var opts = _.clone(this.options); @@ -99,14 +99,14 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } var txids = self._mergeAndSortTxids(summaries); - return self.finish.call(self, txids, callback); + return self._paginateWithDetails.call(self, txids, callback); } ); } }; -AddressHistory.prototype.finish = function(allTxids, callback) { +AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { var self = this; var totalCount = allTxids.length; @@ -123,8 +123,8 @@ AddressHistory.prototype.finish = function(allTxids, callback) { // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { return callback(new Error( - 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - self.address.join(',') + 'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' + + self.addresses.join(',') )); } diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 5dfebf57..9b93ad90 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1507,6 +1507,8 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + // TODO: Sort mempool by timestamp? + async.waterfall([ function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 37c3d697..868c0a70 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -35,6 +35,222 @@ describe('Address Service History', function() { }); }); + describe('#get', function() { + it('will give an error if length of addresses is too long', function(done) { + var node = {}; + var options = {}; + var addresses = []; + for (var i = 0; i < 101; i++) { + addresses.push(address); + } + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.match(/Maximum/); + done(); + }); + }); + it('give error from getAddressSummary with one address', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test')) + } + } + }; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('give error from getAddressSummary with multiple addresses', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test2')) + } + } + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); + }); + }); + it('will query get address summary directly with one address', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub(); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(1); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.equal(options); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(0); + done(); + }); + }); + it('will merge multiple summaries with multiple addresses', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub().returns(txids); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(2); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ + fullTxList: true + }); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(1); + done(); + }); + }); + }); + + describe('#_paginateWithDetails', function() { + it('slice txids based on "from" and "to" (3 to 30)', function() { + var node = {}; + var options = { + from: 3, + to: 30 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([7, 6, 5, 4, 3, 2, 1, 0]); + }); + }); + it('slice txids based on "from" and "to" (0 to 3)', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8]); + }); + }); + it('will given an error if the full details is too long', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history.maxHistoryQueryLength = 1; + history._paginateWithDetails(txids, function(err) { + should.exist(err); + err.message.match(/Maximum/); + }); + }); + it('will give full result without pagination options', function() { + var node = {}; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]); + }); + }); + }); + describe('#_mergeAndSortTxids', function() { it('will merge and sort multiple summaries', function() { var summaries = [ @@ -98,27 +314,42 @@ describe('Address Service History', function() { }); describe('#getDetailedInfo', function() { - it('will add additional information to existing this.transactions', function() { + it('will add additional information to existing this.transactions', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var tx = { + populateInputs: sinon.stub().callsArg(2), + __height: 20, + __timestamp: 1453134151, + isCoinbase: sinon.stub().returns(false), + getFee: sinon.stub().returns(1000) + }; var history = new AddressHistory({ node: { services: { db: { - getTransactionWithBlockInfo: sinon.stub() + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, tx), + tip: { + __height: 300 + } } } }, options: {}, addresses: [] }); + history.getAddressDetailsForTransaction = sinon.stub().returns({ + addresses: {}, + satoshis: 1000, + }); history.getDetailedInfo(txid, function(err) { if (err) { throw err; } - history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); + history.node.services.db.getTransactionWithBlockInfo.callCount.should.equal(1); + done(); }); }); - it('will handle error from getTransactionFromBlock', function() { + it('will handle error from getTransactionFromBlock', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -133,9 +364,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will handle error from populateInputs', function() { + it('will handle error from populateInputs', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -152,9 +384,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will set this.transactions with correct information', function() { + it('will set this.transactions with correct information', function(done) { // block #314159 // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e // outputIndex 1 @@ -215,11 +448,16 @@ describe('Address Service History', function() { info.timestamp.should.equal(1407292005); info.fees.should.equal(20000); info.tx.should.equal(transaction); + done(); }); }); }); + + describe.skip('#getAddressDetailsForTransaction', function() { + }); + describe('#getConfirmationsDetail', function() { - it('the correct confirmations when included in the tip', function() { + it('the correct confirmations when included in the tip', function(done) { var history = new AddressHistory({ node: { services: { @@ -237,6 +475,7 @@ describe('Address Service History', function() { __height: 100 }; history.getConfirmationsDetail(transaction).should.equal(1); + done(); }); }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index c0635e4b..e855b38b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2534,6 +2534,8 @@ describe('Address Service', function() { done(); }); }); + it.skip('will sort txids by timestamp', function(done) { + }); }); describe('#_transformAddressSummaryFromResult', function() { From 687400eab2ae10184801cf428c4605867c9ea412 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 13:53:32 -0500 Subject: [PATCH 013/299] Address Service: Added test for history `getAddressDetailsForTransaction` --- test/services/address/history.unit.js | 61 ++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 868c0a70..5f6266c0 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -2,6 +2,7 @@ var should = require('chai').should(); var sinon = require('sinon'); +var bitcore = require('bitcore-lib'); var Transaction = require('../../../lib/transaction'); var AddressHistory = require('../../../lib/services/address/history'); @@ -453,7 +454,65 @@ describe('Address Service History', function() { }); }); - describe.skip('#getAddressDetailsForTransaction', function() { + describe('#getAddressDetailsForTransaction', function() { + it('will calculate details for the transaction', function(done) { + /* jshint sub:true */ + var tx = bitcore.Transaction({ + 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', + 'version': 1, + 'inputs': [ + { + 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', + 'outputIndex': 0, + 'sequenceNumber': 4294967295, + 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'output': { + 'satoshis': 1000000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + } + ], + 'outputs': [ + { + 'satoshis': 100000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 200000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 50000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 300000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 349990000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + ], + 'nLockTime': 0 + }); + var history = new AddressHistory({ + node: { + network: bitcore.Networks.testnet + }, + options: {}, + addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] + }); + var details = history.getAddressDetailsForTransaction(tx); + should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([ + 0, 1, 2, 3, 4 + ]); + details.satoshis.should.equal(-10000); + done(); + }); }); describe('#getConfirmationsDetail', function() { From 62934b4b667f3118c91a5f596cea89ab14f8e8df Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 14:54:58 -0500 Subject: [PATCH 014/299] Address Service: Removed event listeners prior to stopping --- lib/services/address/index.js | 22 ++++++++++------------ test/services/address/index.unit.js | 14 +++++++++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 9b93ad90..98617419 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -41,8 +41,10 @@ var AddressService = function(options) { this.subscriptions['address/transaction'] = {}; this.subscriptions['address/balance'] = {}; - this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); - this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this._bitcoindTransactionListener = this.transactionHandler.bind(this); + this._bitcoindTransactionLeaveListener = this.transactionLeaveHandler.bind(this); + this.node.services.bitcoind.on('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.on('txleave', this._bitcoindTransactionLeaveListener); this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; @@ -103,6 +105,8 @@ AddressService.prototype.start = function(callback) { AddressService.prototype.stop = function(callback) { // TODO Keep track of ongoing db requests before shutting down + this.node.services.bitcoind.removeListener('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.removeListener('txleave', this._bitcoindTransactionLeaveListener); this.mempoolIndex.close(callback); }; @@ -227,6 +231,10 @@ AddressService.prototype.transactionLeaveHandler = function(txInfo) { AddressService.prototype.transactionHandler = function(txInfo, callback) { var self = this; + if (this.node.stopping) { + return callback(); + } + // Basic transaction format is handled by the daemon // and we can safely assume the buffer is properly formatted. var tx = bitcore.Transaction().fromBuffer(txInfo.buffer); @@ -760,11 +768,6 @@ AddressService.prototype.createInputsStream = function(addressStr, options) { inputStream.end(); }).pipe(inputStream); - - inputStream.on('end', function() { - stream.end(); - }); - return stream; }; @@ -967,7 +970,6 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca }; AddressService.prototype.createOutputsStream = function(addressStr, options) { - var outputStream = new OutputsTransformStream({ address: new Address(addressStr, this.node.network), tipHeight: this.node.services.db.tip.__height @@ -981,10 +983,6 @@ AddressService.prototype.createOutputsStream = function(addressStr, options) { }) .pipe(outputStream); - outputStream.on('end', function() { - stream.end(); - }); - return stream; }; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e855b38b..be0767eb 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -192,14 +192,26 @@ describe('Address Service', function() { describe('#stop', function() { it('will close mempool levelup', function(done) { + var testnode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub(), + removeListener: sinon.stub() + } + } + }; var am = new AddressService({ mempoolMemoryIndex: true, - node: mocknode + node: testnode }); am.mempoolIndex = {}; am.mempoolIndex.close = sinon.stub().callsArg(0); am.stop(function() { am.mempoolIndex.close.callCount.should.equal(1); + am.node.services.bitcoind.removeListener.callCount.should.equal(2); done(); }); }); From a166b6af2320d452a89f0a1cca56500ba2ab1133 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 15:06:18 -0500 Subject: [PATCH 015/299] Address Service: Removed nolonger used constant for cache --- lib/services/address/constants.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 7cc8ef58..2cf334e9 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -41,10 +41,6 @@ exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); -// The total number of transactions that an address can receive before it will start -// to cache the summary to disk. -exports.SUMMARY_CACHE_THRESHOLD = 10000; - // The maximum number of inputs that can be queried at once exports.MAX_INPUTS_QUERY_LENGTH = 50000; // The maximum number of outputs that can be queried at once From d4f2df5c51b0023949f45613ae55ecac8032b344 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 15:55:09 -0500 Subject: [PATCH 016/299] Address Service: Sort mempool txids --- lib/services/address/index.js | 8 +++++--- test/services/address/index.unit.js | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 98617419..9ba17ea9 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1490,6 +1490,10 @@ AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, ca result.txids.sort(function(a, b) { return result.appearanceIds[a] - result.appearanceIds[b]; }); + result.unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + result.unconfirmedTxids.sort(function(a, b) { + return result.unconfirmedAppearanceIds[a] - result.unconfirmedAppearanceIds[b]; + }); callback(null, result); }; @@ -1505,8 +1509,6 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - // TODO: Sort mempool by timestamp? - async.waterfall([ function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { @@ -1549,7 +1551,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { var confirmedTxids = result.txids; - var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + var unconfirmedTxids = result.unconfirmedTxids; var summary = { totalReceived: result.totalReceived, diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index be0767eb..1941b6ea 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2425,6 +2425,13 @@ describe('Address Service', function() { '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964, + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 } }; as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { @@ -2435,6 +2442,11 @@ describe('Address Service', function() { result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); + result.unconfirmedTxids[0].should.equal('f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345'); + result.unconfirmedTxids[1].should.equal('f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9'); + result.unconfirmedTxids[2].should.equal('edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a'); + result.unconfirmedTxids[3].should.equal('ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693'); + result.unconfirmedTxids[4].should.equal('ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681'); done(); }); }); @@ -2546,8 +2558,6 @@ describe('Address Service', function() { done(); }); }); - it.skip('will sort txids by timestamp', function(done) { - }); }); describe('#_transformAddressSummaryFromResult', function() { @@ -2566,6 +2576,10 @@ describe('Address Service', function() { '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 }, + unconfirmedTxids: [ + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d' + ], unconfirmedBalance: 500000 }; var testnode = { From e498e0fac2ebe30fa8d1444c6dbd14eef2f3222b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 16:03:37 -0500 Subject: [PATCH 017/299] Address Service: Include default callback earlier --- lib/services/address/index.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 9ba17ea9..5c38d3ca 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -231,6 +231,14 @@ AddressService.prototype.transactionLeaveHandler = function(txInfo) { AddressService.prototype.transactionHandler = function(txInfo, callback) { var self = this; + if (!callback) { + callback = function(err) { + if (err) { + return log.error(err); + } + }; + } + if (this.node.stopping) { return callback(); } @@ -246,14 +254,6 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) { this.transactionOutputHandler(messages, tx, i, !txInfo.mempool); } - if (!callback) { - callback = function(err) { - if (err) { - return log.error(err); - } - }; - } - function finish(err) { if (err) { return callback(err); From 45029030f1c64d8ba5411e8d14f73ef44bf45b27 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Jan 2016 16:16:53 -0500 Subject: [PATCH 018/299] Address Service: Sort after unconfirmed and confirmed --- lib/services/address/index.js | 6 +++--- test/services/address/index.unit.js | 31 ++++------------------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 5c38d3ca..17199b25 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1343,6 +1343,9 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }, function(result, next) { self._getAddressMempoolSummary(address, options, result, next); + }, + function(result, next) { + self._setAndSortTxidsFromAppearanceIds(result, next); } ], function(err, result) { if (err) { @@ -1379,9 +1382,6 @@ AddressService.prototype._getAddressConfirmedSummary = function(address, options }, function(result, next) { self._getAddressConfirmedOutputsSummary(address, result, options, next); - }, - function(result, next) { - self._setAndSortTxidsFromAppearanceIds(result, next); } ], callback); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 1941b6ea..a35614df 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2092,11 +2092,14 @@ describe('Address Service', function() { var summary = {}; addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function(err, sum) { addressService._getAddressConfirmedSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.callCount.should.equal(1); addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); + addressService._setAndSortTxidsFromAppearanceIds.callCount.should.equal(1); + addressService._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(cache); addressService._transformAddressSummaryFromResult.callCount.should.equal(1); addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); sum.should.equal(summary); @@ -2123,6 +2126,7 @@ describe('Address Service', function() { addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); addressService.getAddressSummary(address, options, function() { log.warn.callCount.should.equal(1); @@ -2151,7 +2155,6 @@ describe('Address Service', function() { var result = {}; as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, result); as._getAddressConfirmedSummary(address, options, function(err) { if (err) { return done(err); @@ -2169,7 +2172,6 @@ describe('Address Service', function() { as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); - as._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(result); done(); }); }); @@ -2220,31 +2222,6 @@ describe('Address Service', function() { done(); }); }); - it('will pass error correctly (sort)', function(done) { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = {}; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); - as._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, new Error('test')); - as._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); }); describe('#_getAddressConfirmedInputsSummary', function() { From 39f8355cd976082ac4185956fc7e1f60029f6c70 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Jan 2016 14:07:28 -0500 Subject: [PATCH 019/299] Address Service: Bump maximum number of addresses default --- lib/services/address/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 2cf334e9..84f9b32a 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -48,7 +48,7 @@ exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; // The maximum number of transactions that can be queried at once exports.MAX_HISTORY_QUERY_LENGTH = 100; // The maximum number of addresses that can be queried at once -exports.MAX_ADDRESSES_QUERY = 100; +exports.MAX_ADDRESSES_QUERY = 10000; module.exports = exports; From 43e472707f0103eb2182cf96b4d42a435adfe253 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Tue, 19 Jan 2016 20:49:17 +0100 Subject: [PATCH 020/299] homepage link 404 on github --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5e0a4478..a92577a2 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "lastBuild": "1.0.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", - "homepage": "https://github.com/bitpay/bitcore-node.js", + "homepage": "https://github.com/bitpay/bitcore-node", "bugs": { "url": "https://github.com/bitpay/bitcore-node/issues" }, From a2acc0c80f289a6223d0b29724c6e3c036e1de0e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Jan 2016 13:09:31 -0500 Subject: [PATCH 021/299] Address Service: Fixed test for max address limit --- test/services/address/history.unit.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 5f6266c0..6ddb7369 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -49,6 +49,7 @@ describe('Address Service History', function() { options: options, addresses: addresses }); + history.maxAddressesQuery = 100; history.get(function(err) { should.exist(err); err.message.match(/Maximum/); From 3d7fb6f234e0d02a2ad6942abc56eb9fad62c3ae Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Jan 2016 13:25:53 -0500 Subject: [PATCH 022/299] Address Service: End stream without pausing first There was an issue where streams would still be held open if "pause" was called before "end", this would lead to http requests from the insight-api not being returned with an error status as soon as possible but would instead stay open. --- lib/services/address/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 17199b25..61eed5ef 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -854,7 +854,6 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { if (inputs.length > self.maxInputsQueryLength) { log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - stream.pause(); stream.end(); } }); @@ -1076,7 +1075,6 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { if (outputs.length > self.maxOutputsQueryLength) { log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - stream.pause(); stream.end(); } }); @@ -1403,7 +1401,6 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, r if (count > self.maxInputsQueryLength) { log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - inputsStream.pause(); inputsStream.end(); } @@ -1464,7 +1461,6 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, if (count > self.maxOutputsQueryLength) { log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - outputStream.pause(); outputStream.end(); } From 98bd8ee56022188f84eaad9f0b816cd77967c53f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Jan 2016 11:14:04 -0500 Subject: [PATCH 023/299] DB Service: Include a version number for upgrading purposes --- lib/services/db.js | 69 ++++++++++++++++++-- test/services/db.unit.js | 135 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 5 deletions(-) diff --git a/lib/services/db.js b/lib/services/db.js index c3c3dfd4..6cb183d1 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -38,6 +38,10 @@ function DB(options) { Service.call(this, options); + // Used to keep track of the version of the indexes + // to determine during an upgrade if a reindex is required + this.version = 2; + this.tip = null; this.genesis = null; @@ -66,6 +70,7 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; DB.PREFIXES = { + VERSION: new Buffer('ff', 'hex'), BLOCKS: new Buffer('01', 'hex'), TIP: new Buffer('04', 'hex') }; @@ -90,6 +95,48 @@ DB.prototype._setDataPath = function() { } }; +DB.prototype._checkVersion = function(callback) { + var self = this; + var options = { + keyEncoding: 'binary', + valueEncoding: 'binary' + }; + self.store.get(DB.PREFIXES.TIP, options, function(err) { + if (err instanceof levelup.errors.NotFoundError) { + // The database is brand new and doesn't have a tip stored + // we can skip version checking + return callback(); + } else if (err) { + return callback(err); + } + self.store.get(DB.PREFIXES.VERSION, options, function(err, buffer) { + var version; + if (err instanceof levelup.errors.NotFoundError) { + // The initial version (1) of the database didn't store the version number + version = 1; + } else if (err) { + return callback(err); + } else { + version = buffer.readUInt32BE(); + } + if (self.version !== version) { + return callback(new Error( + 'The version of the database "' + version + '" does not match the expected version "' + + self.version + '". A reindex (can take several hours) is required or to switch ' + + 'versions of software to match.' + )); + } + callback(); + }); + }); +}; + +DB.prototype._setVersion = function(callback) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(this.version); + this.store.put(DB.PREFIXES.VERSION, versionBuffer, callback); +}; + /** * Called by Node to start the service. * @param {Function} callback @@ -116,14 +163,26 @@ DB.prototype.start = function(callback) { }); }); - self.loadTip(function(err) { - if(err) { + async.series([ + function(next) { + self._checkVersion(next); + }, + function(next) { + self._setVersion(next); + } + ], function(err) { + if (err) { return callback(err); } + self.loadTip(function(err) { + if (err) { + return callback(err); + } - self.sync(); - self.emit('ready'); - setImmediate(callback); + self.sync(); + self.emit('ready'); + setImmediate(callback); + }); }); }; diff --git a/test/services/db.unit.js b/test/services/db.unit.js index bf4f0c4e..5b75ee2e 100644 --- a/test/services/db.unit.js +++ b/test/services/db.unit.js @@ -104,6 +104,135 @@ describe('DB Service', function() { }); }); + describe('#_checkVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will handle an error while retrieving the tip', function() { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new Error('test')); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will handle an error while retrieving the version', function() { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new Error('test')); + } + callCount++; + setImmediate(callback); + }); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will NOT check the version if a tip is not found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); + db._checkVersion(done); + }); + it('will NOT give an error if the versions match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 2; + db._checkVersion(done); + }); + it('will give an error if the versions do NOT match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 3; + db._checkVersion(function(err) { + should.exist(err); + err.message.should.match(/^The version of the database/); + done(); + }); + }); + it('will default to version 1 if the version is NOT found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new levelup.errors.NotFoundError()); + } + callCount++; + setImmediate(callback); + }); + db.version = 1; + db._checkVersion(done); + }); + }); + + describe('#_setVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will give an error from the store', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, new Error('test')); + db._setVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will set the version', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, null); + db.version = 5; + db._setVersion(function(err) { + if (err) { + return done(err); + } + db.store.put.args[0][0].should.deep.equal(new Buffer('ff', 'hex')); + db.store.put.args[0][1].should.deep.equal(new Buffer('00000005', 'hex')); + done(); + }); + }); + }); + describe('#start', function() { var TestDB; @@ -126,6 +255,8 @@ describe('DB Service', function() { }; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); var readyFired = false; db.on('ready', function() { @@ -144,6 +275,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); db.start(function() { db.sync = function() { @@ -161,6 +294,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.node.stopping = true; db.sync = sinon.stub(); db.start(function() { From 995b4b57d4aed791b9c023ab37a42d2f078a740d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Jan 2016 13:47:26 -0500 Subject: [PATCH 024/299] DB: Include docs on how to recreate the database --- docs/services/db.md | 9 +++++++++ lib/services/db.js | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/services/db.md b/docs/services/db.md index 57439247..73a6cbb2 100644 --- a/docs/services/db.md +++ b/docs/services/db.md @@ -1,6 +1,15 @@ # Database Service This service synchronizes a leveldb database with the [Bitcoin Service](bitcoind.md) block chain by connecting and disconnecting blocks to build new indexes that can be queried. Other services can extend the data that is indexed by implementing a `blockHandler` method, similar to the built-in [Address Service](address.md). +## How to Reindex + +If you need to be able to recreate the database from historical transactions in blocks: +- Shutdown your node +- Remove the `bitcore-node.db` directory in the data directory (e.g. `~/.bitcore/bitcore-node.db`) +- Start your node again + +The database will then ask bitcoind for all the blocks again and recreate the database. This is sometimes required during upgrading as the format of the keys and values has changed. For "livenet" this can take half a day or more, for "testnet" this can take around an hour. + ## Adding Indexes For a service to include additional block data, it can implement a `blockHandler` method that will be run to when there are new blocks added or removed. diff --git a/lib/services/db.js b/lib/services/db.js index 6cb183d1..22cbadf7 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -120,10 +120,12 @@ DB.prototype._checkVersion = function(callback) { version = buffer.readUInt32BE(); } if (self.version !== version) { + var helpUrl = 'https://github.com/bitpay/bitcore-node/blob/master/docs/services/db.md#how-to-reindex'; return callback(new Error( 'The version of the database "' + version + '" does not match the expected version "' + - self.version + '". A reindex (can take several hours) is required or to switch ' + - 'versions of software to match.' + self.version + '". A recreation of "' + self.dataPath + '" (can take several hours) is ' + + 'required or to switch versions of software to match. Please see ' + helpUrl + + ' for more information.' )); } callback(); From 419aa5785b0ef2e3072763585d0a701ab1cae507 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 28 Jan 2016 14:31:58 -0500 Subject: [PATCH 025/299] Bump package version to v2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 33ad01af..94c2034f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "1.0.1-dev", + "version": "2.0.0", "lastBuild": "1.0.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", From c091ca9d671a9fe3778fd6fd7315787e8b80a871 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 28 Jan 2016 15:17:00 -0500 Subject: [PATCH 026/299] Bump development version to v2.0.0-dev --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 94c2034f..ad4cef9c 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.0.0", - "lastBuild": "1.0.1", + "version": "2.0.0-dev", + "lastBuild": "2.0.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 663b100084895c07ae25b0311eab83c6826d84b6 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 28 Jan 2016 17:42:16 -0500 Subject: [PATCH 027/299] Added arm support. --- bin/build | 5 ++--- bin/variables.sh | 32 +++++++++++++++++++++++++++----- package.json | 3 ++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/bin/build b/bin/build index a56c4a95..bed89c1d 100755 --- a/bin/build +++ b/bin/build @@ -5,6 +5,7 @@ depends_dir=$($root_dir/bin/variables.sh depends_dir) host=$(${root_dir}/bin/variables.sh host) btc_dir="${root_dir}/libbitcoind" patch_sha=$($root_dir/bin/variables.sh patch_sha) +config_lib_dir=$($root_dir/bin/variables.sh config_lib_dir) export CPPFLAGS="-I${depends_dir}/${host}/include/boost -I${depends_dir}/${host}/include -L${depends_dir}/${host}/lib" echo "Using BTC directory: ${btc_dir}" @@ -153,9 +154,7 @@ apply the current patch from "${root_dir}"/etc/bitcoin.patch? (y/N): " echo './autogen.sh' ./autogen.sh || exit -1 - boost_libdir="--with-boost-libdir=${depends_dir}/${host}/lib" - - full_options="${options} ${boost_libdir}" + full_options="${options} ${config_lib_dir}" echo "running the configure script with the following options:\n :::[\"${full_options}\"]:::" ${full_options} diff --git a/bin/variables.sh b/bin/variables.sh index 65cae0b1..8b35e0cf 100755 --- a/bin/variables.sh +++ b/bin/variables.sh @@ -1,14 +1,25 @@ #!/bin/bash exec 2> /dev/null - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." +root_dir="$(cd "$(dirname $0)" && pwd)/.." +if [ "${root_dir}" == "" ]; then + root_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." +fi bitcoin_dir="${root_dir}"/libbitcoind cache_dir="${root_dir}"/cache -platform=`uname -a | awk '{print tolower($1)}'` -arch=`uname -m` -host="${arch}"-"${platform}" +host= +compute_host () { + platform=`uname -a | awk '{print tolower($1)}'` + arch=`uname -m` + if [ "${arch:0:3}" == "arm" ]; then + host="arm-linux-gnueabihf" + else + host="${arch}"-"${platform}" + fi +} + +compute_host mac_response= check_mac_build_system () { @@ -35,6 +46,13 @@ libsecp256k1="${cache_dir}"/src/secp256k1/.libs/libsecp256k1.a ssl="${cache_dir}"/depends/"${host}"/lib/libssl.a crypto="${cache_dir}"/depends/"${host}"/lib/libcrypto.a +config_lib_dir= +if [ "${platform}" == "darwin" ]; then + config_lib_dir="--with-boost-libdir=${depends_dir}/${host}/lib" +else + config_lib_dir="--prefix=${depends_dir}/${host}" +fi + if test x"$1" = x'anl'; then if [ "${platform}" != "darwin" ]; then echo -n "-lanl" @@ -129,3 +147,7 @@ fi if test -z "$1" -o x"$1" = x'bitcoind'; then echo -n "${cache_dir}"/src/.libs/libbitcoind.a fi + +if test -z "$1" -o x"$1" = x'config_lib_dir'; then + echo -n "${config_lib_dir}" +fi diff --git a/package.json b/package.json index ad4cef9c..57eddbb9 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,8 @@ "linux" ], "cpu": [ - "x64" + "x64", + "arm" ], "license": "MIT" } From b80e3e19e2e410c5d88502d739d1bbac771a5572 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 1 Feb 2016 13:42:08 -0500 Subject: [PATCH 028/299] Added ARM prebuilt binaries for Leveldown - this is necessary because leveldown does not supply an ARM binary - butcher needs a binary so that compiler tool chains are not necessary --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 57eddbb9..04b17b05 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", - "leveldown": "^1.4.3", + "leveldown": "bitpay/leveldown#bitpay-1.4.4", "levelup": "^1.3.1", "liftoff": "^2.2.0", "memdown": "^1.0.0", From 0311d137ae536d64c87593e6ef8aa6e1c919d8b3 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 1 Feb 2016 14:41:30 -0500 Subject: [PATCH 029/299] Bump development version to v2.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 04b17b05..a7e4826c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.0.0-dev", + "version": "2.0.1", "lastBuild": "2.0.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", From ca19994326d5b88958bf0ab870f950e4c7806f49 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 1 Feb 2016 15:49:20 -0500 Subject: [PATCH 030/299] Bump development version to v2.0.1-dev --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a7e4826c..11ab01a0 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.0.1", - "lastBuild": "2.0.0", + "version": "2.0.1-dev", + "lastBuild": "2.0.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 93e5dbfc3400f068efd8aa1b86ff57a6e8f3cd8f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 2 Feb 2016 13:27:45 -0500 Subject: [PATCH 031/299] Address Service: Limit the number of simultaneous requests --- lib/services/address/constants.js | 2 ++ lib/services/address/history.js | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 84f9b32a..961f6e1f 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -49,6 +49,8 @@ exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; exports.MAX_HISTORY_QUERY_LENGTH = 100; // The maximum number of addresses that can be queried at once exports.MAX_ADDRESSES_QUERY = 10000; +// The maximum number of simultaneous requests +exports.MAX_ADDRESSES_LIMIT = 50; module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 1d707279..998dbfde 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -24,6 +24,7 @@ function AddressHistory(args) { this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; + this.maxAddressesLimit = args.options.maxAddressesLimit || constants.MAX_ADDRESSES_LIMIT; this.addressStrings = []; for (var i = 0; i < this.addresses.length; i++) { @@ -89,8 +90,9 @@ AddressHistory.prototype.get = function(callback) { } else { var opts = _.clone(this.options); opts.fullTxList = true; - async.map( + async.mapLimit( self.addresses, + self.maxAddressesLimit, function(address, next) { self.node.services.address.getAddressSummary(address, opts, next); }, From f473ddeddde934ee0095dc0b700f60abacd6391c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 3 Feb 2016 12:28:32 -0500 Subject: [PATCH 032/299] Lower and include new concurrency limits --- lib/services/address/constants.js | 2 +- lib/services/db.js | 6 ++++++ lib/transaction.js | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 961f6e1f..3653e9cb 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -50,7 +50,7 @@ exports.MAX_HISTORY_QUERY_LENGTH = 100; // The maximum number of addresses that can be queried at once exports.MAX_ADDRESSES_QUERY = 10000; // The maximum number of simultaneous requests -exports.MAX_ADDRESSES_LIMIT = 50; +exports.MAX_ADDRESSES_LIMIT = 5; module.exports = exports; diff --git a/lib/services/db.js b/lib/services/db.js index 22cbadf7..eca15bff 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -51,6 +51,7 @@ function DB(options) { this._setDataPath(); this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; + this.maxTransactionLimit = options.maxTransactionLimit || DB.MAX_TRANSACTION_LIMIT; this.levelupStore = leveldown; if (options.store) { @@ -75,6 +76,11 @@ DB.PREFIXES = { TIP: new Buffer('04', 'hex') }; +// The maximum number of transactions to query at once +// Used for populating previous inputs +DB.MAX_TRANSACTION_LIMIT = 5; + +// The default maxiumum number of files open for leveldb DB.DEFAULT_MAX_OPEN_FILES = 200; /** diff --git a/lib/transaction.js b/lib/transaction.js index bc7bda44..f55df37b 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -5,6 +5,8 @@ var levelup = require('levelup'); var bitcore = require('bitcore-lib'); var Transaction = bitcore.Transaction; +var MAX_TRANSACTION_LIMIT = 5; + Transaction.prototype.populateInputs = function(db, poolTransactions, callback) { var self = this; @@ -12,8 +14,9 @@ Transaction.prototype.populateInputs = function(db, poolTransactions, callback) return setImmediate(callback); } - async.each( + async.eachLimit( this.inputs, + db.maxTransactionLimit || MAX_TRANSACTION_LIMIT, function(input, next) { self._populateInput(db, input, poolTransactions, next); }, From 6e8f3ee917115ce07493ebdce86a759be66c0ba1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 3 Feb 2016 18:29:33 -0500 Subject: [PATCH 033/299] Add regtest from bitcore-lib --- lib/node.js | 14 +------------- lib/services/address/index.js | 9 +++++---- lib/services/bitcoind.js | 7 ++++++- lib/services/db.js | 9 +++++---- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/lib/node.js b/lib/node.js index 7853dc11..737cd307 100644 --- a/lib/node.js +++ b/lib/node.js @@ -71,19 +71,7 @@ Node.prototype._setNetwork = function(config) { if (config.network === 'testnet') { this.network = Networks.get('testnet'); } else if (config.network === 'regtest') { - Networks.remove(Networks.testnet); - Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); + Networks.enableRegtest(); this.network = Networks.get('regtest'); } else { this.network = Networks.defaultNetwork; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 61eed5ef..486ef0f4 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -121,13 +121,14 @@ AddressService.prototype._setMempoolIndexPath = function() { AddressService.prototype._getDBPathFor = function(dbname) { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); var path; - var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { path = this.node.datadir + '/' + dbname; } else if (this.node.network === Networks.testnet) { - path = this.node.datadir + '/testnet3/' + dbname; - } else if (this.node.network === regtest) { - path = this.node.datadir + '/regtest/' + dbname; + if (this.node.network.regtestEnabled) { + path = this.node.datadir + '/regtest/' + dbname; + } else { + path = this.node.datadir + '/testnet3/' + dbname; + } } else { throw new Error('Unknown network: ' + this.network); } diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index e7a67467..edc2dad6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -154,9 +154,14 @@ Bitcoin.prototype.start = function(callback) { this._loadConfiguration(); + var networkName = this.node.network.name; + if (this.node.network.regtestEnabled) { + networkName = 'regtest'; + } + bindings.start({ datadir: this.node.datadir, - network: this.node.network.name + network: networkName }, function(err) { if(err) { return callback(err); diff --git a/lib/services/db.js b/lib/services/db.js index eca15bff..679935ca 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -89,13 +89,14 @@ DB.DEFAULT_MAX_OPEN_FILES = 200; */ DB.prototype._setDataPath = function() { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); - var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { this.dataPath = this.node.datadir + '/bitcore-node.db'; } else if (this.node.network === Networks.testnet) { - this.dataPath = this.node.datadir + '/testnet3/bitcore-node.db'; - } else if (this.node.network === regtest) { - this.dataPath = this.node.datadir + '/regtest/bitcore-node.db'; + if (this.node.network.regtestEnabled) { + this.dataPath = this.node.datadir + '/regtest/bitcore-node.db'; + } else { + this.dataPath = this.node.datadir + '/testnet3/bitcore-node.db'; + } } else { throw new Error('Unknown network: ' + this.network); } From 83eba52657974574717f7f24c9d7aedfd07033c5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 4 Feb 2016 17:26:17 -0500 Subject: [PATCH 034/299] Tests: Fix tests to use enable/disableRegtest --- test/node.unit.js | 24 ++----------------- test/services/address/index.unit.js | 37 ++++++++++++++++++----------- test/services/db.unit.js | 16 ++----------- 3 files changed, 27 insertions(+), 50 deletions(-) diff --git a/test/node.unit.js b/test/node.unit.js index 4a20f37e..1e843fe0 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -21,29 +21,9 @@ describe('Bitcore Node', function() { Node.prototype._loadConfiguration = sinon.spy(); Node.prototype._initialize = sinon.spy(); }); + after(function() { - var regtest = Networks.get('regtest'); - if (regtest) { - Networks.remove(regtest); - } - // restore testnet - Networks.add({ - name: 'testnet', - alias: 'testnet', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0x0b110907, - port: 18333, - dnsSeeds: [ - 'testnet-seed.bitcoin.petertodd.org', - 'testnet-seed.bluematt.me', - 'testnet-seed.alexykot.me', - 'testnet-seed.bitcoin.schildbach.de' - ], - }); + Networks.disableRegtest(); }); describe('@constructor', function() { diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index a35614df..67bdb481 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -271,19 +271,7 @@ describe('Address Service', function() { }); it('should load the db with regtest', function() { // Switch to use regtest - Networks.remove(Networks.testnet); - Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); + Networks.enableRegtest(); var regtest = Networks.get('regtest'); var testnode = { network: regtest, @@ -299,7 +287,7 @@ describe('Address Service', function() { node: testnode }); am.mempoolIndexPath.should.equal(process.env.HOME + '/.bitcoin/regtest/bitcore-addressmempool.db'); - Networks.remove(regtest); + Networks.disableRegtest(); }); }); @@ -735,6 +723,7 @@ describe('Address Service', function() { describe('#createInputsStream', function() { it('transform stream from buffer into object', function(done) { var testnode = { + network: Networks.livenet, services: { bitcoind: { on: sinon.stub() @@ -777,6 +766,7 @@ describe('Address Service', function() { it('will stream all keys', function() { var streamStub = sinon.stub().returns({}); var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -808,6 +798,7 @@ describe('Address Service', function() { it('will stream keys based on a range of block heights', function() { var streamStub = sinon.stub().returns({}); var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -1144,6 +1135,7 @@ describe('Address Service', function() { describe('#createOutputsStream', function() { it('transform stream from buffer into object', function(done) { var testnode = { + network: Networks.livenet, services: { bitcoind: { on: sinon.stub() @@ -1188,6 +1180,7 @@ describe('Address Service', function() { it('will stream all keys', function() { var streamStub = sinon.stub().returns({}); var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -1219,6 +1212,7 @@ describe('Address Service', function() { it('will stream keys based on a range of block heights', function() { var streamStub = sinon.stub().returns({}); var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2030,6 +2024,7 @@ describe('Address Service', function() { }); it('will handle error from _getAddressConfirmedSummary', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2052,6 +2047,7 @@ describe('Address Service', function() { }); it('will handle error from _getAddressMempoolSummary', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2075,6 +2071,7 @@ describe('Address Service', function() { }); it('will pass cache and summary between functions correctly', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2108,6 +2105,7 @@ describe('Address Service', function() { }); it('will log if there is a slow query', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2139,6 +2137,7 @@ describe('Address Service', function() { describe('#_getAddressConfirmedSummary', function() { it('will pass arguments correctly', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2177,6 +2176,7 @@ describe('Address Service', function() { }); it('will pass error correctly (inputs)', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2200,6 +2200,7 @@ describe('Address Service', function() { }); it('will pass error correctly (outputs)', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2229,6 +2230,7 @@ describe('Address Service', function() { var streamStub = new stream.Readable(); streamStub._read = function() { /* do nothing */ }; var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2265,6 +2267,7 @@ describe('Address Service', function() { var streamStub = new stream.Readable(); streamStub._read = function() { /* do nothing */ }; var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2296,6 +2299,7 @@ describe('Address Service', function() { var streamStub = new stream.Readable(); streamStub._read = function() { /* do nothing */ }; var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub(), @@ -2350,6 +2354,7 @@ describe('Address Service', function() { var streamStub = new stream.Readable(); streamStub._read = function() { /* do nothing */ }; var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2386,6 +2391,7 @@ describe('Address Service', function() { describe('#_setAndSortTxidsFromAppearanceIds', function() { it('will sort correctly', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2432,6 +2438,7 @@ describe('Address Service', function() { describe('#_getAddressMempoolSummary', function() { it('skip if options not enabled', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2460,6 +2467,7 @@ describe('Address Service', function() { }); it('include all txids and balance from inputs and outputs', function(done) { var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -2560,6 +2568,7 @@ describe('Address Service', function() { unconfirmedBalance: 500000 }; var testnode = { + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() diff --git a/test/services/db.unit.js b/test/services/db.unit.js index 5b75ee2e..60568cce 100644 --- a/test/services/db.unit.js +++ b/test/services/db.unit.js @@ -77,19 +77,7 @@ describe('DB Service', function() { }); it('should load the db with regtest', function() { // Switch to use regtest - // Networks.remove(Networks.testnet); - Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); + Networks.enableRegtest(); var regtest = Networks.get('regtest'); var config = { node: { @@ -100,7 +88,7 @@ describe('DB Service', function() { }; var db = new DB(config); db.dataPath.should.equal(process.env.HOME + '/.bitcoin/regtest/bitcore-node.db'); - Networks.remove(regtest); + Networks.disableRegtest(); }); }); From 17e8173d1456fc98830881b5def77d45e07abe03 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 4 Feb 2016 17:28:10 -0500 Subject: [PATCH 035/299] Dependencies: Temporarily switch to development version of bitcore-lib --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 11ab01a0..9aa49e6a 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "dependencies": { "async": "^1.3.0", "bindings": "^1.2.1", - "bitcore-lib": "^0.13.7", + "bitcore-lib": "bitpay/bitcore-lib#9702105ad9e78977f52df6ce231d87c47742f9be", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", From 0d6bc9833392af35e4e9c00d8d4fed40be87b2c1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 5 Feb 2016 10:13:31 -0500 Subject: [PATCH 036/299] Dependencies: Updated bitcore-lib to version ^0.13.13 This release includes new API for regtest with `enableRegtest()` and `disableRegtest()` --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9aa49e6a..c466321e 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "dependencies": { "async": "^1.3.0", "bindings": "^1.2.1", - "bitcore-lib": "bitpay/bitcore-lib#9702105ad9e78977f52df6ce231d87c47742f9be", + "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", From e56fdf457f23bea31829f1330e8aaf6ec28dbf64 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 5 Feb 2016 10:33:11 -0500 Subject: [PATCH 037/299] Added network name to bitcoind.getInfo --- integration/regtest.js | 1 + src/libbitcoind.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/integration/regtest.js b/integration/regtest.js index 17f0b8e6..e7dba7a8 100644 --- a/integration/regtest.js +++ b/integration/regtest.js @@ -477,6 +477,7 @@ describe('Daemon Binding Functionality', function() { describe('#getInfo', function() { it('will get information', function() { var info = bitcoind.getInfo(); + info.network.should.equal('regtest'); should.exist(info); should.exist(info.version); should.exist(info.blocks); diff --git a/src/libbitcoind.cc b/src/libbitcoind.cc index b7b87b04..882dabbb 100644 --- a/src/libbitcoind.cc +++ b/src/libbitcoind.cc @@ -1418,6 +1418,7 @@ NAN_METHOD(GetInfo) { Nan::Set(obj, New("connections").ToLocalChecked(), New((int)vNodes.size())->ToInt32()); Nan::Set(obj, New("difficulty").ToLocalChecked(), New((double)GetDifficulty())); Nan::Set(obj, New("testnet").ToLocalChecked(), New(Params().NetworkIDString() == "test")); + Nan::Set(obj, New("network").ToLocalChecked(), New(Params().NetworkIDString()).ToLocalChecked()); Nan::Set(obj, New("relayfee").ToLocalChecked(), New(::minRelayTxFee.GetFeePerK())); // double Nan::Set(obj, New("errors").ToLocalChecked(), New(GetWarnings("statusbar")).ToLocalChecked()); From e7e33313cf12612b4957d75ac183ecd41e706e87 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 8 Feb 2016 12:41:49 -0500 Subject: [PATCH 038/299] add noBalance options + mempoolAddressIndex --- lib/services/address/history.js | 6 +- lib/services/address/index.js | 181 ++++++++++++++++++-------------- 2 files changed, 108 insertions(+), 79 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 998dbfde..b7750068 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -44,6 +44,7 @@ function AddressHistory(args) { AddressHistory.prototype._mergeAndSortTxids = function(summaries) { var appearanceIds = {}; var unconfirmedAppearanceIds = {}; + for (var i = 0; i < summaries.length; i++) { var summary = summaries[i]; for (var key in summary.appearanceIds) { @@ -79,6 +80,8 @@ AddressHistory.prototype.get = function(callback) { return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } + this.options.noBalance = true; + if (this.addresses.length === 1) { var address = this.addresses[0]; self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { @@ -89,10 +92,11 @@ AddressHistory.prototype.get = function(callback) { }); } else { var opts = _.clone(this.options); + opts.fullTxList = true; async.mapLimit( self.addresses, - self.maxAddressesLimit, + self.maxaddressesLimit, function(address, next) { self.node.services.address.getAddressSummary(address, opts, next); }, diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 61eed5ef..f67fa8ae 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -57,6 +57,7 @@ var AddressService = function(options) { } this.mempoolIndex = null; // Used for larger mempool indexes this.mempoolSpentIndex = {}; // Used for small quick synchronous lookups + this.mempoolAddressIndex = {}; // Used to check if an address is on the spend pool }; inherits(AddressService, BaseService); @@ -70,6 +71,7 @@ AddressService.prototype.start = function(callback) { var self = this; async.series([ + function(next) { // Flush any existing mempool index if (fs.existsSync(self.mempoolIndexPath)) { @@ -88,8 +90,7 @@ AddressService.prototype.start = function(callback) { }, function(next) { self.mempoolIndex = levelup( - self.mempoolIndexPath, - { + self.mempoolIndexPath, { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', @@ -154,20 +155,17 @@ AddressService.prototype.getAPIMethods = function() { * Called by the Bus to get the available events for this service. */ AddressService.prototype.getPublishEvents = function() { - return [ - { - name: 'address/transaction', - scope: this, - subscribe: this.subscribe.bind(this, 'address/transaction'), - unsubscribe: this.unsubscribe.bind(this, 'address/transaction') - }, - { - name: 'address/balance', - scope: this, - subscribe: this.subscribe.bind(this, 'address/balance'), - unsubscribe: this.unsubscribe.bind(this, 'address/balance') - } - ]; + return [{ + name: 'address/transaction', + scope: this, + subscribe: this.subscribe.bind(this, 'address/transaction'), + unsubscribe: this.unsubscribe.bind(this, 'address/transaction') + }, { + name: 'address/balance', + scope: this, + subscribe: this.subscribe.bind(this, 'address/balance'), + unsubscribe: this.unsubscribe.bind(this, 'address/balance') + }]; }; /** @@ -305,6 +303,14 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { continue; } + var hashBufferHex = addressInfo.hashBuffer.toString('hex'); + + if (add) { + this.mempoolAddressIndex[hashBufferHex] = true; + } else { + delete this.mempoolAddressIndex[hashBufferHex]; + } + // Update output index var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); @@ -397,6 +403,12 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { value: inputValue }); + var inputHashBufferHex = inputHashBuffer.toString('hex'); + if (add) { + this.mempoolAddressIndex[inputHashBufferHex] = true; + } else { + delete this.mempoolAddressIndex[inputHashBufferHex]; + } } if (!callback) { @@ -446,7 +458,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var script = output.script; - if(!script) { + if (!script) { log.debug('Invalid script'); continue; } @@ -462,7 +474,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); + height, txidBuffer, outputIndex); var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -494,11 +506,11 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { this.transactionEventHandler(txmessages[addressKey]); } - if(tx.isCoinbase()) { + if (tx.isCoinbase()) { continue; } - for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { + for (var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { var input = inputs[inputIndex]; var inputHash; @@ -560,14 +572,14 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { * @param {Boolean} obj.rejected - If the transaction was not accepted in the mempool */ AddressService.prototype.transactionEventHandler = function(obj) { - if(this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { + if (this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { var emitters = this.subscriptions['address/transaction'][obj.addressInfo.hashHex]; var address = new Address({ hashBuffer: obj.addressInfo.hashBuffer, network: this.node.network, type: obj.addressInfo.addressType }); - for(var i = 0; i < emitters.length; i++) { + for (var i = 0; i < emitters.length; i++) { emitters[i].emit('address/transaction', { rejected: obj.rejected, height: obj.height, @@ -591,7 +603,7 @@ AddressService.prototype.transactionEventHandler = function(obj) { * @param {String} obj.addressType */ AddressService.prototype.balanceEventHandler = function(block, obj) { - if(this.subscriptions['address/balance'][obj.hashHex]) { + if (this.subscriptions['address/balance'][obj.hashHex]) { var emitters = this.subscriptions['address/balance'][obj.hashHex]; var address = new Address({ hashBuffer: obj.hashBuffer, @@ -599,10 +611,10 @@ AddressService.prototype.balanceEventHandler = function(block, obj) { type: obj.addressType }); this.getBalance(address, true, function(err, balance) { - if(err) { + if (err) { return this.emit(err); } - for(var i = 0; i < emitters.length; i++) { + for (var i = 0; i < emitters.length; i++) { emitters[i].emit('address/balance', address, balance, block); } }); @@ -621,9 +633,9 @@ AddressService.prototype.subscribe = function(name, emitter, addresses) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); $.checkArgument(Array.isArray(addresses), 'Second argument is expected to be an Array of addresses'); - for(var i = 0; i < addresses.length; i++) { + for (var i = 0; i < addresses.length; i++) { var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(!this.subscriptions[name][hashHex]) { + if (!this.subscriptions[name][hashHex]) { this.subscriptions[name][hashHex] = []; } this.subscriptions[name][hashHex].push(emitter); @@ -641,16 +653,16 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); $.checkArgument(Array.isArray(addresses) || _.isUndefined(addresses), 'Second argument is expected to be an Array of addresses or undefined'); - if(!addresses) { + if (!addresses) { return this.unsubscribeAll(name, emitter); } - for(var i = 0; i < addresses.length; i++) { + for (var i = 0; i < addresses.length; i++) { var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(this.subscriptions[name][hashHex]) { + if (this.subscriptions[name][hashHex]) { var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); - if(index > -1) { + if (index > -1) { emitters.splice(index, 1); } } @@ -665,10 +677,10 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { AddressService.prototype.unsubscribeAll = function(name, emitter) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - for(var hashHex in this.subscriptions[name]) { + for (var hashHex in this.subscriptions[name]) { var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); - if(index > -1) { + if (index > -1) { emitters.splice(index, 1); } } @@ -683,7 +695,7 @@ AddressService.prototype.unsubscribeAll = function(name, emitter) { */ AddressService.prototype.getBalance = function(address, queryMempool, callback) { this.getUnspentOutputs(address, queryMempool, function(err, outputs) { - if(err) { + if (err) { return callback(err); } @@ -852,7 +864,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { stream.on('data', function(input) { inputs.push(input); if (inputs.length > self.maxInputsQueryLength) { - log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address ' + addressStr); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); stream.end(); } @@ -871,7 +883,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { return callback(error); } - if(options.queryMempool) { + if (options.queryMempool) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { if (err) { return callback(err); @@ -1092,7 +1104,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { return callback(error); } - if(options.queryMempool) { + if (options.queryMempool) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return callback(err); @@ -1176,7 +1188,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, callback) { var self = this; - if(!Array.isArray(addresses)) { + if (!Array.isArray(addresses)) { addresses = [addresses]; } @@ -1184,9 +1196,9 @@ AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, c async.eachSeries(addresses, function(address, next) { self.getUnspentOutputsForAddress(address, queryMempool, function(err, unspents) { - if(err && err instanceof errors.NoOutputs) { + if (err && err instanceof errors.NoOutputs) { return next(); - } else if(err) { + } else if (err) { return next(err); } @@ -1208,10 +1220,12 @@ AddressService.prototype.getUnspentOutputsForAddress = function(address, queryMe var self = this; - this.getOutputs(address, {queryMempool: queryMempool}, function(err, outputs) { + this.getOutputs(address, { + queryMempool: queryMempool + }, function(err, outputs) { if (err) { return callback(err); - } else if(!outputs.length) { + } else if (!outputs.length) { return callback(new errors.NoOutputs('Address ' + address + ' has no outputs'), []); } @@ -1336,6 +1350,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } async.waterfall([ + function(next) { self._getAddressConfirmedSummary(address, options, next); }, @@ -1357,9 +1372,7 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb var seconds = Math.round(timeDelta / 1000); log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); } - callback(null, summary); - }); }; @@ -1375,6 +1388,7 @@ AddressService.prototype._getAddressConfirmedSummary = function(address, options }; async.waterfall([ + function(next) { self._getAddressConfirmedInputsSummary(address, baseResult, options, next); }, @@ -1421,8 +1435,8 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, r AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); var self = this; var count = 0; @@ -1434,25 +1448,27 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, var txid = output.txid; var outputIndex = output.outputIndex; - // Bitcoind's isSpent only works for confirmed transactions - var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = output.height; + if (!options.noBalance) { + // Bitcoind's isSpent only works for confirmed transactions + var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = output.height; - if (!spentDB) { - result.balance += output.satoshis; - } + if (!spentDB) { + result.balance += output.satoshis; + } - if (options.queryMempool) { - // Check to see if this output is spent in the mempool and if so - // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(txid, 'hex'), // TODO: get buffer directly - outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - if (spentMempool) { - result.unconfirmedBalance -= output.satoshis; + if (options.queryMempool) { + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + result.unconfirmedBalance -= output.satoshis; + } } } @@ -1504,38 +1520,47 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var addressStr = address.toString(); var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + var hashBufferHex = hashBuffer.toString('hex'); + + if (!this.mempoolAddressIndex[hashBufferHex]) { + return callback(null, result); + } async.waterfall([ + function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { if (err) { return next(err); } - for(var i = 0; i < mempoolInputs.length; i++) { + for (var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } next(null, result); }); - }, function(result, next) { + }, + function(result, next) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return next(err); } - for(var i = 0; i < mempoolOutputs.length; i++) { - var output = mempoolOutputs[i]; - - result.unconfirmedAppearanceIds[output.txid] = output.timestamp; - - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(output.txid, 'hex'), // TODO: get buffer directly - output.outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - // Only add this to the balance if it's not spent in the mempool already - if (!spentMempool) { - result.unconfirmedBalance += output.satoshis; + if (!options.noBalance) { + for (var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; + + result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + result.unconfirmedBalance += output.satoshis; + } } } next(null, result); From c1d3f351f23c7834599301aea62e740c1573ec41 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 8 Feb 2016 13:21:53 -0500 Subject: [PATCH 039/299] add address index to mempool + noBalance options --- lib/services/address/history.js | 2 +- test/services/address/history.unit.js | 3 ++- test/services/address/index.unit.js | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index b7750068..946eac26 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -96,7 +96,7 @@ AddressHistory.prototype.get = function(callback) { opts.fullTxList = true; async.mapLimit( self.addresses, - self.maxaddressesLimit, + self.maxAddressesLimit, function(address, next) { self.node.services.address.getAddressSummary(address, opts, next); }, diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 6ddb7369..917ac735 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -154,7 +154,8 @@ describe('Address Service History', function() { history.node.services.address.getAddressSummary.callCount.should.equal(2); history.node.services.address.getAddressSummary.args[0][0].should.equal(address); history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ - fullTxList: true + fullTxList: true, + noBalance: true, }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index a35614df..929c0c84 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2519,6 +2519,10 @@ describe('Address Service', function() { 0 ); as.mempoolSpentIndex[spentIndexSyncKey] = true; + + var hashBufferHex = address.hashBuffer.toString('hex'); + as.mempoolAddressIndex[hashBufferHex] = true; + as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { From c65c2bad205dbd437e3ea4aa29185c09883e6409 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 8 Feb 2016 13:40:27 -0500 Subject: [PATCH 040/299] add mempoolADdressIndex test --- test/services/address/index.unit.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 929c0c84..4d4061e1 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -1977,6 +1977,7 @@ describe('Address Service', function() { am.mempoolIndex.batch = function(operations, callback) { callback.should.be.a('function'); Object.keys(am.mempoolSpentIndex).length.should.equal(14); + Object.keys(am.mempoolAddressIndex).length.should.equal(5); for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('put'); } From 6dfb3542308a79157d8972fcf0a1e7f6e36bc3c0 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 8 Feb 2016 14:37:29 -0500 Subject: [PATCH 041/299] Bump package version to v2.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c466321e..95b19a4e 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.0.1-dev", + "version": "2.1.0", "lastBuild": "2.0.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", From dae5c9d3d555044a00d04597c8e5bc6917c0fdb2 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 8 Feb 2016 15:07:32 -0500 Subject: [PATCH 042/299] fix regtests --- lib/services/address/history.js | 6 +++--- lib/services/address/index.js | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 946eac26..2d1dcd34 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -80,18 +80,18 @@ AddressHistory.prototype.get = function(callback) { return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } - this.options.noBalance = true; + var opts = _.clone(this.options); + opts.noBalance = true; if (this.addresses.length === 1) { var address = this.addresses[0]; - self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + self.node.services.address.getAddressSummary(address, opts, function(err, summary) { if (err) { return callback(err); } return self._paginateWithDetails.call(self, summary.txids, callback); }); } else { - var opts = _.clone(this.options); opts.fullTxList = true; async.mapLimit( diff --git a/lib/services/address/index.js b/lib/services/address/index.js index f67fa8ae..94c4db9f 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1447,12 +1447,13 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, var txid = output.txid; var outputIndex = output.outputIndex; + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = output.height; if (!options.noBalance) { + // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = output.height; if (!spentDB) { result.balance += output.satoshis; @@ -1546,12 +1547,12 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, if (err) { return next(err); } - if (!options.noBalance) { - for (var i = 0; i < mempoolOutputs.length; i++) { - var output = mempoolOutputs[i]; + for (var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; - result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + if (!options.noBalance) { var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly output.outputIndex From 53735025dd4d5b89701a0c4c80aac088ea779ce9 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 8 Feb 2016 15:37:30 -0500 Subject: [PATCH 043/299] Bump development version to v2.1.0-dev --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 95b19a4e..8d3a4e7d 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.1.0", - "lastBuild": "2.0.1", + "version": "2.1.0-dev", + "lastBuild": "2.1.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 9f87156adc7c2710d7a355b32de50dd38c75f2f3 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 9 Feb 2016 10:26:09 -0500 Subject: [PATCH 044/299] fix format --- lib/services/address/index.js | 116 +++++++++++++------------- test/services/address/history.unit.js | 5 +- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 94c4db9f..da903766 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -90,7 +90,8 @@ AddressService.prototype.start = function(callback) { }, function(next) { self.mempoolIndex = levelup( - self.mempoolIndexPath, { + self.mempoolIndexPath, + { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', @@ -155,17 +156,20 @@ AddressService.prototype.getAPIMethods = function() { * Called by the Bus to get the available events for this service. */ AddressService.prototype.getPublishEvents = function() { - return [{ - name: 'address/transaction', - scope: this, - subscribe: this.subscribe.bind(this, 'address/transaction'), - unsubscribe: this.unsubscribe.bind(this, 'address/transaction') - }, { - name: 'address/balance', - scope: this, - subscribe: this.subscribe.bind(this, 'address/balance'), - unsubscribe: this.unsubscribe.bind(this, 'address/balance') - }]; + return [ + { + name: 'address/transaction', + scope: this, + subscribe: this.subscribe.bind(this, 'address/transaction'), + unsubscribe: this.unsubscribe.bind(this, 'address/transaction') + }, + { + name: 'address/balance', + scope: this, + subscribe: this.subscribe.bind(this, 'address/balance'), + unsubscribe: this.unsubscribe.bind(this, 'address/balance') + } + ]; }; /** @@ -305,7 +309,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var hashBufferHex = addressInfo.hashBuffer.toString('hex'); - if (add) { + if(add) { this.mempoolAddressIndex[hashBufferHex] = true; } else { delete this.mempoolAddressIndex[hashBufferHex]; @@ -404,7 +408,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { }); var inputHashBufferHex = inputHashBuffer.toString('hex'); - if (add) { + if(add) { this.mempoolAddressIndex[inputHashBufferHex] = true; } else { delete this.mempoolAddressIndex[inputHashBufferHex]; @@ -458,7 +462,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var script = output.script; - if (!script) { + if(!script) { log.debug('Invalid script'); continue; } @@ -474,7 +478,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); + height, txidBuffer, outputIndex); var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, @@ -506,11 +510,11 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { this.transactionEventHandler(txmessages[addressKey]); } - if (tx.isCoinbase()) { + if(tx.isCoinbase()) { continue; } - for (var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { + for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { var input = inputs[inputIndex]; var inputHash; @@ -572,14 +576,14 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { * @param {Boolean} obj.rejected - If the transaction was not accepted in the mempool */ AddressService.prototype.transactionEventHandler = function(obj) { - if (this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { + if(this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { var emitters = this.subscriptions['address/transaction'][obj.addressInfo.hashHex]; var address = new Address({ hashBuffer: obj.addressInfo.hashBuffer, network: this.node.network, type: obj.addressInfo.addressType }); - for (var i = 0; i < emitters.length; i++) { + for(var i = 0; i < emitters.length; i++) { emitters[i].emit('address/transaction', { rejected: obj.rejected, height: obj.height, @@ -603,7 +607,7 @@ AddressService.prototype.transactionEventHandler = function(obj) { * @param {String} obj.addressType */ AddressService.prototype.balanceEventHandler = function(block, obj) { - if (this.subscriptions['address/balance'][obj.hashHex]) { + if(this.subscriptions['address/balance'][obj.hashHex]) { var emitters = this.subscriptions['address/balance'][obj.hashHex]; var address = new Address({ hashBuffer: obj.hashBuffer, @@ -611,10 +615,10 @@ AddressService.prototype.balanceEventHandler = function(block, obj) { type: obj.addressType }); this.getBalance(address, true, function(err, balance) { - if (err) { + if(err) { return this.emit(err); } - for (var i = 0; i < emitters.length; i++) { + for(var i = 0; i < emitters.length; i++) { emitters[i].emit('address/balance', address, balance, block); } }); @@ -633,9 +637,9 @@ AddressService.prototype.subscribe = function(name, emitter, addresses) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); $.checkArgument(Array.isArray(addresses), 'Second argument is expected to be an Array of addresses'); - for (var i = 0; i < addresses.length; i++) { + for(var i = 0; i < addresses.length; i++) { var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if (!this.subscriptions[name][hashHex]) { + if(!this.subscriptions[name][hashHex]) { this.subscriptions[name][hashHex] = []; } this.subscriptions[name][hashHex].push(emitter); @@ -653,16 +657,16 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); $.checkArgument(Array.isArray(addresses) || _.isUndefined(addresses), 'Second argument is expected to be an Array of addresses or undefined'); - if (!addresses) { + if(!addresses) { return this.unsubscribeAll(name, emitter); } - for (var i = 0; i < addresses.length; i++) { + for(var i = 0; i < addresses.length; i++) { var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if (this.subscriptions[name][hashHex]) { + if(this.subscriptions[name][hashHex]) { var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); - if (index > -1) { + if(index > -1) { emitters.splice(index, 1); } } @@ -677,10 +681,10 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { AddressService.prototype.unsubscribeAll = function(name, emitter) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - for (var hashHex in this.subscriptions[name]) { + for(var hashHex in this.subscriptions[name]) { var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); - if (index > -1) { + if(index > -1) { emitters.splice(index, 1); } } @@ -695,7 +699,7 @@ AddressService.prototype.unsubscribeAll = function(name, emitter) { */ AddressService.prototype.getBalance = function(address, queryMempool, callback) { this.getUnspentOutputs(address, queryMempool, function(err, outputs) { - if (err) { + if(err) { return callback(err); } @@ -864,7 +868,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { stream.on('data', function(input) { inputs.push(input); if (inputs.length > self.maxInputsQueryLength) { - log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address ' + addressStr); + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); stream.end(); } @@ -883,7 +887,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { return callback(error); } - if (options.queryMempool) { + if(options.queryMempool) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { if (err) { return callback(err); @@ -1104,7 +1108,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { return callback(error); } - if (options.queryMempool) { + if(options.queryMempool) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return callback(err); @@ -1188,7 +1192,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, callback) { var self = this; - if (!Array.isArray(addresses)) { + if(!Array.isArray(addresses)) { addresses = [addresses]; } @@ -1196,9 +1200,9 @@ AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, c async.eachSeries(addresses, function(address, next) { self.getUnspentOutputsForAddress(address, queryMempool, function(err, unspents) { - if (err && err instanceof errors.NoOutputs) { + if(err && err instanceof errors.NoOutputs) { return next(); - } else if (err) { + } else if(err) { return next(err); } @@ -1220,12 +1224,10 @@ AddressService.prototype.getUnspentOutputsForAddress = function(address, queryMe var self = this; - this.getOutputs(address, { - queryMempool: queryMempool - }, function(err, outputs) { + this.getOutputs(address, {queryMempool: queryMempool}, function(err, outputs) { if (err) { return callback(err); - } else if (!outputs.length) { + } else if(!outputs.length) { return callback(new errors.NoOutputs('Address ' + address + ' has no outputs'), []); } @@ -1350,7 +1352,6 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb } async.waterfall([ - function(next) { self._getAddressConfirmedSummary(address, options, next); }, @@ -1372,7 +1373,9 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb var seconds = Math.round(timeDelta / 1000); log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); } + callback(null, summary); + }); }; @@ -1388,7 +1391,6 @@ AddressService.prototype._getAddressConfirmedSummary = function(address, options }; async.waterfall([ - function(next) { self._getAddressConfirmedInputsSummary(address, baseResult, options, next); }, @@ -1435,8 +1437,8 @@ AddressService.prototype._getAddressConfirmedInputsSummary = function(address, r AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { $.checkArgument(address instanceof Address); $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); var self = this; var count = 0; @@ -1450,16 +1452,16 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result.totalReceived += output.satoshis; result.appearanceIds[txid] = output.height; - if (!options.noBalance) { + if(!options.noBalance) { // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - if (!spentDB) { + if(!spentDB) { result.balance += output.satoshis; } - if (options.queryMempool) { + if(options.queryMempool) { // Check to see if this output is spent in the mempool and if so // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( @@ -1467,7 +1469,7 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, outputIndex ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - if (spentMempool) { + if(spentMempool) { result.unconfirmedBalance -= output.satoshis; } } @@ -1523,43 +1525,41 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; var hashBufferHex = hashBuffer.toString('hex'); - if (!this.mempoolAddressIndex[hashBufferHex]) { + if(!this.mempoolAddressIndex[hashBufferHex]) { return callback(null, result); } async.waterfall([ - function(next) { self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { if (err) { return next(err); } - for (var i = 0; i < mempoolInputs.length; i++) { + for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } next(null, result); }); - }, - function(result, next) { + }, function(result, next) { self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { if (err) { return next(err); } - for (var i = 0; i < mempoolOutputs.length; i++) { + for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; result.unconfirmedAppearanceIds[output.txid] = output.timestamp; - if (!options.noBalance) { + if(!options.noBalance) { var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly output.outputIndex ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already - if (!spentMempool) { + if(!spentMempool) { result.unconfirmedBalance += output.satoshis; } } diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 917ac735..ab3ee5c8 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -122,7 +122,9 @@ describe('Address Service History', function() { history.get(function() { history.node.services.address.getAddressSummary.callCount.should.equal(1); history.node.services.address.getAddressSummary.args[0][0].should.equal(address); - history.node.services.address.getAddressSummary.args[0][1].should.equal(options); + history.node.services.address.getAddressSummary.args[0][1].should.equal({ + noBalance: true, + }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); history._mergeAndSortTxids.callCount.should.equal(0); @@ -155,7 +157,6 @@ describe('Address Service History', function() { history.node.services.address.getAddressSummary.args[0][0].should.equal(address); history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ fullTxList: true, - noBalance: true, }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); From d0c2fa61d8526d23f102778f8da472f0139b18a8 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 9 Feb 2016 10:57:40 -0500 Subject: [PATCH 045/299] fix tests --- lib/services/address/encoding.js | 9 +++++++++ lib/services/address/index.js | 17 +++++++++-------- test/services/address/history.unit.js | 3 ++- test/services/address/index.unit.js | 5 +++-- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 81421a18..ad4b55eb 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -19,6 +19,15 @@ exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { return key.toString('binary'); }; +exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) { + var key = Buffer.concat([ + hashBuffer, + hashTypeBuffer, + ]); + return key; +}; + + exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(height); diff --git a/lib/services/address/index.js b/lib/services/address/index.js index da903766..b5a98e18 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -307,12 +307,12 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { continue; } - var hashBufferHex = addressInfo.hashBuffer.toString('hex'); + var addressIndexKey = encoding.encodeMempoolAddressIndexKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer); if(add) { - this.mempoolAddressIndex[hashBufferHex] = true; + this.mempoolAddressIndex[addressIndexKey] = true; } else { - delete this.mempoolAddressIndex[hashBufferHex]; + delete this.mempoolAddressIndex[addressIndexKey]; } // Update output index @@ -407,11 +407,12 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { value: inputValue }); - var inputHashBufferHex = inputHashBuffer.toString('hex'); + var addressIndexKey = encoding.encodeMempoolAddressIndexKey(inputHashBuffer, inputHashType); + if(add) { - this.mempoolAddressIndex[inputHashBufferHex] = true; + this.mempoolAddressIndex[addressIndexKey] = true; } else { - delete this.mempoolAddressIndex[inputHashBufferHex]; + delete this.mempoolAddressIndex[addressIndexKey]; } } @@ -1523,9 +1524,9 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var addressStr = address.toString(); var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - var hashBufferHex = hashBuffer.toString('hex'); + var addressIndexKey = encoding.encodeMempoolAddressIndexKey(hashBuffer, hashTypeBuffer); - if(!this.mempoolAddressIndex[hashBufferHex]) { + if(!this.mempoolAddressIndex[addressIndexKey]) { return callback(null, result); } diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index ab3ee5c8..4c6306dd 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -122,7 +122,7 @@ describe('Address Service History', function() { history.get(function() { history.node.services.address.getAddressSummary.callCount.should.equal(1); history.node.services.address.getAddressSummary.args[0][0].should.equal(address); - history.node.services.address.getAddressSummary.args[0][1].should.equal({ + history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ noBalance: true, }); history._paginateWithDetails.callCount.should.equal(1); @@ -157,6 +157,7 @@ describe('Address Service History', function() { history.node.services.address.getAddressSummary.args[0][0].should.equal(address); history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ fullTxList: true, + noBalance: true, }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 4d4061e1..62745395 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2521,8 +2521,9 @@ describe('Address Service', function() { ); as.mempoolSpentIndex[spentIndexSyncKey] = true; - var hashBufferHex = address.hashBuffer.toString('hex'); - as.mempoolAddressIndex[hashBufferHex] = true; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + var addressIndex = encoding.encodeMempoolAddressIndexKey(address.hashBuffer, hashTypeBuffer); + as.mempoolAddressIndex[addressIndex] = true; as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); From e7895b4b34001c25ec71be4f37abb076ce230618 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 9 Feb 2016 15:30:40 -0500 Subject: [PATCH 046/299] use key as binary --- lib/services/address/encoding.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index ad4b55eb..8ebce51c 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -24,7 +24,7 @@ exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) { hashBuffer, hashTypeBuffer, ]); - return key; + return key.toString('binary'); }; From 3bb3d82aaca05c78b1d389f1d03fb8b79a0e32dd Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 10 Feb 2016 15:03:34 -0500 Subject: [PATCH 047/299] add counter for address mempool index --- lib/services/address/index.js | 31 +++++++++++++++++++---------- test/services/address/index.unit.js | 5 ++++- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index b5a98e18..35829758 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -275,6 +275,25 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) { }; +AddressService.prototype._updateAddressIndex = function(key, add) { + + var currentValue = this.mempoolAddressIndex[key] || 0; + + if(add) { + if (currentValue > 0) + this.mempoolAddressIndex[key] = currentValue + 1; + else + this.mempoolAddressIndex[key] = 1; + } else { + if (currentValue < 1) { + delete this.mempoolAddressIndex[key]; + } else { + this.mempoolAddressIndex[key]--; + } + } +}; + + /** * This function will update the mempool address index with the necessary * information for further lookups. @@ -309,11 +328,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var addressIndexKey = encoding.encodeMempoolAddressIndexKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer); - if(add) { - this.mempoolAddressIndex[addressIndexKey] = true; - } else { - delete this.mempoolAddressIndex[addressIndexKey]; - } + this._updateAddressIndex(addressIndexKey, add); // Update output index var outputIndexBuffer = new Buffer(4); @@ -409,11 +424,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var addressIndexKey = encoding.encodeMempoolAddressIndexKey(inputHashBuffer, inputHashType); - if(add) { - this.mempoolAddressIndex[addressIndexKey] = true; - } else { - delete this.mempoolAddressIndex[addressIndexKey]; - } + this._updateAddressIndex(addressIndexKey, add); } if (!callback) { diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 62745395..d5632c8f 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -9,6 +9,7 @@ var bitcorenode = require('../../../'); var AddressService = bitcorenode.services.Address; var blockData = require('../../data/livenet-345003.json'); var bitcore = require('bitcore-lib'); +var _ = bitcore.deps._; var memdown = require('memdown'); var leveldown = require('leveldown'); var Networks = bitcore.Networks; @@ -1978,6 +1979,7 @@ describe('Address Service', function() { callback.should.be.a('function'); Object.keys(am.mempoolSpentIndex).length.should.equal(14); Object.keys(am.mempoolAddressIndex).length.should.equal(5); + _.values(am.mempoolAddressIndex).should.deep.equal([1,1,12,1,1]); for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('put'); } @@ -2013,6 +2015,7 @@ describe('Address Service', function() { for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('del'); } + Object.keys(am.mempoolAddressIndex).length.should.equal(0); }; am.updateMempoolIndex(tx, false); }); @@ -2523,7 +2526,7 @@ describe('Address Service', function() { var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; var addressIndex = encoding.encodeMempoolAddressIndexKey(address.hashBuffer, hashTypeBuffer); - as.mempoolAddressIndex[addressIndex] = true; + as.mempoolAddressIndex[addressIndex] = 1; as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); From 02f2234004fdc41092b090fd03e18d14bafd0466 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 10 Feb 2016 15:05:05 -0500 Subject: [PATCH 048/299] rm extra commas --- test/services/address/history.unit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 4c6306dd..2b6df06c 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -123,7 +123,7 @@ describe('Address Service History', function() { history.node.services.address.getAddressSummary.callCount.should.equal(1); history.node.services.address.getAddressSummary.args[0][0].should.equal(address); history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ - noBalance: true, + noBalance: true }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); @@ -157,7 +157,7 @@ describe('Address Service History', function() { history.node.services.address.getAddressSummary.args[0][0].should.equal(address); history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ fullTxList: true, - noBalance: true, + noBalance: true }); history._paginateWithDetails.callCount.should.equal(1); history._paginateWithDetails.args[0][0].should.equal(txids); From 1a68ca4fae32b6255cd8af6fa9d2cdcff9e4c9bf Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 10 Feb 2016 15:38:02 -0500 Subject: [PATCH 049/299] add tests to _updateAddressIndex --- lib/services/address/index.js | 6 ++-- test/services/address/index.unit.js | 50 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 35829758..bdbfe342 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -276,14 +276,14 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) { }; AddressService.prototype._updateAddressIndex = function(key, add) { - var currentValue = this.mempoolAddressIndex[key] || 0; if(add) { - if (currentValue > 0) + if (currentValue > 0) { this.mempoolAddressIndex[key] = currentValue + 1; - else + } else { this.mempoolAddressIndex[key] = 1; + } } else { if (currentValue < 1) { delete this.mempoolAddressIndex[key]; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index d5632c8f..3c2b250b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2433,6 +2433,56 @@ describe('Address Service', function() { }); }); + + describe.only('#_updateAddressIndex', function() { + var as; + beforeEach(function(){ + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + }); + it('should add using 2 keys', function() { + _.values(as.mempoolAddressIndex).should.deep.equal([]); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index2', true); + as._updateAddressIndex('index2', true); + as.mempoolAddressIndex.should.deep.equal({ + "index1": 4, + "index2": 2 + }); + }); + it('should add/remove using 2 keys', function() { + _.values(as.mempoolAddressIndex).should.deep.equal([]); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', true); + as._updateAddressIndex('index1', false); + + as._updateAddressIndex('index2', true); + as._updateAddressIndex('index2', true); + as._updateAddressIndex('index2', false); + as._updateAddressIndex('index2', false); + as._updateAddressIndex('index2', false); + as.mempoolAddressIndex.should.deep.equal({ + "index1": 3 + }); + }); + }); + + describe('#_getAddressMempoolSummary', function() { it('skip if options not enabled', function(done) { var testnode = { From 6e600b5def5ca1eff68c9aea7f2ae23dfcbc0a28 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 10 Feb 2016 16:08:27 -0500 Subject: [PATCH 050/299] refactor test --- test/services/address/index.unit.js | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 3c2b250b..437c8673 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2434,9 +2434,8 @@ describe('Address Service', function() { }); - describe.only('#_updateAddressIndex', function() { - var as; - beforeEach(function(){ + describe('#_updateAddressIndex', function() { + it('should add using 2 keys', function() { var testnode = { services: { bitcoind: { @@ -2445,12 +2444,11 @@ describe('Address Service', function() { }, datadir: 'testdir' }; - as = new AddressService({ + var as = new AddressService({ mempoolMemoryIndex: true, node: testnode }); - }); - it('should add using 2 keys', function() { + _.values(as.mempoolAddressIndex).should.deep.equal([]); as._updateAddressIndex('index1', true); as._updateAddressIndex('index1', true); @@ -2463,7 +2461,20 @@ describe('Address Service', function() { "index2": 2 }); }); + it('should add/remove using 2 keys', function() { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); _.values(as.mempoolAddressIndex).should.deep.equal([]); as._updateAddressIndex('index1', true); as._updateAddressIndex('index1', true); From 4d03aaa73fbc389f01079f723ab9e88a250bd53a Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 10 Feb 2016 16:18:27 -0500 Subject: [PATCH 051/299] use mocknode --- test/services/address/index.unit.js | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 437c8673..8e8527f4 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2436,17 +2436,9 @@ describe('Address Service', function() { describe('#_updateAddressIndex', function() { it('should add using 2 keys', function() { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; var as = new AddressService({ mempoolMemoryIndex: true, - node: testnode + node: mocknode }); _.values(as.mempoolAddressIndex).should.deep.equal([]); @@ -2463,17 +2455,9 @@ describe('Address Service', function() { }); it('should add/remove using 2 keys', function() { - var testnode = { - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; var as = new AddressService({ mempoolMemoryIndex: true, - node: testnode + node: mocknode }); _.values(as.mempoolAddressIndex).should.deep.equal([]); as._updateAddressIndex('index1', true); From 4894f1abec36d85097f4c8dcb6130ab5a590bb55 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Tue, 9 Feb 2016 14:32:51 -0500 Subject: [PATCH 052/299] Enable Cross-Compiling support 1. To use this feature, set CC and CXX env variables to the appropriate cross compiler 2. Example, for cross compiling to ARM, use: CC=arm-linux-gnueabihf-gcc-4.9 CXX=arm-linux-gnueabihf-g++-4.9 npm install 3. You can still compile without setting CC and CXX, you can still just run npm install --- bin/build | 11 +++++++---- bin/get-tarball-name.js | 4 +++- bin/package.js | 2 +- bin/upload.js | 2 +- bin/variables.sh | 41 +++++++++++++++++++++++++++++++++++------ etc/bitcoin.patch | 22 ++++++++-------------- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/bin/build b/bin/build index bed89c1d..8c0fbe64 100755 --- a/bin/build +++ b/bin/build @@ -1,9 +1,10 @@ #!/bin/bash root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." options=`cat ${root_dir}/bin/config_options.sh` +host=$(${root_dir}/bin/variables.sh host) || exit -1 depends_dir=$($root_dir/bin/variables.sh depends_dir) -host=$(${root_dir}/bin/variables.sh host) btc_dir="${root_dir}/libbitcoind" +sys=$($root_dir/bin/variables.sh sys) patch_sha=$($root_dir/bin/variables.sh patch_sha) config_lib_dir=$($root_dir/bin/variables.sh config_lib_dir) export CPPFLAGS="-I${depends_dir}/${host}/include/boost -I${depends_dir}/${host}/include -L${depends_dir}/${host}/lib" @@ -51,7 +52,7 @@ compare_patch () { cache_files () { cache_file="${root_dir}"/cache/cache.tar pushd "${btc_dir}" || exit -1 - find . -type f \( -name "*.h" -or -name "*.hpp" -or -name \ + find src depends/${host} -type f \( -name "*.h" -or -name "*.hpp" -or -name \ "*.ipp" -or -name "*.a" \) | tar -cf "${cache_file}" -T - if test $? -ne 0; then echo "We were trying to copy over your cached artifacts, but there was an issue." @@ -154,7 +155,8 @@ apply the current patch from "${root_dir}"/etc/bitcoin.patch? (y/N): " echo './autogen.sh' ./autogen.sh || exit -1 - full_options="${options} ${config_lib_dir}" + config_host="--host ${host}" + full_options="${options} ${config_host} ${config_lib_dir}" echo "running the configure script with the following options:\n :::[\"${full_options}\"]:::" ${full_options} @@ -181,4 +183,5 @@ if test x"$1" = x'debug'; then debug=--debug fi -node-gyp ${debug} rebuild +echo "running::: 'node-gyp ${sys} ${debug} rebuild'" +node-gyp ${sys} ${debug} rebuild diff --git a/bin/get-tarball-name.js b/bin/get-tarball-name.js index e931904b..c7ea0ffb 100644 --- a/bin/get-tarball-name.js +++ b/bin/get-tarball-name.js @@ -1,10 +1,12 @@ 'use strict'; +var execSync = require('child_process').execSync; + function getTarballName() { var packageRoot = __dirname + '/..'; var version = require(packageRoot + '/package.json').version; var platform = process.platform; - var arch = process.arch; + var arch = execSync(packageRoot + '/bin/variables.sh arch').toString(); var abi = process.versions.modules; var tarballName = 'libbitcoind-' + version + '-node' + abi + '-' + platform + '-' + arch + '.tgz'; return tarballName; diff --git a/bin/package.js b/bin/package.js index d5b00d07..c03639bd 100644 --- a/bin/package.js +++ b/bin/package.js @@ -2,7 +2,7 @@ var exec = require('child_process').exec; var bindings = require('bindings'); -var index = require('../'); +var index = require('../lib'); var log = index.log; var packageRoot = bindings.getRoot(bindings.getFileName()); diff --git a/bin/upload.js b/bin/upload.js index b1c66bf0..4376bb20 100644 --- a/bin/upload.js +++ b/bin/upload.js @@ -3,7 +3,7 @@ var fs = require('fs'); var AWS = require('aws-sdk'); var bindings = require('bindings'); -var index = require('../'); +var index = require('../lib'); var log = index.log; var config = require(process.env.HOME + '/.bitcore-node-upload.json'); diff --git a/bin/variables.sh b/bin/variables.sh index 8b35e0cf..21b962ed 100755 --- a/bin/variables.sh +++ b/bin/variables.sh @@ -8,18 +8,37 @@ fi bitcoin_dir="${root_dir}"/libbitcoind cache_dir="${root_dir}"/cache -host= -compute_host () { +get_host_and_platform () { platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` if [ "${arch:0:3}" == "arm" ]; then - host="arm-linux-gnueabihf" - else - host="${arch}"-"${platform}" + platform="linux-gnueabihf" + arch="arm" + fi + if [ -n "${CXX}" ] && [ -n "${CC}" ]; then + cc_target=$("${CC}" -v 2>&1 | awk '/Target:/ {print $2}') + cxx_target=$("${CXX}" -v 2>&1 | awk '/Target:/ {print $2}') + IFS='-' read -ra SYS <<< "${cc_target}" + if [ "${SYS[0]}" != "${arch}" ]; then + if [ -n "${SYS[1]}" ] && [ -n "${SYS[2]}" ] && hash "${CXX}" && hash "${CC}" && [ -n "${cc_target}" ] && [ -n "${cxx_target}" ]; then + #try and see if we've got a cross compiler, if not then auto detect + arch="${SYS[0]}" + platform="${SYS[1]}"-"${SYS[2]}" + else + error_message="You've specified a cross compiler, but we could not compute the host-platform-triplet for cross compilation. Please set CC and CXX environment variables with host-platform-triplet-*. Example: CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++" + return_error_message + fi + fi fi } -compute_host +return_error_message () { + echo "${error_message}" + exit -1 +} + +get_host_and_platform +host="${arch}"-"${platform}" mac_response= check_mac_build_system () { @@ -115,6 +134,10 @@ if test -z "$1" -o x"$1" = x'host'; then echo -n "${host}" fi +if test -z "$1" -o x"$1" = x'arch'; then + echo -n "${arch}" +fi + if test -z "$1" -o x"$1" = x'bdb'; then if [ "${BITCORENODE_ENV}" == "test" ]; then echo -n "${cache_dir}"/depends/"${host}"/lib/libdb_cxx.a @@ -144,6 +167,12 @@ if test -z "$1" -o x"$1" = x'wallet_enabled'; then fi fi +if test -z "$1" -o x"$1" = x'sys'; then + if [ -n "${SYS}" ]; then + echo -n "--arch=${SYS[0]}" + fi +fi + if test -z "$1" -o x"$1" = x'bitcoind'; then echo -n "${cache_dir}"/src/.libs/libbitcoind.a fi diff --git a/etc/bitcoin.patch b/etc/bitcoin.patch index 3eda2bc7..d03b22d4 100644 --- a/etc/bitcoin.patch +++ b/etc/bitcoin.patch @@ -163,23 +163,21 @@ index e7aa48d..df0f7ae 100644 endef diff --git a/src/Makefile.am b/src/Makefile.am -index 2461f82..7be6d6e 100644 +index 2461f82..e7e9ecf 100644 --- a/src/Makefile.am +++ b/src/Makefile.am -@@ -1,6 +1,12 @@ +@@ -1,6 +1,10 @@ DIST_SUBDIRS = secp256k1 AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) +noinst_LTLIBRARIES = +libbitcoind_la_LIBADD = +libbitcoind_la_LDFLAGS = -no-undefined -+STATIC_BOOST_LIBS = -+STATIC_BDB_LIBS = -+STATIC_EXTRA_LIBS = $(STATIC_BOOST_LIBS) $(LIBLEVELDB) $(LIBMEMENV) ++STATIC_EXTRA_LIBS = $(LIBLEVELDB) $(LIBMEMENV) if EMBEDDED_LEVELDB LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include -@@ -49,16 +55,16 @@ BITCOIN_INCLUDES += $(BDB_CPPFLAGS) +@@ -49,16 +53,16 @@ BITCOIN_INCLUDES += $(BDB_CPPFLAGS) EXTRA_LIBRARIES += libbitcoin_wallet.a endif @@ -203,7 +201,7 @@ index 2461f82..7be6d6e 100644 if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif -@@ -66,6 +72,9 @@ endif +@@ -66,6 +70,9 @@ endif if BUILD_BITCOIN_UTILS bin_PROGRAMS += bitcoin-cli bitcoin-tx endif @@ -213,7 +211,7 @@ index 2461f82..7be6d6e 100644 .PHONY: FORCE # bitcoin core # -@@ -170,8 +179,11 @@ obj/build.h: FORCE +@@ -170,8 +177,11 @@ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj @$(top_srcdir)/share/genbuild.sh $(abs_top_builddir)/src/obj/build.h \ $(abs_top_srcdir) @@ -226,7 +224,7 @@ index 2461f82..7be6d6e 100644 # server: shared between bitcoind and bitcoin-qt libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) libbitcoin_server_a_SOURCES = \ -@@ -310,9 +322,18 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h +@@ -310,9 +320,18 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h bitcoind_SOURCES = bitcoind.cpp bitcoind_CPPFLAGS = $(BITCOIN_INCLUDES) bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -245,17 +243,13 @@ index 2461f82..7be6d6e 100644 endif bitcoind_LDADD = \ -@@ -327,10 +348,21 @@ bitcoind_LDADD = \ +@@ -327,10 +346,17 @@ bitcoind_LDADD = \ if ENABLE_WALLET bitcoind_LDADD += libbitcoin_wallet.a -+STATIC_EXTRA_LIBS += $(STATIC_BDB_LIBS) +libbitcoind_la_SOURCES += $(libbitcoin_wallet_a_SOURCES) endif -+STATIC_BOOST_LIBS += ../depends/$(ARCH_PLATFORM)/lib/libboost_filesystem-mt.a ../depends/$(ARCH_PLATFORM)/lib/libboost_system-mt.a ../depends/$(ARCH_PLATFORM)/lib/libboost_chrono-mt.a ../depends/$(ARCH_PLATFORM)/lib/libboost_thread-mt.a ../depends/$(ARCH_PLATFORM)/lib/libboost_program_options-mt.a -+STATIC_BDB_LIBS += ../depends/$(ARCH_PLATFORM)/lib/libdb_cxx.a -+ bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) -# +libbitcoind_la_LIBADD += $(SSL_LIBS) $(LIBSECP256K1) $(CRYPTO_LIBS) $(STATIC_EXTRA_LIBS) From afce33e5ff7b6779df7bf5ba6646e38b907afa1c Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 10 Feb 2016 15:00:04 -0500 Subject: [PATCH 053/299] Fixed test to refer to variables.sh for the architecture. --- test/bin/get-tarball-name.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bin/get-tarball-name.js b/test/bin/get-tarball-name.js index fa2b0601..6f8bdbde 100644 --- a/test/bin/get-tarball-name.js +++ b/test/bin/get-tarball-name.js @@ -3,13 +3,14 @@ var should = require('chai').should(); var path = require('path'); var getTarballName = require('../../bin/get-tarball-name'); +var execSync = require('child_process').execSync; describe('#getTarballName', function() { it('will return the expected tarball name', function() { var name = getTarballName(); var version = require(path.resolve(__dirname + '../../../package.json')).version; var platform = process.platform; - var arch = process.arch; + var arch = execSync(path.resolve(__dirname) + '/../../bin/variables.sh arch'); var abi = process.versions.modules; var expected = 'libbitcoind-' + version + '-node' + abi + '-' + platform + '-' + arch + '.tgz'; name.should.equal(expected); From 610b9ea269b6875c7ca990a7b69e5e949606ddf7 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 10 Feb 2016 17:18:08 -0500 Subject: [PATCH 054/299] Added a doc fragment in build.md about cross compilation and clarified the error message. --- bin/variables.sh | 2 +- docs/build.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/bin/variables.sh b/bin/variables.sh index 21b962ed..0988e0fa 100755 --- a/bin/variables.sh +++ b/bin/variables.sh @@ -25,7 +25,7 @@ get_host_and_platform () { arch="${SYS[0]}" platform="${SYS[1]}"-"${SYS[2]}" else - error_message="You've specified a cross compiler, but we could not compute the host-platform-triplet for cross compilation. Please set CC and CXX environment variables with host-platform-triplet-*. Example: CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++" + error_message="You've specified a cross compiler, but we could not compute the host-platform-triplet for cross compilation. Please set CC and CXX environment variables with host-platform-triplet-*. Also ensure the cross compiler exists on your system and is available on your path. Example: CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++" return_error_message fi fi diff --git a/docs/build.md b/docs/build.md index 1846356f..d22b8411 100644 --- a/docs/build.md +++ b/docs/build.md @@ -80,6 +80,34 @@ And finally run the build which will take several minutes. A script in the "bin" npm install ``` +## Cross Compilation +If you desire to cross compile to ARM or Windows from a system that has cross compilation tools available for use, please use the following directions: + +Using a Debian (Jessie) system as the host system (the system that will be doing the compiling): + +```bash +echo -n "deb http://emdebian.org/tools/debian/ jessie main" | sudo tee -a /etc/apt/sources.list +sudo dpkg --add-architecture armhf #or whatever arch you are interested in compiling for +sudo apt-get update #you will get GPG KEY warnings, you can decide if you would like to trust the key +sudo apt-get install crossbuild-essential-armhf +``` + +Next is to use the cross compilation toolchain instead of the defaults: + +```bash +CXX=arm-linux-gnueabihf-g++ CC=arm-linux-gnueabihf-gcc npm install +``` + +The only thing different is the setting of CC/CXX environment variables. Please make sure those compilers (arm-linux-gnueabihf-gcc) actually exist and are on your path. + +```bash +arm-linux-gnueabihf-g++ -v +arm-linux-gnueabihf-gcc -v +``` + +You should get output with the last line ending with something like this: +gcc version 4.9.2 ( 4.9.2-10) + Once everything is built, you can run bitcore-node via: ```bash From e36cdb717a47e10c7da5164be7df9555e3bf034f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 11 Feb 2016 10:42:30 -0500 Subject: [PATCH 055/299] rm empty keys --- lib/services/address/index.js | 2 +- test/services/address/index.unit.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index bdbfe342..0a3c3555 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -285,7 +285,7 @@ AddressService.prototype._updateAddressIndex = function(key, add) { this.mempoolAddressIndex[key] = 1; } } else { - if (currentValue < 1) { + if (currentValue <= 1) { delete this.mempoolAddressIndex[key]; } else { this.mempoolAddressIndex[key]--; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 8e8527f4..ce971636 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2470,6 +2470,9 @@ describe('Address Service', function() { as._updateAddressIndex('index2', true); as._updateAddressIndex('index2', false); as._updateAddressIndex('index2', false); + as.mempoolAddressIndex.should.deep.equal({ + "index1": 3 + }); as._updateAddressIndex('index2', false); as.mempoolAddressIndex.should.deep.equal({ "index1": 3 From 8c1022148023d794a980c796ff34cd2e517d507f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 4 Mar 2016 13:47:22 -0500 Subject: [PATCH 056/299] bindings: fixes confirmation issue with orphaned block transactions --- integration/regtest-node.js | 62 +++++++++++++++++++++++++++++++++++++ src/libbitcoind.cc | 6 +++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index e05e3039..9523f79a 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -796,4 +796,66 @@ describe('Node Functionality', function() { }); }); + + describe('Orphaned Transactions', function() { + var orphanedTransaction; + + before(function(done) { + var count; + var invalidatedBlockHash; + + async.series([ + function(next) { + client.getBlockCount(function(err, response) { + if (err) { + return next(err); + } + count = response.result; + next(); + }); + }, + function(next) { + client.getBlockHash(count, function(err, response) { + if (err) { + return next(err); + } + invalidatedBlockHash = response.result; + next(); + }); + }, + function(next) { + client.getBlock(invalidatedBlockHash, function(err, response) { + if (err) { + return next(err); + } + orphanedTransaction = response.result.tx[1]; + next(); + }); + }, + function(next) { + client.invalidateBlock(invalidatedBlockHash, next); + } + ], function(err) { + if (err) { + throw err; + } + done(); + }); + }); + + it('will not show confirmation count for orphaned transaction', function(done) { + // This test verifies that in the situation that the transaction is not in the mempool and + // is included in an orphaned block transaction index that the confirmation count will be unconfirmed. + node.services.bitcoind.getTransactionWithBlockInfo(orphanedTransaction, false, function(err, data) { + if (err) { + return done(err); + } + should.exist(data.height); + data.height.should.equal(-1); + done(); + }); + }); + + }); + }); diff --git a/src/libbitcoind.cc b/src/libbitcoind.cc index 882dabbb..4627b6d3 100644 --- a/src/libbitcoind.cc +++ b/src/libbitcoind.cc @@ -1230,7 +1230,11 @@ async_get_tx_and_info(uv_work_t *req) { data->height = -1; } else { blockIndex = mapBlockIndex[blockHash]; - data->height = blockIndex->nHeight; + if (!chainActive.Contains(blockIndex)) { + data->height = -1; + } else { + data->height = blockIndex->nHeight; + } } } From b55ecf304416e3ed7a19ac436680b1481073b188 Mon Sep 17 00:00:00 2001 From: Jan Pochyla Date: Fri, 1 Apr 2016 18:10:14 +0200 Subject: [PATCH 057/299] clamp tx pagination to 0 --- lib/services/address/history.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 2d1dcd34..88b5c96b 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -119,8 +119,8 @@ AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { // Slice the page starting with the most recent var txids; if (self.options.from >= 0 && self.options.to >= 0) { - var fromOffset = totalCount - self.options.from; - var toOffset = totalCount - self.options.to; + var fromOffset = Math.max(0, totalCount - self.options.from); + var toOffset = Math.max(0, totalCount - self.options.to); txids = allTxids.slice(toOffset, fromOffset); } else { txids = allTxids; From 6147be5c499b985bfa80d60c28b8c9b46a730f1e Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 8 Apr 2016 10:33:57 -0400 Subject: [PATCH 058/299] Bump package version to v2.1.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8d3a4e7d..144bcf43 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.1.0-dev", + "version": "2.1.1", "lastBuild": "2.1.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", From 07c317df80bfcd985adb951499daecf3bade380a Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 8 Apr 2016 11:31:22 -0400 Subject: [PATCH 059/299] Bump development version to v2.1.1-dev --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 144bcf43..8b31e793 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.1.1", - "lastBuild": "2.1.0", + "version": "2.1.1-dev", + "lastBuild": "2.1.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 7e70bbfa7d32725789204c68910b81429e4c8b9a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 19:35:24 -0400 Subject: [PATCH 060/299] bitcoind: bitcoind service using rpc and zmq with address index --- PATCH_VERSION | 1 - benchmarks/blockhandler.js | 75 - bin/build | 187 -- bin/clean | 9 - bin/config_options.sh | 2 - bin/config_options_debug.sh | 2 - bin/config_options_test.sh | 2 - bin/get-tarball-name.js | 19 - bin/install | 36 - bin/package.js | 58 - bin/patch-bitcoin | 36 - bin/start-libbitcoind.js | 61 - bin/start.js | 8 - bin/upload.js | 52 - bin/variables.sh | 182 -- binding.gyp | 59 - cache/.gitignore | 3 - etc/bitcoin.patch | 387 ---- example/client.js | 49 - index.js | 13 - lib/services/address/constants.js | 56 - lib/services/address/encoding.js | 307 ---- lib/services/address/history.js | 266 --- lib/services/address/index.js | 1611 ---------------- .../address/streams/inputs-transform.js | 40 - .../address/streams/outputs-transform.js | 42 - lib/services/bitcoind.js | 713 ++++++-- lib/services/db.js | 812 --------- package.json | 33 +- src/libbitcoind.cc | 1614 ----------------- src/libbitcoind.h | 19 - 31 files changed, 576 insertions(+), 6178 deletions(-) delete mode 100644 PATCH_VERSION delete mode 100644 benchmarks/blockhandler.js delete mode 100755 bin/build delete mode 100755 bin/clean delete mode 100644 bin/config_options.sh delete mode 100644 bin/config_options_debug.sh delete mode 100644 bin/config_options_test.sh delete mode 100644 bin/get-tarball-name.js delete mode 100755 bin/install delete mode 100644 bin/package.js delete mode 100755 bin/patch-bitcoin delete mode 100644 bin/start-libbitcoind.js delete mode 100755 bin/start.js delete mode 100644 bin/upload.js delete mode 100755 bin/variables.sh delete mode 100644 binding.gyp delete mode 100644 cache/.gitignore delete mode 100644 etc/bitcoin.patch delete mode 100644 example/client.js delete mode 100644 lib/services/address/constants.js delete mode 100644 lib/services/address/encoding.js delete mode 100644 lib/services/address/history.js delete mode 100644 lib/services/address/index.js delete mode 100644 lib/services/address/streams/inputs-transform.js delete mode 100644 lib/services/address/streams/outputs-transform.js delete mode 100644 lib/services/db.js delete mode 100644 src/libbitcoind.cc delete mode 100644 src/libbitcoind.h diff --git a/PATCH_VERSION b/PATCH_VERSION deleted file mode 100644 index 4b8f7b07..00000000 --- a/PATCH_VERSION +++ /dev/null @@ -1 +0,0 @@ -v0.11.2 diff --git a/benchmarks/blockhandler.js b/benchmarks/blockhandler.js deleted file mode 100644 index 3b46e750..00000000 --- a/benchmarks/blockhandler.js +++ /dev/null @@ -1,75 +0,0 @@ -'use strict'; - -var benchmark = require('benchmark'); -var async = require('async'); -var sinon = require('sinon'); -var bitcore = require('bitcore-lib'); -var Block = bitcore.Block; -var AddressService = require('../lib/services/address'); -var maxTime = 20; - -var blockData1 = require('./data/block-367238.json'); -var blockData2 = require('./data/block-367239.json'); -var blockData3 = require('./data/block-367240.json'); - -console.log('Address Service Block Handler'); -console.log('-----------------------------'); - -async.series([ - function(next) { - - var c = 0; - var blocks = [ - Block.fromBuffer(new Buffer(blockData1, 'hex')), - Block.fromBuffer(new Buffer(blockData2, 'hex')), - Block.fromBuffer(new Buffer(blockData3, 'hex')) - ]; - var blocksLength = 3; - var node = { - services: { - bitcoind : { - on: sinon.stub() - } - } - }; - var addressService = new AddressService({node: node}); - - function blockHandler(deffered) { - if (c >= blocksLength) { - c = 0; - } - var block = blocks[c]; - addressService.blockHandler(block, true, function(err, operations) { - if (err) { - throw err; - } - deffered.resolve(); - }); - c++; - } - - var suite = new benchmark.Suite(); - - suite.add('blockHandler', blockHandler, { - defer: true, - maxTime: maxTime - }); - - suite - .on('cycle', function(event) { - console.log(String(event.target)); - }) - .on('complete', function() { - console.log('Fastest is ' + this.filter('fastest').pluck('name')); - console.log('----------------------------------------------------------------------'); - next(); - }) - .run(); - } -], function(err) { - if (err) { - throw err; - } - console.log('Finished'); - process.exit(); -}); diff --git a/bin/build b/bin/build deleted file mode 100755 index 8c0fbe64..00000000 --- a/bin/build +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -options=`cat ${root_dir}/bin/config_options.sh` -host=$(${root_dir}/bin/variables.sh host) || exit -1 -depends_dir=$($root_dir/bin/variables.sh depends_dir) -btc_dir="${root_dir}/libbitcoind" -sys=$($root_dir/bin/variables.sh sys) -patch_sha=$($root_dir/bin/variables.sh patch_sha) -config_lib_dir=$($root_dir/bin/variables.sh config_lib_dir) -export CPPFLAGS="-I${depends_dir}/${host}/include/boost -I${depends_dir}/${host}/include -L${depends_dir}/${host}/lib" -echo "Using BTC directory: ${btc_dir}" - -cd "${root_dir}" || exit -1 - -build_dependencies () { - if [ -d "${btc_dir}" ]; then - pushd "${depends_dir}" || exit -1 - echo "using host for dependencies: ${host}" - if [ "${test}" = true ]; then - make HOST=${host} NO_QT=1 NO_UPNP=1 - else - make HOST=${host} NO_QT=1 NO_WALLET=1 NO_UPNP=1 - fi - if test $? -eq 0; then - popd || exit -1 - else - echo "Bitcoin's dependency building failed, please check the previous output for details." - exit -1 - fi - fi -} - -get_patch_file () { - if test -e "${root_dir/PATCH_VERSION}"; then - tag=`cat "${root_dir}/PATCH_VERSION" | xargs` || exit -1 - else - echo "no tag file found, please create it in the root of the project as so: 'echo \"v0.10.2\" > PATCH_VERSION'" - exit 1 - fi -} - -compare_patch () { - cd "${btc_dir}" || exit -1 - get_patch_file - echo "running the diff command from HEAD to ${tag}" - last_commit=$(git rev-parse HEAD) - diff=$(git show ${last_commit}) - stripped_diff=$( echo -n "${diff}" | tail -n $( expr `echo -n "${diff}" | wc -l` - 5 ) ) - matching_patch=`echo -n "${stripped_diff}" | diff -w "${root_dir}/etc/bitcoin.patch" -` -} - -cache_files () { - cache_file="${root_dir}"/cache/cache.tar - pushd "${btc_dir}" || exit -1 - find src depends/${host} -type f \( -name "*.h" -or -name "*.hpp" -or -name \ -"*.ipp" -or -name "*.a" \) | tar -cf "${cache_file}" -T - - if test $? -ne 0; then - echo "We were trying to copy over your cached artifacts, but there was an issue." - exit -1 - fi - tar xf "${cache_file}" -C "${root_dir}"/cache - if test $? -ne 0; then - echo "We were trying to untar your cache, but there was an issue." - exit -1 - fi - rm -fr "${cache_file}" >/dev/null 2>&1 - popd || exit -1 -} - -debug= -if [ "${BITCORENODE_ENV}" == "debug" ]; then - options=`cat ${root_dir}/bin/config_options_debug.sh` || exit -1 -fi - -test=false -if [ "${BITCORENODE_ENV}" == "test" ]; then - test=true - options=`cat ${root_dir}/bin/config_options_test.sh` || exit -1 -fi - -if hash shasum 2>/dev/null; then - shasum_cmd="shasum -a 256" -else - shasum_cmd="sha256sum" -fi - -patch_file_sha=$(${shasum_cmd} "${root_dir}/etc/bitcoin.patch" | awk '{print $1}') -last_patch_file_sha= -if [ -e "${patch_sha}" ]; then - echo "Patch file sha exists, let's see if the patch has changed since last build..." - last_patch_file_sha=$(cat "${patch_sha}") -fi -shared_file_built=false -if [ "${last_patch_file_sha}" == "${patch_file_sha}" ]; then - echo "Patch file contents matches the sha from the patch file itself, so no reason to rebuild the bindings unless there are no prebuilt bindings." - shared_file_built=true -fi - -if [ "${shared_file_built}" = false ]; then - echo "Looks like the patch to bitcoin changed since last build -or- this is the first build, so rebuilding libbitcoind itself..." - mac_response=$($root_dir/bin/variables.sh mac_dependencies) - if [ "${mac_response}" != "" ]; then - echo "${mac_response}" - exit -1 - fi - only_make=false - if [ -d "${btc_dir}" ]; then - echo "running compare patch..." - compare_patch - repatch=false - if [[ "${matching_patch}" =~ [^\s\\] ]]; then - echo "Warning! libbitcoind is not patched with:\ - ${root_dir}/etc/bitcoin.patch." - echo -n "Would you like to remove the current patch, checkout the tag: ${tag} and \ -apply the current patch from "${root_dir}"/etc/bitcoin.patch? (y/N): " - if [ "${BITCORENODE_ASSUME_YES}" = true ]; then - input=y - echo "" - else - read input - fi - if [[ "${input}" =~ ^y|^Y ]]; then - repatch=true - echo "Removing directory: \"${btc_dir}\" and starting over!" - rm -fr "${btc_dir}" >/dev/null 2>&1 - fi - fi - if [ "${repatch}" = false ]; then - echo "Running make inside libbitcoind (assuming you've previously patched and configured libbitcoind)..." - cd "${btc_dir}" || exit -1 - only_make=true - fi - fi - - if [ "${only_make}" = false ]; then - echo "Cloning, patching, and building libbitcoind..." - get_patch_file - echo "attempting to checkout tag: ${tag} of bitcoin from github..." - cd "${root_dir}" || exit -1 - #versions of git prior to 2.x will not clone correctly with --branch - git clone --depth 1 https://github.com/bitcoin/bitcoin.git libbitcoind - cd "${btc_dir}" || exit -1 - git fetch --tags - git checkout "${tag}" - echo '../patch-bitcoin.sh' "${btc_dir}" - ../bin/patch-bitcoin "${btc_dir}" - - if ! test -d .git; then - echo 'Please point this script to an upstream bitcoin git repo.' - exit -1 - fi - - fi - build_dependencies - echo './autogen.sh' - ./autogen.sh || exit -1 - - config_host="--host ${host}" - full_options="${options} ${config_host} ${config_lib_dir}" - echo "running the configure script with the following options:\n :::[\"${full_options}\"]:::" - ${full_options} - - echo 'make V=1' - make V=1 || exit -1 - - echo "Creating the sha marker for the patching in libbitcoind..." - echo "Writing patch sha file to: \"${patch_sha}\"" - echo -n `${shasum_cmd} "${root_dir}"/etc/bitcoin.patch | awk '{print $1}'` > "${patch_sha}" - cache_files - echo 'Build finished successfully.' -else - echo 'Using existing static library.' -fi - -# Building the Bindings - -set -e - -cd "${root_dir}" - -debug=--debug=false -if test x"$1" = x'debug'; then - debug=--debug -fi - -echo "running::: 'node-gyp ${sys} ${debug} rebuild'" -node-gyp ${sys} ${debug} rebuild diff --git a/bin/clean b/bin/clean deleted file mode 100755 index 414df579..00000000 --- a/bin/clean +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -cd "${root_dir}" -pushd "${root_dir}"/libbitcoind -make clean -popd -node-gyp clean -rm -fr cache/* diff --git a/bin/config_options.sh b/bin/config_options.sh deleted file mode 100644 index d51d99a0..00000000 --- a/bin/config_options.sh +++ /dev/null @@ -1,2 +0,0 @@ -./configure --enable-tests=no --enable-daemonlib --with-gui=no --without-qt --without-miniupnpc --without-bdb --disable-wallet --without-utils - diff --git a/bin/config_options_debug.sh b/bin/config_options_debug.sh deleted file mode 100644 index fb5845e0..00000000 --- a/bin/config_options_debug.sh +++ /dev/null @@ -1,2 +0,0 @@ -./configure --enable-debug --enable-tests=no --enable-daemonlib --with-gui=no --without-qt --without-miniupnpc --without-bdb --disable-wallet --without-utils - diff --git a/bin/config_options_test.sh b/bin/config_options_test.sh deleted file mode 100644 index 3ee83701..00000000 --- a/bin/config_options_test.sh +++ /dev/null @@ -1,2 +0,0 @@ -./configure --enable-debug --enable-tests=no --enable-daemonlib --with-gui=no --without-qt --without-miniupnpc - diff --git a/bin/get-tarball-name.js b/bin/get-tarball-name.js deleted file mode 100644 index c7ea0ffb..00000000 --- a/bin/get-tarball-name.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -var execSync = require('child_process').execSync; - -function getTarballName() { - var packageRoot = __dirname + '/..'; - var version = require(packageRoot + '/package.json').version; - var platform = process.platform; - var arch = execSync(packageRoot + '/bin/variables.sh arch').toString(); - var abi = process.versions.modules; - var tarballName = 'libbitcoind-' + version + '-node' + abi + '-' + platform + '-' + arch + '.tgz'; - return tarballName; -} - -if (require.main === module) { - process.stdout.write(getTarballName()); -} - -module.exports = getTarballName; diff --git a/bin/install b/bin/install deleted file mode 100755 index c6e3a916..00000000 --- a/bin/install +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." - -cd "${root_dir}" - -tarball_name=`node bin/get-tarball-name.js` -bucket_name="bitcore-node" -binary_url="https://${bucket_name}.s3.amazonaws.com/${tarball_name}" - -echo "Downloading binary: ${binary_url}" - -is_curl=true -if hash curl 2>/dev/null; then - curl --fail -I $binary_url >/dev/null 2>&1 -else - is_curl=false - wget --server-response --spider $binary_url >/dev/null 2>&1 -fi - -if test $? -eq 0; then - if [ "${is_curl}" = true ]; then - curl $binary_url > $tarball_name - else - wget $binary_url - fi - if test -e "${tarball_name}"; then - echo "Unpacking binary distribution" - tar -xvzf $tarball_name - if test $? -eq 0; then - exit 0 - fi - fi -fi -echo "Prebuild binary could not be downloaded, building from source..." -./bin/build diff --git a/bin/package.js b/bin/package.js deleted file mode 100644 index c03639bd..00000000 --- a/bin/package.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -var exec = require('child_process').exec; -var bindings = require('bindings'); -var index = require('../lib'); -var log = index.log; - -var packageRoot = bindings.getRoot(bindings.getFileName()); -var binaryPath = bindings({ - path: true, - bindings: 'bitcoind.node' -}); -var relativeBinaryPath = binaryPath.replace(packageRoot + '/', ''); -var tarballName = require('./get-tarball-name')(); - -log.info('Signing binding binary: "' + binaryPath + '"'); - -var signCommand = 'gpg --yes --out ' + binaryPath + '.sig --detach-sig ' + binaryPath; - -var signchild = exec(signCommand, function(error, stdout, stderr) { - if (error) { - throw error; - } - - if (stdout) { - log.info('GPG:', stdout); - } - - if (stderr) { - log.error(stderr); - } - - log.info('Packaging tarball: "' + tarballName + '"'); - - // Create a tarball of both the binding and the signature - var tarCommand = 'tar -C ' + - packageRoot + ' -cvzf ' + - tarballName + ' ' + - relativeBinaryPath + ' ' + - relativeBinaryPath + '.sig'; - - var tarchild = exec(tarCommand, function (error, stdout, stderr) { - - if (error) { - throw error; - } - - if (stdout) { - log.info('Tar:', stdout); - } - - if (stderr) { - log.error(stderr); - } - - }); - -}); diff --git a/bin/patch-bitcoin b/bin/patch-bitcoin deleted file mode 100755 index aeaab6b1..00000000 --- a/bin/patch-bitcoin +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -#root_dir="$(readlink -f "$(dirname "$0")")/.." -cd "$root_dir" -dir=$(test -n "$1" && echo "$1" || echo "${HOME}/bitcoin") -patch_file="$(pwd)/etc/bitcoin.patch" - -cd "$dir" || exit 1 - -if ! test -d .git; then - echo 'Please point this script to an upstream bitcoin git repo.' - exit 1 -fi - -if test $? -ne 0; then - echo 'Unable to checkout necessary commit.' - echo 'Please pull the latest HEAD from the upstream bitcoin repo.' - exit 1 -fi -git checkout -b "libbitcoind-$(date '+%Y.%m.%d')" || exit 1 - -patch -p1 < "$patch_file" || exit 1 - -git add --all || exit 1 - -[ -n "$( git config user.name )" ] \ - || git config user.name 'Bitcore Build' - -[ -n "$( git config user.email )" ] \ - || git config user.email "$( id -n -u )@$( hostname -f )" - -git commit -a -m 'allow compiling of libbitcoind.so.' || exit 1 - -echo 'Patch completed successfully.' -exit 0 diff --git a/bin/start-libbitcoind.js b/bin/start-libbitcoind.js deleted file mode 100644 index e2f1c637..00000000 --- a/bin/start-libbitcoind.js +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env node - -'use strict'; - -var index = require('..'); -var log = index.log; - -process.title = 'libbitcoind'; - -/** - * daemon - */ -var daemon = require('../').services.Bitcoin({ - node: { - datadir: process.env.BITCORENODE_DIR || process.env.HOME + '/.bitcoin', - network: { - name: process.env.BITCORENODE_NETWORK || 'livenet' - } - } -}); - -daemon.start(function() { - log.info('ready'); -}); - -daemon.on('error', function(err) { - log.info('error="%s"', err.message); -}); - -daemon.on('open', function(status) { - log.info('status="%s"', status); -}); - -function exitHandler(options, err) { - log.info('Stopping daemon'); - if (err) { - log.error('uncaught exception:', err); - if(err.stack) { - console.log(err.stack); - } - process.exit(-1); - } - if (options.sigint) { - daemon.stop(function(err) { - if(err) { - log.error('Failed to stop services: ' + err); - return process.exit(1); - } - - log.info('Halted'); - process.exit(0); - }); - } -} - -//catches uncaught exceptions - - -process.on('uncaughtException', exitHandler.bind(null, {exit:true})); -//catches ctrl+c event -process.on('SIGINT', exitHandler.bind(null, {sigint:true})); diff --git a/bin/start.js b/bin/start.js deleted file mode 100755 index 6fda5aff..00000000 --- a/bin/start.js +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env node - -'use strict'; - -var start = require('../lib/scaffold/start'); -var defaultConfig = require('../lib/scaffold/default-config'); - -start(defaultConfig()); diff --git a/bin/upload.js b/bin/upload.js deleted file mode 100644 index 4376bb20..00000000 --- a/bin/upload.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var AWS = require('aws-sdk'); -var bindings = require('bindings'); -var index = require('../lib'); -var log = index.log; - -var config = require(process.env.HOME + '/.bitcore-node-upload.json'); - -AWS.config.region = config.region; -AWS.config.update({ - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey -}); - -var packageRoot = bindings.getRoot(bindings.getFileName()); -var tarballName = require('./get-tarball-name')(); -var bucketName = 'bitcore-node'; -var url = 'https://' + bucketName + '.s3.amazonaws.com/' + tarballName; -var localPath = packageRoot + '/' + tarballName; - -log.info('Uploading package: ' + localPath); - -var fileStream = fs.createReadStream(localPath); - -fileStream.on('error', function(err) { - if (err) { - throw err; - } -}); - -fileStream.on('open', function() { - - var s3 = new AWS.S3(); - - var params = { - ACL: 'public-read', - Key: tarballName, - Body: fileStream, - Bucket: bucketName - }; - - s3.putObject(params, function(err, data) { - if (err) { - throw err; - } else { - log.info('Successfully uploaded to: ' + url); - } - }); - -}); diff --git a/bin/variables.sh b/bin/variables.sh deleted file mode 100755 index 0988e0fa..00000000 --- a/bin/variables.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash - -exec 2> /dev/null -root_dir="$(cd "$(dirname $0)" && pwd)/.." -if [ "${root_dir}" == "" ]; then - root_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." -fi -bitcoin_dir="${root_dir}"/libbitcoind -cache_dir="${root_dir}"/cache - -get_host_and_platform () { - platform=`uname -a | awk '{print tolower($1)}'` - arch=`uname -m` - if [ "${arch:0:3}" == "arm" ]; then - platform="linux-gnueabihf" - arch="arm" - fi - if [ -n "${CXX}" ] && [ -n "${CC}" ]; then - cc_target=$("${CC}" -v 2>&1 | awk '/Target:/ {print $2}') - cxx_target=$("${CXX}" -v 2>&1 | awk '/Target:/ {print $2}') - IFS='-' read -ra SYS <<< "${cc_target}" - if [ "${SYS[0]}" != "${arch}" ]; then - if [ -n "${SYS[1]}" ] && [ -n "${SYS[2]}" ] && hash "${CXX}" && hash "${CC}" && [ -n "${cc_target}" ] && [ -n "${cxx_target}" ]; then - #try and see if we've got a cross compiler, if not then auto detect - arch="${SYS[0]}" - platform="${SYS[1]}"-"${SYS[2]}" - else - error_message="You've specified a cross compiler, but we could not compute the host-platform-triplet for cross compilation. Please set CC and CXX environment variables with host-platform-triplet-*. Also ensure the cross compiler exists on your system and is available on your path. Example: CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++" - return_error_message - fi - fi - fi -} - -return_error_message () { - echo "${error_message}" - exit -1 -} - -get_host_and_platform -host="${arch}"-"${platform}" - -mac_response= -check_mac_build_system () { - if [ "${platform}" == "darwin" ]; then - if [ ! -e "/usr/include/stdlib.h" ]; then - if hash xcode-select 2>/dev/null; then - mac_response="Please run 'xcode-select --install' from the command line because it seems that you've got Xcode, but not the Xcode command line tools that are required for compiling this project from source..." - else - mac_response="please use the App Store to install Xcode and Xcode command line tools. After Xcode is installed, please run: 'xcode-select --install' from the command line" - fi - fi - fi -} - -depends_dir="${bitcoin_dir}"/depends -thread="${cache_dir}"/depends/"${host}"/lib/libboost_thread-mt.a -filesystem="${cache_dir}"/depends/"${host}"/lib/libboost_filesystem-mt.a -chrono="${cache_dir}"/depends/"${host}"/lib/libboost_chrono-mt.a -program_options="${cache_dir}"/depends/"${host}"/lib/libboost_program_options-mt.a -system="${cache_dir}"/depends/"${host}"/lib/libboost_system-mt.a -leveldb="${cache_dir}"/src/leveldb/libleveldb.a -memenv="${cache_dir}"/src/leveldb/libmemenv.a -libsecp256k1="${cache_dir}"/src/secp256k1/.libs/libsecp256k1.a -ssl="${cache_dir}"/depends/"${host}"/lib/libssl.a -crypto="${cache_dir}"/depends/"${host}"/lib/libcrypto.a - -config_lib_dir= -if [ "${platform}" == "darwin" ]; then - config_lib_dir="--with-boost-libdir=${depends_dir}/${host}/lib" -else - config_lib_dir="--prefix=${depends_dir}/${host}" -fi - -if test x"$1" = x'anl'; then - if [ "${platform}" != "darwin" ]; then - echo -n "-lanl" - fi -fi - -if test x"$1" = x'cache_dir'; then - echo -n "${cache_dir}" -fi - -if test x"$1" = x'btcdir'; then - echo -n "${bitcoin_dir}" -fi - -if test -z "$1" -o x"$1" = x'thread'; then - echo -n "${thread}" -fi - -if test -z "$1" -o x"$1" = x'filesystem'; then - echo -n "${filesystem}" -fi - -if test -z "$1" -o x"$1" = x'program_options'; then - echo -n "${program_options}" -fi - -if test -z "$1" -o x"$1" = x'system'; then - echo -n "${system}" -fi - -if test -z "$1" -o x"$1" = x'ssl'; then - echo -n "${ssl}" -fi - -if test -z "$1" -o x"$1" = x'crypto'; then - echo -n "${crypto}" -fi - -if test -z "$1" -o x"$1" = x'chrono'; then - echo -n "${chrono}" -fi - -if test -z "$1" -o x"$1" = x'depends_dir'; then - echo -n "${depends_dir}" -fi - -if test -z "$1" -o x"$1" = x'leveldb'; then - echo -n "${leveldb}" -fi - -if test -z "$1" -o x"$1" = x'memenv'; then - echo -n "${memenv}" -fi - -if test -z "$1" -o x"$1" = x'libsecp256k1'; then - echo -n "${libsecp256k1}" -fi - -if test -z "$1" -o x"$1" = x'host'; then - echo -n "${host}" -fi - -if test -z "$1" -o x"$1" = x'arch'; then - echo -n "${arch}" -fi - -if test -z "$1" -o x"$1" = x'bdb'; then - if [ "${BITCORENODE_ENV}" == "test" ]; then - echo -n "${cache_dir}"/depends/"${host}"/lib/libdb_cxx.a - fi -fi - -if test -z "$1" -o x"$1" = x'patch_sha'; then - echo -n "${root_dir}"/cache/patch_sha.txt -fi - -if test -z "$1" -o x"$1" = x'load_archive'; then - if [ "${os}" == "osx" ]; then - echo -n "-Wl,-all_load -Wl,--no-undefined" - else - echo -n "-Wl,--whole-archive ${filesystem} ${thread} "${cache_dir}"/src/.libs/libbitcoind.a -Wl,--no-whole-archive" - fi -fi - -if test -z "$1" -o x"$1" = x'mac_dependencies'; then - check_mac_build_system - echo -n "${mac_response}" -fi - -if test -z "$1" -o x"$1" = x'wallet_enabled'; then - if [ "${BITCORENODE_ENV}" == "test" ]; then - echo -n "-DENABLE_WALLET" - fi -fi - -if test -z "$1" -o x"$1" = x'sys'; then - if [ -n "${SYS}" ]; then - echo -n "--arch=${SYS[0]}" - fi -fi - -if test -z "$1" -o x"$1" = x'bitcoind'; then - echo -n "${cache_dir}"/src/.libs/libbitcoind.a -fi - -if test -z "$1" -o x"$1" = x'config_lib_dir'; then - echo -n "${config_lib_dir}" -fi diff --git a/binding.gyp b/binding.gyp deleted file mode 100644 index 3db1adb1..00000000 --- a/binding.gyp +++ /dev/null @@ -1,59 +0,0 @@ -{ - "targets": [ - { - "target_name": "libbitcoind", - "include_dirs" : [ - "\"$($(package)_cxxflags) $($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$(boost_archiver_$(host_os))\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam - endef - -diff --git a/src/Makefile.am b/src/Makefile.am -index 2461f82..e7e9ecf 100644 ---- a/src/Makefile.am -+++ b/src/Makefile.am -@@ -1,6 +1,10 @@ - DIST_SUBDIRS = secp256k1 - AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) - -+noinst_LTLIBRARIES = -+libbitcoind_la_LIBADD = -+libbitcoind_la_LDFLAGS = -no-undefined -+STATIC_EXTRA_LIBS = $(LIBLEVELDB) $(LIBMEMENV) - - if EMBEDDED_LEVELDB - LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include -@@ -49,16 +53,16 @@ BITCOIN_INCLUDES += $(BDB_CPPFLAGS) - EXTRA_LIBRARIES += libbitcoin_wallet.a - endif - --if BUILD_BITCOIN_LIBS --lib_LTLIBRARIES = libbitcoinconsensus.la --LIBBITCOIN_CONSENSUS=libbitcoinconsensus.la --else --LIBBITCOIN_CONSENSUS= --endif -- -+LIBBITCOIN_CONSENSUS = - bin_PROGRAMS = - TESTS = - -+if BUILD_BITCOIN_LIBS -+noinst_LTLIBRARIES += libbitcoinconsensus.la -+LIBBITCOIN_CONSENSUS += libbitcoinconsensus.la -+endif -+ -+if !ENABLE_DAEMONLIB - if BUILD_BITCOIND - bin_PROGRAMS += bitcoind - endif -@@ -66,6 +70,9 @@ endif - if BUILD_BITCOIN_UTILS - bin_PROGRAMS += bitcoin-cli bitcoin-tx - endif -+else -+noinst_LTLIBRARIES += libbitcoind.la -+endif - - .PHONY: FORCE - # bitcoin core # -@@ -170,8 +177,11 @@ obj/build.h: FORCE - @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh $(abs_top_builddir)/src/obj/build.h \ - $(abs_top_srcdir) --libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h - -+ARCH_PLATFORM = $(shell ../../bin/variables.sh host) -+ -+libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h -+clientversion.cpp: obj/build.h - # server: shared between bitcoind and bitcoin-qt - libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) - libbitcoin_server_a_SOURCES = \ -@@ -310,9 +320,18 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h - bitcoind_SOURCES = bitcoind.cpp - bitcoind_CPPFLAGS = $(BITCOIN_INCLUDES) - bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -+libbitcoind_la_SOURCES = bitcoind.cpp -+libbitcoind_la_SOURCES += $(libbitcoin_util_a_SOURCES) -+libbitcoind_la_SOURCES += $(libbitcoin_univalue_a_SOURCES) -+libbitcoind_la_SOURCES += $(libbitcoin_crypto_a_SOURCES) -+libbitcoind_la_SOURCES += $(libbitcoin_common_a_SOURCES) -+libbitcoind_la_SOURCES += $(libbitcoin_server_a_SOURCES) -+libbitcoind_la_SOURCES += $(crypto_libbitcoin_crypto_a_SOURCES) -+libbitcoind_la_SOURCES += $(univalue_libbitcoin_univalue_a_SOURCES) - - if TARGET_WINDOWS - bitcoind_SOURCES += bitcoind-res.rc -+libbitcoind_la_SOURCES += bitcoind-res.rc - endif - - bitcoind_LDADD = \ -@@ -327,10 +346,17 @@ bitcoind_LDADD = \ - - if ENABLE_WALLET - bitcoind_LDADD += libbitcoin_wallet.a -+libbitcoind_la_SOURCES += $(libbitcoin_wallet_a_SOURCES) - endif - - bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) --# -+libbitcoind_la_LIBADD += $(SSL_LIBS) $(LIBSECP256K1) $(CRYPTO_LIBS) $(STATIC_EXTRA_LIBS) -+libbitcoind_la_CPPFLAGS = $(BITCOIN_INCLUDES) -+if TARGET_DARWIN -+libbitcoind_la_LDFLAGS += -Wl,-all_load -+else -+libbitcoind_la_LDFLAGS += -Wl,--whole-archive $(STATIC_EXTRA_LIBS) -Wl,--no-whole-archive -+endif - - # bitcoin-cli binary # - bitcoin_cli_SOURCES = bitcoin-cli.cpp -diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp -index 6e2758a..0352a9d 100644 ---- a/src/bitcoind.cpp -+++ b/src/bitcoind.cpp -@@ -33,6 +33,10 @@ - - static bool fDaemon; - -+#if ENABLE_DAEMONLIB -+extern void WaitForShutdown(boost::thread_group* threadGroup); -+#endif -+ - void WaitForShutdown(boost::thread_group* threadGroup) - { - bool fShutdown = ShutdownRequested(); -@@ -166,6 +170,7 @@ bool AppInit(int argc, char* argv[]) - return fRet; - } - -+#if !ENABLE_DAEMONLIB - int main(int argc, char* argv[]) - { - SetupEnvironment(); -@@ -175,3 +180,4 @@ int main(int argc, char* argv[]) - - return (AppInit(argc, argv) ? 0 : 1); - } -+#endif -diff --git a/src/init.cpp b/src/init.cpp -index a04e4e0..33d0bc7 100644 ---- a/src/init.cpp -+++ b/src/init.cpp -@@ -638,21 +638,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) - umask(077); - } - -- // Clean shutdown on SIGTERM -- struct sigaction sa; -- sa.sa_handler = HandleSIGTERM; -- sigemptyset(&sa.sa_mask); -- sa.sa_flags = 0; -- sigaction(SIGTERM, &sa, NULL); -- sigaction(SIGINT, &sa, NULL); -- -- // Reopen debug.log on SIGHUP -- struct sigaction sa_hup; -- sa_hup.sa_handler = HandleSIGHUP; -- sigemptyset(&sa_hup.sa_mask); -- sa_hup.sa_flags = 0; -- sigaction(SIGHUP, &sa_hup, NULL); -- - #if defined (__SVR4) && defined (__sun) - // ignore SIGPIPE on Solaris - signal(SIGPIPE, SIG_IGN); -diff --git a/src/init.h b/src/init.h -index dcb2b29..5ce68ba 100644 ---- a/src/init.h -+++ b/src/init.h -@@ -18,6 +18,11 @@ class thread_group; - - extern CWallet* pwalletMain; - -+#if ENABLE_DAEMONLIB -+#include -+#include -+#endif -+ - void StartShutdown(); - bool ShutdownRequested(); - void Shutdown(); -diff --git a/src/main.cpp b/src/main.cpp -index fe072ec..9f677cf 100644 ---- a/src/main.cpp -+++ b/src/main.cpp -@@ -1105,6 +1105,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa - - // Store transaction in memory - pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); -+ GetNodeSignals().TxToMemPool(tx); - } - - SyncWithWallets(tx, NULL); -diff --git a/src/net.cpp b/src/net.cpp -index e4b22f9..33fe6f9 100644 ---- a/src/net.cpp -+++ b/src/net.cpp -@@ -432,8 +432,10 @@ void CNode::PushVersion() - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), addrYou.ToString(), id); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), id); -+ std::vector bitcore; -+ bitcore.push_back("bitcore"); //the dash character is removed from the comments section - PushMessage("version", PROTOCOL_VERSION, nLocalServices, nTime, addrYou, addrMe, -- nLocalHostNonce, FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, std::vector()), nBestHeight, true); -+ nLocalHostNonce, FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, bitcore), nBestHeight, true); - } - - -diff --git a/src/net.h b/src/net.h -index 17502b9..c9ae1b2 100644 ---- a/src/net.h -+++ b/src/net.h -@@ -99,6 +99,8 @@ struct CNodeSignals - { - boost::signals2::signal GetHeight; - boost::signals2::signal ProcessMessages; -+ boost::signals2::signal TxToMemPool; -+ boost::signals2::signal TxLeaveMemPool; - boost::signals2::signal SendMessages; - boost::signals2::signal InitializeNode; - boost::signals2::signal FinalizeNode; -diff --git a/src/txmempool.cpp b/src/txmempool.cpp -index c3d1b60..03e265d 100644 ---- a/src/txmempool.cpp -+++ b/src/txmempool.cpp -@@ -133,6 +133,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem - if (!mapTx.count(hash)) - continue; - const CTransaction& tx = mapTx[hash].GetTx(); -+ GetNodeSignals().TxLeaveMemPool(tx); - if (fRecursive) { - for (unsigned int i = 0; i < tx.vout.size(); i++) { - std::map::iterator it = mapNextTx.find(COutPoint(hash, i)); diff --git a/example/client.js b/example/client.js deleted file mode 100644 index ded2b61e..00000000 --- a/example/client.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict'; - -var socket = require('socket.io-client')('http://localhost:3000'); -socket.on('connect', function(){ - console.log('connected'); -}); - -socket.on('disconnect', function(){ - console.log('disconnected'); -}); - -var message = { - method: 'getOutputs', - params: ['2NChMRHVCxTPq9KeyvHQUSbfLaQY55Zzzp8', true] -}; - -socket.send(message, function(response) { - if(response.error) { - console.log('Error', response.error); - return; - } - - console.log(response.result); -}); - -var message2 = { - method: 'getTransaction', - params: ['4f793f67fc7465f14fa3a8d3727fa7d133cdb2f298234548b94a5f08b6f4103e', true] -}; - -socket.send(message2, function(response) { - if(response.error) { - console.log('Error', response.error); - return; - } - - console.log(response.result); -}); - -socket.on('transaction', function(obj) { - console.log(JSON.stringify(obj, null, 2)); -}); - -socket.on('address/transaction', function(obj) { - console.log(JSON.stringify(obj, null, 2)); -}); - -socket.emit('subscribe', 'transaction'); -socket.emit('subscribe', 'address/transaction', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); \ No newline at end of file diff --git a/index.js b/index.js index 4b2aa56a..0a210849 100644 --- a/index.js +++ b/index.js @@ -1,26 +1,13 @@ 'use strict'; -var semver = require('semver'); -var packageData = require('./package.json'); - -function nodeVersionCheck(version, expected) { - if (!semver.satisfies(version, expected)) { - throw new Error('Node.js version ' + version + ' is expected to be ' + expected); - } -} -nodeVersionCheck(process.versions.node, packageData.engines.node); - module.exports = require('./lib'); -module.exports.nodeVersionCheck = nodeVersionCheck; module.exports.Node = require('./lib/node'); module.exports.Transaction = require('./lib/transaction'); module.exports.Service = require('./lib/service'); module.exports.errors = require('./lib/errors'); module.exports.services = {}; -module.exports.services.Address = require('./lib/services/address'); module.exports.services.Bitcoin = require('./lib/services/bitcoind'); -module.exports.services.DB = require('./lib/services/db'); module.exports.services.Web = require('./lib/services/web'); module.exports.scaffold = {}; diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js deleted file mode 100644 index 3653e9cb..00000000 --- a/lib/services/address/constants.js +++ /dev/null @@ -1,56 +0,0 @@ -'use strict'; - -var exports = {}; - -exports.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height - SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height - SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output -}; - -exports.MEMPREFIXES = { - OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address - SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address - SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output -}; - -// To save space, we're only storing the PubKeyHash or ScriptHash in our index. -// To avoid intentional unspendable collisions, which have been seen on the blockchain, -// we must store the hash type (PK or Script) as well. -exports.HASH_TYPES = { - PUBKEY: new Buffer('01', 'hex'), - REDEEMSCRIPT: new Buffer('02', 'hex') -}; - -// Translates from our enum type back into the hash types returned by -// bitcore-lib/address. -exports.HASH_TYPES_READABLE = { - '01': 'pubkeyhash', - '02': 'scripthash' -}; - -exports.HASH_TYPES_MAP = { - 'pubkeyhash': exports.HASH_TYPES.PUBKEY, - 'scripthash': exports.HASH_TYPES.REDEEMSCRIPT -}; - -exports.SPACER_MIN = new Buffer('00', 'hex'); -exports.SPACER_MAX = new Buffer('ff', 'hex'); -exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex'); -exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); -exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); -exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); - -// The maximum number of inputs that can be queried at once -exports.MAX_INPUTS_QUERY_LENGTH = 50000; -// The maximum number of outputs that can be queried at once -exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; -// The maximum number of transactions that can be queried at once -exports.MAX_HISTORY_QUERY_LENGTH = 100; -// The maximum number of addresses that can be queried at once -exports.MAX_ADDRESSES_QUERY = 10000; -// The maximum number of simultaneous requests -exports.MAX_ADDRESSES_LIMIT = 5; - -module.exports = exports; - diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js deleted file mode 100644 index 8ebce51c..00000000 --- a/lib/services/address/encoding.js +++ /dev/null @@ -1,307 +0,0 @@ -'use strict'; - -var bitcore = require('bitcore-lib'); -var BufferReader = bitcore.encoding.BufferReader; -var Address = bitcore.Address; -var PublicKey = bitcore.PublicKey; -var constants = require('./constants'); -var $ = bitcore.util.preconditions; - -var exports = {}; - -exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - txidBuffer, - outputIndexBuffer - ]); - return key.toString('binary'); -}; - -exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) { - var key = Buffer.concat([ - hashBuffer, - hashTypeBuffer, - ]); - return key.toString('binary'); -}; - - -exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - constants.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - heightBuffer, - txidBuffer, - outputIndexBuffer - ]); - return key; -}; - -exports.decodeOutputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var txid = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - txid: txid, - outputIndex: outputIndex - }; -}; - -exports.encodeOutputValue = function(satoshis, scriptBuffer) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([satoshisBuffer, scriptBuffer]); -}; - -exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]); -}; - -exports.decodeOutputValue = function(buffer) { - var satoshis = buffer.readDoubleBE(0); - var scriptBuffer = buffer.slice(8, buffer.length); - return { - satoshis: satoshis, - scriptBuffer: scriptBuffer - }; -}; - -exports.decodeOutputMempoolValue = function(buffer) { - var satoshis = buffer.readDoubleBE(0); - var timestamp = buffer.readDoubleBE(8); - var scriptBuffer = buffer.slice(16, buffer.length); - return { - satoshis: satoshis, - timestamp: timestamp, - scriptBuffer: scriptBuffer - }; -}; - -exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - constants.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - heightBuffer, - prevTxIdBuffer, - outputIndexBuffer - ]); -}; - -exports.decodeInputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var prevTxId = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - prevTxId: prevTxId, - outputIndex: outputIndex - }; -}; - -exports.encodeInputValue = function(txidBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - txidBuffer, - inputIndexBuffer - ]); -}; - -exports.decodeInputValue = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - txid: txid, - inputIndex: inputIndex - }; -}; - -exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - constants.PREFIXES.SPENTSMAP, - outputTxIdBuffer, - outputIndexBuffer - ]); -}; - -exports.decodeInputKeyMap = function(buffer) { - var txid = buffer.slice(1, 33); - var outputIndex = buffer.readUInt32BE(33); - return { - outputTxId: txid, - outputIndex: outputIndex - }; -}; - -exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - inputTxIdBuffer, - inputIndexBuffer - ]); -}; - -exports.decodeInputValueMap = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - inputTxId: txid, - inputIndex: inputIndex - }; -}; - -exports.encodeSummaryCacheKey = function(address) { - return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_MAP[address.type]]); -}; - -exports.decodeSummaryCacheKey = function(buffer, network) { - var hashBuffer = buffer.read(20); - var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')]; - var address = new Address({ - hashBuffer: hashBuffer, - type: type, - network: network - }); - return address; -}; - -exports.encodeSummaryCacheValue = function(cache, tipHeight, tipHash) { - var tipHashBuffer = new Buffer(tipHash, 'hex'); - var buffer = new Buffer(new Array(20)); - buffer.writeUInt32BE(tipHeight); - buffer.writeDoubleBE(cache.result.totalReceived, 4); - buffer.writeDoubleBE(cache.result.balance, 12); - var txidBuffers = []; - for (var i = 0; i < cache.result.txids.length; i++) { - var buf = new Buffer(new Array(36)); - var txid = cache.result.txids[i]; - buf.write(txid, 'hex'); - buf.writeUInt32BE(cache.result.appearanceIds[txid], 32); - txidBuffers.push(buf); - } - var txidsBuffer = Buffer.concat(txidBuffers); - var value = Buffer.concat([tipHashBuffer, buffer, txidsBuffer]); - - return value; -}; - -exports.decodeSummaryCacheValue = function(buffer) { - - var hash = buffer.slice(0, 32).toString('hex'); - var height = buffer.readUInt32BE(32); - var totalReceived = buffer.readDoubleBE(36); - var balance = buffer.readDoubleBE(44); - - // read 32 byte chunks until exhausted - var appearanceIds = {}; - var txids = []; - var pos = 52; - while(pos < buffer.length) { - var txid = buffer.slice(pos, pos + 32).toString('hex'); - var txidHeight = buffer.readUInt32BE(pos + 32); - txids.push(txid); - appearanceIds[txid] = txidHeight; - pos += 36; - } - - var cache = { - height: height, - hash: hash, - result: { - appearanceIds: appearanceIds, - txids: txids, - totalReceived: totalReceived, - balance: balance, - unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache - unconfirmedBalance: 0 - } - }; - - return cache; -}; - -exports.getAddressInfo = function(addressStr) { - var addrObj = bitcore.Address(addressStr); - var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; - - return { - hashBuffer: addrObj.hashBuffer, - hashTypeBuffer: hashTypeBuffer, - hashTypeReadable: addrObj.type - }; -}; - -/** - * This function is optimized to return address information about an output script - * without constructing a Bitcore Address instance. - * @param {Script} - An instance of a Bitcore Script - * @param {Network|String} - The network for the address - */ -exports.extractAddressInfoFromScript = function(script, network) { - $.checkArgument(network, 'Second argument is expected to be a network'); - var hashBuffer; - var addressType; - var hashTypeBuffer; - if (script.isPublicKeyHashOut()) { - hashBuffer = script.chunks[2].buf; - hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - addressType = Address.PayToPublicKeyHash; - } else if (script.isScriptHashOut()) { - hashBuffer = script.chunks[1].buf; - hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; - addressType = Address.PayToScriptHash; - } else if (script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - var address = Address.fromPublicKey(new PublicKey(pubkey), network); - hashBuffer = address.hashBuffer; - hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - // pay-to-publickey doesn't have an address, however for compatibility - // purposes, we can create an address - addressType = Address.PayToPublicKeyHash; - } else { - return false; - } - return { - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - addressType: addressType - }; -}; - -module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js deleted file mode 100644 index 88b5c96b..00000000 --- a/lib/services/address/history.js +++ /dev/null @@ -1,266 +0,0 @@ -'use strict'; - -var bitcore = require('bitcore-lib'); -var async = require('async'); -var _ = bitcore.deps._; - -var constants = require('./constants'); - -/** - * This represents an instance that keeps track of data over a series of - * asynchronous I/O calls to get the transaction history for a group of - * addresses. History can be queried by start and end block heights to limit large sets - * of results (uses leveldb key streaming). - */ -function AddressHistory(args) { - this.node = args.node; - this.options = args.options; - - if(Array.isArray(args.addresses)) { - this.addresses = args.addresses; - } else { - this.addresses = [args.addresses]; - } - - this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; - this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; - this.maxAddressesLimit = args.options.maxAddressesLimit || constants.MAX_ADDRESSES_LIMIT; - - this.addressStrings = []; - for (var i = 0; i < this.addresses.length; i++) { - var address = this.addresses[i]; - if (address instanceof bitcore.Address) { - this.addressStrings.push(address.toString()); - } else if (_.isString(address)) { - this.addressStrings.push(address); - } else { - throw new TypeError('Addresses are expected to be strings'); - } - } - - this.detailedArray = []; -} - -AddressHistory.prototype._mergeAndSortTxids = function(summaries) { - var appearanceIds = {}; - var unconfirmedAppearanceIds = {}; - - for (var i = 0; i < summaries.length; i++) { - var summary = summaries[i]; - for (var key in summary.appearanceIds) { - appearanceIds[key] = summary.appearanceIds[key]; - delete summary.appearanceIds[key]; - } - for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { - unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey]; - delete summary.unconfirmedAppearanceIds[key]; - } - } - var confirmedTxids = Object.keys(appearanceIds); - confirmedTxids.sort(function(a, b) { - // Confirmed are sorted by height - return appearanceIds[a] - appearanceIds[b]; - }); - var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); - unconfirmedTxids.sort(function(a, b) { - // Unconfirmed are sorted by timestamp - return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; - }); - return confirmedTxids.concat(unconfirmedTxids); -}; - -/** - * This function will give detailed history for the configured - * addresses. See AddressService.prototype.getAddressHistory - * for complete documentation about options and response format. - */ -AddressHistory.prototype.get = function(callback) { - var self = this; - if (this.addresses.length > this.maxAddressesQuery) { - return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); - } - - var opts = _.clone(this.options); - opts.noBalance = true; - - if (this.addresses.length === 1) { - var address = this.addresses[0]; - self.node.services.address.getAddressSummary(address, opts, function(err, summary) { - if (err) { - return callback(err); - } - return self._paginateWithDetails.call(self, summary.txids, callback); - }); - } else { - - opts.fullTxList = true; - async.mapLimit( - self.addresses, - self.maxAddressesLimit, - function(address, next) { - self.node.services.address.getAddressSummary(address, opts, next); - }, - function(err, summaries) { - if (err) { - return callback(err); - } - var txids = self._mergeAndSortTxids(summaries); - return self._paginateWithDetails.call(self, txids, callback); - } - ); - } - -}; - -AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { - var self = this; - var totalCount = allTxids.length; - - // Slice the page starting with the most recent - var txids; - if (self.options.from >= 0 && self.options.to >= 0) { - var fromOffset = Math.max(0, totalCount - self.options.from); - var toOffset = Math.max(0, totalCount - self.options.to); - txids = allTxids.slice(toOffset, fromOffset); - } else { - txids = allTxids; - } - - // Verify that this query isn't too long - if (txids.length > self.maxHistoryQueryLength) { - return callback(new Error( - 'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' + - self.addresses.join(',') - )); - } - - // Reverse to include most recent at the top - txids.reverse(); - - async.eachSeries( - txids, - function(txid, next) { - self.getDetailedInfo(txid, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); - -}; - -/** - * This function will transform items from the combinedArray into - * the detailedArray with the full transaction, satoshis and confirmation. - * @param {Object} txInfo - An item from the `combinedArray` - * @param {Function} next - */ -AddressHistory.prototype.getDetailedInfo = function(txid, next) { - var self = this; - var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; - - self.node.services.db.getTransactionWithBlockInfo( - txid, - queryMempool, - function(err, transaction) { - if (err) { - return next(err); - } - - transaction.populateInputs(self.node.services.db, [], function(err) { - if (err) { - return next(err); - } - - var addressDetails = self.getAddressDetailsForTransaction(transaction); - - self.detailedArray.push({ - addresses: addressDetails.addresses, - satoshis: addressDetails.satoshis, - height: transaction.__height, - confirmations: self.getConfirmationsDetail(transaction), - timestamp: transaction.__timestamp, - // TODO bitcore-lib should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - tx: transaction - }); - - next(); - }); - } - ); -}; - -/** - * A helper function for `getDetailedInfo` for getting the confirmations. - * @param {Transaction} transaction - A transaction with a populated __height value. - */ -AddressHistory.prototype.getConfirmationsDetail = function(transaction) { - var confirmations = 0; - if (transaction.__height >= 0) { - confirmations = this.node.services.db.tip.__height - transaction.__height + 1; - } - return confirmations; -}; - -AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) { - var result = { - addresses: {}, - satoshis: 0 - }; - - for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { - var input = transaction.inputs[inputIndex]; - if (!input.script) { - continue; - } - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { - var inputAddressString = inputAddress.toString(); - if (this.addressStrings.indexOf(inputAddressString) >= 0) { - if (!result.addresses[inputAddressString]) { - result.addresses[inputAddressString] = { - inputIndexes: [inputIndex], - outputIndexes: [] - }; - } else { - result.addresses[inputAddressString].inputIndexes.push(inputIndex); - } - result.satoshis -= input.output.satoshis; - } - } - } - - for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { - var output = transaction.outputs[outputIndex]; - if (!output.script) { - continue; - } - var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress) { - var outputAddressString = outputAddress.toString(); - if (this.addressStrings.indexOf(outputAddressString) >= 0) { - if (!result.addresses[outputAddressString]) { - result.addresses[outputAddressString] = { - inputIndexes: [], - outputIndexes: [outputIndex] - }; - } else { - result.addresses[outputAddressString].outputIndexes.push(outputIndex); - } - result.satoshis += output.satoshis; - } - } - } - - return result; - -}; - -module.exports = AddressHistory; diff --git a/lib/services/address/index.js b/lib/services/address/index.js deleted file mode 100644 index eef4ed07..00000000 --- a/lib/services/address/index.js +++ /dev/null @@ -1,1611 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var BaseService = require('../../service'); -var inherits = require('util').inherits; -var async = require('async'); -var mkdirp = require('mkdirp'); -var index = require('../../'); -var log = index.log; -var errors = index.errors; -var bitcore = require('bitcore-lib'); -var Networks = bitcore.Networks; -var levelup = require('levelup'); -var leveldown = require('leveldown'); -var memdown = require('memdown'); -var $ = bitcore.util.preconditions; -var _ = bitcore.deps._; -var Hash = bitcore.crypto.Hash; -var EventEmitter = require('events').EventEmitter; -var Address = bitcore.Address; -var AddressHistory = require('./history'); -var constants = require('./constants'); -var encoding = require('./encoding'); -var InputsTransformStream = require('./streams/inputs-transform'); -var OutputsTransformStream = require('./streams/outputs-transform'); - - -/** - * The Address Service builds upon the Database Service and the Bitcoin Service to add additional - * functionality for getting information by base58check encoded addresses. This includes getting the - * balance for an address, the history for a collection of addresses, and unspent outputs for - * constructing transactions. This is typically the core functionality for building a wallet. - * @param {Object} options - * @param {Node} options.node - An instance of the node - * @param {String} options.name - An optional name of the service - */ -var AddressService = function(options) { - BaseService.call(this, options); - - this.subscriptions = {}; - this.subscriptions['address/transaction'] = {}; - this.subscriptions['address/balance'] = {}; - - this._bitcoindTransactionListener = this.transactionHandler.bind(this); - this._bitcoindTransactionLeaveListener = this.transactionLeaveHandler.bind(this); - this.node.services.bitcoind.on('tx', this._bitcoindTransactionListener); - this.node.services.bitcoind.on('txleave', this._bitcoindTransactionLeaveListener); - - this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; - this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; - - this._setMempoolIndexPath(); - if (options.mempoolMemoryIndex) { - this.levelupStore = memdown; - } else { - this.levelupStore = leveldown; - } - this.mempoolIndex = null; // Used for larger mempool indexes - this.mempoolSpentIndex = {}; // Used for small quick synchronous lookups - this.mempoolAddressIndex = {}; // Used to check if an address is on the spend pool -}; - -inherits(AddressService, BaseService); - -AddressService.dependencies = [ - 'bitcoind', - 'db' -]; - -AddressService.prototype.start = function(callback) { - var self = this; - - async.series([ - - function(next) { - // Flush any existing mempool index - if (fs.existsSync(self.mempoolIndexPath)) { - leveldown.destroy(self.mempoolIndexPath, next); - } else { - setImmediate(next); - } - }, - function(next) { - // Setup new mempool index - if (!fs.existsSync(self.mempoolIndexPath)) { - mkdirp(self.mempoolIndexPath, next); - } else { - setImmediate(next); - } - }, - function(next) { - self.mempoolIndex = levelup( - self.mempoolIndexPath, - { - db: self.levelupStore, - keyEncoding: 'binary', - valueEncoding: 'binary', - fillCache: false, - maxOpenFiles: 200 - }, - next - ); - } - ], callback); - -}; - -AddressService.prototype.stop = function(callback) { - // TODO Keep track of ongoing db requests before shutting down - this.node.services.bitcoind.removeListener('tx', this._bitcoindTransactionListener); - this.node.services.bitcoind.removeListener('txleave', this._bitcoindTransactionLeaveListener); - this.mempoolIndex.close(callback); -}; - -/** - * This function will set `this.mempoolIndexPath` based on `this.node.network`. - * @private - */ -AddressService.prototype._setMempoolIndexPath = function() { - this.mempoolIndexPath = this._getDBPathFor('bitcore-addressmempool.db'); -}; - -AddressService.prototype._getDBPathFor = function(dbname) { - $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); - var path; - if (this.node.network === Networks.livenet) { - path = this.node.datadir + '/' + dbname; - } else if (this.node.network === Networks.testnet) { - if (this.node.network.regtestEnabled) { - path = this.node.datadir + '/regtest/' + dbname; - } else { - path = this.node.datadir + '/testnet3/' + dbname; - } - } else { - throw new Error('Unknown network: ' + this.network); - } - return path; -}; - -/** - * Called by the Node to get the available API methods for this service, - * that can be exposed over the JSON-RPC interface. - */ -AddressService.prototype.getAPIMethods = function() { - return [ - ['getBalance', this, this.getBalance, 2], - ['getOutputs', this, this.getOutputs, 2], - ['getUnspentOutputs', this, this.getUnspentOutputs, 2], - ['getInputForOutput', this, this.getInputForOutput, 2], - ['isSpent', this, this.isSpent, 2], - ['getAddressHistory', this, this.getAddressHistory, 2], - ['getAddressSummary', this, this.getAddressSummary, 1] - ]; -}; - -/** - * Called by the Bus to get the available events for this service. - */ -AddressService.prototype.getPublishEvents = function() { - return [ - { - name: 'address/transaction', - scope: this, - subscribe: this.subscribe.bind(this, 'address/transaction'), - unsubscribe: this.unsubscribe.bind(this, 'address/transaction') - }, - { - name: 'address/balance', - scope: this, - subscribe: this.subscribe.bind(this, 'address/balance'), - unsubscribe: this.unsubscribe.bind(this, 'address/balance') - } - ]; -}; - -/** - * Will process each output of a transaction from the daemon "tx" event, and construct - * an object with the data for the message to be relayed to any subscribers for an address. - * - * @param {Object} messages - An object to collect messages - * @param {Transaction} tx - Instance of the transaction - * @param {Number} outputIndex - The index of the output in the transaction - * @param {Boolean} rejected - If the transaction was rejected by the mempool - */ -AddressService.prototype.transactionOutputHandler = function(messages, tx, outputIndex, rejected) { - var script = tx.outputs[outputIndex].script; - - // If the script is invalid skip - if (!script) { - return; - } - - var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); - if (!addressInfo) { - return; - } - - addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); - - // Collect data to publish to address subscribers - if (messages[addressInfo.hashHex]) { - messages[addressInfo.hashHex].outputIndexes.push(outputIndex); - } else { - messages[addressInfo.hashHex] = { - tx: tx, - outputIndexes: [outputIndex], - addressInfo: addressInfo, - rejected: rejected - }; - } -}; - -/** - * This will handle data from the daemon "txleave" that a transaction has left the mempool. - * @param {Object} txInfo - The data from the daemon.on('txleave') event - * @param {Buffer} txInfo.buffer - The transaction buffer - * @param {String} txInfo.hash - The hash of the transaction - */ -AddressService.prototype.transactionLeaveHandler = function(txInfo) { - var tx = bitcore.Transaction().fromBuffer(txInfo.buffer); - this.updateMempoolIndex(tx, false); -}; - -/** - * This will handle data from the daemon "tx" event, go through each of the outputs - * and send messages by calling `transactionEventHandler` to any subscribers for a - * particular address. - * @param {Object} txInfo - The data from the daemon.on('tx') event - * @param {Buffer} txInfo.buffer - The transaction buffer - * @param {Boolean} txInfo.mempool - If the transaction was accepted in the mempool - * @param {String} txInfo.hash - The hash of the transaction - * @param {Function} [callback] - Optional callback - */ -AddressService.prototype.transactionHandler = function(txInfo, callback) { - var self = this; - - if (!callback) { - callback = function(err) { - if (err) { - return log.error(err); - } - }; - } - - if (this.node.stopping) { - return callback(); - } - - // Basic transaction format is handled by the daemon - // and we can safely assume the buffer is properly formatted. - var tx = bitcore.Transaction().fromBuffer(txInfo.buffer); - - var messages = {}; - - var outputsLength = tx.outputs.length; - for (var i = 0; i < outputsLength; i++) { - this.transactionOutputHandler(messages, tx, i, !txInfo.mempool); - } - - function finish(err) { - if (err) { - return callback(err); - } - for (var key in messages) { - self.transactionEventHandler(messages[key]); - self.balanceEventHandler(null, messages[key].addressInfo); - } - callback(); - } - - if (txInfo.mempool) { - self.updateMempoolIndex(tx, true, finish); - } else { - setImmediate(finish); - } - -}; - -AddressService.prototype._updateAddressIndex = function(key, add) { - var currentValue = this.mempoolAddressIndex[key] || 0; - - if(add) { - if (currentValue > 0) { - this.mempoolAddressIndex[key] = currentValue + 1; - } else { - this.mempoolAddressIndex[key] = 1; - } - } else { - if (currentValue <= 1) { - delete this.mempoolAddressIndex[key]; - } else { - this.mempoolAddressIndex[key]--; - } - } -}; - - -/** - * This function will update the mempool address index with the necessary - * information for further lookups. - * @param {Transaction} - An instance of a Bitcore Transaction - * @param {Boolean} - Add/remove from the index - */ -AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { - /* jshint maxstatements: 100 */ - - var operations = []; - var timestampBuffer = new Buffer(new Array(8)); - timestampBuffer.writeDoubleBE(new Date().getTime()); - - var action = 'put'; - if (!add) { - action = 'del'; - } - - var txid = tx.hash; - var txidBuffer = new Buffer(txid, 'hex'); - - var outputLength = tx.outputs.length; - for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { - var output = tx.outputs[outputIndex]; - if (!output.script) { - continue; - } - var addressInfo = encoding.extractAddressInfoFromScript(output.script, this.node.network); - if (!addressInfo) { - continue; - } - - var addressIndexKey = encoding.encodeMempoolAddressIndexKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer); - - this._updateAddressIndex(addressIndexKey, add); - - // Update output index - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - - var outKey = Buffer.concat([ - constants.MEMPREFIXES.OUTPUTS, - addressInfo.hashBuffer, - addressInfo.hashTypeBuffer, - txidBuffer, - outputIndexBuffer - ]); - - var outValue = encoding.encodeOutputMempoolValue( - output.satoshis, - timestampBuffer, - output._scriptBuffer - ); - - operations.push({ - type: action, - key: outKey, - value: outValue - }); - - } - var inputLength = tx.inputs.length; - for (var inputIndex = 0; inputIndex < inputLength; inputIndex++) { - - var input = tx.inputs[inputIndex]; - - var inputOutputIndexBuffer = new Buffer(4); - inputOutputIndexBuffer.writeUInt32BE(input.outputIndex); - - // Add an additional small spent index for fast synchronous lookups - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - input.prevTxId, - input.outputIndex - ); - if (add) { - this.mempoolSpentIndex[spentIndexSyncKey] = true; - } else { - delete this.mempoolSpentIndex[spentIndexSyncKey]; - } - - // Add a more detailed spent index with values - var spentIndexKey = Buffer.concat([ - constants.MEMPREFIXES.SPENTSMAP, - input.prevTxId, - inputOutputIndexBuffer - ]); - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - var inputIndexValue = Buffer.concat([ - txidBuffer, - inputIndexBuffer - ]); - operations.push({ - type: action, - key: spentIndexKey, - value: inputIndexValue - }); - - // Update input index - var inputHashBuffer; - var inputHashType; - if (input.script.isPublicKeyHashIn()) { - inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = constants.HASH_TYPES.PUBKEY; - } else if (input.script.isScriptHashIn()) { - inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; - } else { - continue; - } - var inputKey = Buffer.concat([ - constants.MEMPREFIXES.SPENTS, - inputHashBuffer, - inputHashType, - input.prevTxId, - inputOutputIndexBuffer - ]); - var inputValue = Buffer.concat([ - txidBuffer, - inputIndexBuffer, - timestampBuffer - ]); - operations.push({ - type: action, - key: inputKey, - value: inputValue - }); - - var addressIndexKey = encoding.encodeMempoolAddressIndexKey(inputHashBuffer, inputHashType); - - this._updateAddressIndex(addressIndexKey, add); - } - - if (!callback) { - callback = function(err) { - if (err) { - return log.error(err); - } - }; - } - - this.mempoolIndex.batch(operations, callback); -}; - -/** - * The Database Service will run this function when blocks are connected and - * disconnected to the chain during syncing and reorganizations. - * @param {Block} block - An instance of a Bitcore Block - * @param {Boolean} addOutput - If the block is being removed or added to the chain - * @param {Function} callback - */ -AddressService.prototype.blockHandler = function(block, addOutput, callback) { - var txs = block.transactions; - var height = block.__height; - - var action = 'put'; - if (!addOutput) { - action = 'del'; - } - - var operations = []; - - var transactionLength = txs.length; - for (var i = 0; i < transactionLength; i++) { - - var tx = txs[i]; - var txid = tx.id; - var txidBuffer = new Buffer(txid, 'hex'); - var inputs = tx.inputs; - var outputs = tx.outputs; - - // Subscription messages - var txmessages = {}; - - var outputLength = outputs.length; - for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { - var output = outputs[outputIndex]; - - var script = output.script; - - if(!script) { - log.debug('Invalid script'); - continue; - } - - var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); - if (!addressInfo) { - continue; - } - - // We need to use the height for indexes (and not the timestamp) because the - // the timestamp has unreliable sequential ordering. The next block - // can have a time that is previous to the previous block (however not - // less than the mean of the 11 previous blocks) and not greater than 2 - // hours in the future. - var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); - var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); - operations.push({ - type: action, - key: key, - value: value - }); - - addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); - - // Collect data for subscribers - if (txmessages[addressInfo.hashHex]) { - txmessages[addressInfo.hashHex].outputIndexes.push(outputIndex); - } else { - txmessages[addressInfo.hashHex] = { - tx: tx, - height: height, - outputIndexes: [outputIndex], - addressInfo: addressInfo, - timestamp: block.header.timestamp - }; - } - - this.balanceEventHandler(block, addressInfo); - - } - - // Publish events to any subscribers for this transaction - for (var addressKey in txmessages) { - this.transactionEventHandler(txmessages[addressKey]); - } - - if(tx.isCoinbase()) { - continue; - } - - for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { - - var input = inputs[inputIndex]; - var inputHash; - var inputHashType; - - if (input.script.isPublicKeyHashIn()) { - inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = constants.HASH_TYPES.PUBKEY; - } else if (input.script.isScriptHashIn()) { - inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; - } else { - continue; - } - - var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); - - // To be able to query inputs by address and spent height - var inputKey = encoding.encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); - var inputValue = encoding.encodeInputValue(txidBuffer, inputIndex); - - operations.push({ - type: action, - key: inputKey, - value: inputValue - }); - - // To be able to search for an input spending an output - var inputKeyMap = encoding.encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); - var inputValueMap = encoding.encodeInputValueMap(txidBuffer, inputIndex); - - operations.push({ - type: action, - key: inputKeyMap, - value: inputValueMap - }); - - } - } - - setImmediate(function() { - callback(null, operations); - }); -}; - -/** - * This function is responsible for emitting events to any subscribers to the - * `address/transaction` event. - * @param {Object} obj - * @param {Transaction} obj.tx - The transaction - * @param {Object} obj.addressInfo - * @param {String} obj.addressInfo.hashHex - The hex string of address hash for the subscription - * @param {String} obj.addressInfo.hashBuffer - The address hash buffer - * @param {String} obj.addressInfo.addressType - The address type - * @param {Array} obj.outputIndexes - Indexes of the inputs that includes the address - * @param {Array} obj.inputIndexes - Indexes of the outputs that includes the address - * @param {Date} obj.timestamp - The time of the block the transaction was included - * @param {Number} obj.height - The height of the block the transaction was included - * @param {Boolean} obj.rejected - If the transaction was not accepted in the mempool - */ -AddressService.prototype.transactionEventHandler = function(obj) { - if(this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { - var emitters = this.subscriptions['address/transaction'][obj.addressInfo.hashHex]; - var address = new Address({ - hashBuffer: obj.addressInfo.hashBuffer, - network: this.node.network, - type: obj.addressInfo.addressType - }); - for(var i = 0; i < emitters.length; i++) { - emitters[i].emit('address/transaction', { - rejected: obj.rejected, - height: obj.height, - timestamp: obj.timestamp, - inputIndexes: obj.inputIndexes, - outputIndexes: obj.outputIndexes, - address: address, - tx: obj.tx - }); - } - } -}; - -/** - * The function is responsible for emitting events to any subscribers for the - * `address/balance` event. - * @param {Block} block - * @param {Object} obj - * @param {String} obj.hashHex - * @param {Buffer} obj.hashBuffer - * @param {String} obj.addressType - */ -AddressService.prototype.balanceEventHandler = function(block, obj) { - if(this.subscriptions['address/balance'][obj.hashHex]) { - var emitters = this.subscriptions['address/balance'][obj.hashHex]; - var address = new Address({ - hashBuffer: obj.hashBuffer, - network: this.node.network, - type: obj.addressType - }); - this.getBalance(address, true, function(err, balance) { - if(err) { - return this.emit(err); - } - for(var i = 0; i < emitters.length; i++) { - emitters[i].emit('address/balance', address, balance, block); - } - }); - } -}; - -/** - * The Bus will use this function to subscribe to the available - * events for this service. For information about the available events - * please see `getPublishEvents`. - * @param {String} name - The name of the event - * @param {EventEmitter} emitter - An event emitter instance - * @param {Array} addresses - An array of addresses to subscribe - */ -AddressService.prototype.subscribe = function(name, emitter, addresses) { - $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - $.checkArgument(Array.isArray(addresses), 'Second argument is expected to be an Array of addresses'); - - for(var i = 0; i < addresses.length; i++) { - var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(!this.subscriptions[name][hashHex]) { - this.subscriptions[name][hashHex] = []; - } - this.subscriptions[name][hashHex].push(emitter); - } -}; - -/** - * The Bus will use this function to unsubscribe to the available - * events for this service. - * @param {String} name - The name of the event - * @param {EventEmitter} emitter - An event emitter instance - * @param {Array} addresses - An array of addresses to subscribe - */ -AddressService.prototype.unsubscribe = function(name, emitter, addresses) { - $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - $.checkArgument(Array.isArray(addresses) || _.isUndefined(addresses), 'Second argument is expected to be an Array of addresses or undefined'); - - if(!addresses) { - return this.unsubscribeAll(name, emitter); - } - - for(var i = 0; i < addresses.length; i++) { - var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(this.subscriptions[name][hashHex]) { - var emitters = this.subscriptions[name][hashHex]; - var index = emitters.indexOf(emitter); - if(index > -1) { - emitters.splice(index, 1); - } - } - } -}; - -/** - * A helper function for the `unsubscribe` method to unsubscribe from all addresses. - * @param {String} name - The name of the event - * @param {EventEmitter} emitter - An instance of an event emitter - */ -AddressService.prototype.unsubscribeAll = function(name, emitter) { - $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - - for(var hashHex in this.subscriptions[name]) { - var emitters = this.subscriptions[name][hashHex]; - var index = emitters.indexOf(emitter); - if(index > -1) { - emitters.splice(index, 1); - } - } -}; - -/** - * Will sum the total of all unspent outputs to calculate the balance - * for an address. - * @param {String} address - The base58check encoded address - * @param {Boolean} queryMempool - Include mempool in the results - * @param {Function} callback - */ -AddressService.prototype.getBalance = function(address, queryMempool, callback) { - this.getUnspentOutputs(address, queryMempool, function(err, outputs) { - if(err) { - return callback(err); - } - - var satoshis = outputs.map(function(output) { - return output.satoshis; - }); - - var sum = satoshis.reduce(function(a, b) { - return a + b; - }, 0); - - return callback(null, sum); - }); -}; - -/** - * Will give the input that spends an output if it exists with: - * inputTxId - The input txid hex string - * inputIndex - A number with the spending input index - * @param {String|Buffer} txid - The transaction hash with the output - * @param {Number} outputIndex - The output index in the transaction - * @param {Object} options - * @param {Object} options.queryMempool - Include mempool in results - * @param {Function} callback - */ -AddressService.prototype.getInputForOutput = function(txid, outputIndex, options, callback) { - $.checkArgument(_.isNumber(outputIndex)); - $.checkArgument(_.isObject(options)); - $.checkArgument(_.isFunction(callback)); - var self = this; - var txidBuffer; - if (Buffer.isBuffer(txid)) { - txidBuffer = txid; - } else { - txidBuffer = new Buffer(txid, 'hex'); - } - if (options.queryMempool) { - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); - if (this.mempoolSpentIndex[spentIndexSyncKey]) { - return this._getSpentMempool(txidBuffer, outputIndex, callback); - } - } - var key = encoding.encodeInputKeyMap(txidBuffer, outputIndex); - var dbOptions = { - valueEncoding: 'binary', - keyEncoding: 'binary' - }; - this.node.services.db.store.get(key, dbOptions, function(err, buffer) { - if (err instanceof levelup.errors.NotFoundError) { - return callback(null, false); - } else if (err) { - return callback(err); - } - var value = encoding.decodeInputValueMap(buffer); - callback(null, { - inputTxId: value.inputTxId.toString('hex'), - inputIndex: value.inputIndex - }); - }); -}; - -/** - * A streaming equivalent to `getInputs`, and returns a transform stream with data - * emitted in the same format as `getInputs`. - * - * @param {String} addressStr - The relevant address - * @param {Object} options - Additional options for query the outputs - * @param {Number} [options.start] - The relevant start block height - * @param {Number} [options.end] - The relevant end block height - * @param {Function} callback - */ -AddressService.prototype.createInputsStream = function(addressStr, options) { - var inputStream = new InputsTransformStream({ - address: new Address(addressStr, this.node.network), - tipHeight: this.node.services.db.tip.__height - }); - - var stream = this.createInputsDBStream(addressStr, options) - .on('error', function(err) { - // Forward the error - inputStream.emit('error', err); - inputStream.end(); - }).pipe(inputStream); - - return stream; - -}; - -AddressService.prototype.createInputsDBStream = function(addressStr, options) { - var stream; - var addrObj = encoding.getAddressInfo(addressStr); - var hashBuffer = addrObj.hashBuffer; - var hashTypeBuffer = addrObj.hashTypeBuffer; - - if (options.start >= 0 && options.end >= 0) { - - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end, 0); - - var startBuffer = new Buffer(4); - // Because the key has additional data following it, we don't have an ability - // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number - // to be one value larger to include it. - var adjustedStart = options.start + 1; - startBuffer.writeUInt32BE(adjustedStart, 0); - - stream = this.node.services.db.store.createReadStream({ - gt: Buffer.concat([ - constants.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - endBuffer - ]), - lt: Buffer.concat([ - constants.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), - lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } - - return stream; -}; - -/** - * Will give inputs that spend previous outputs for an address as an object with: - * address - The base58check encoded address - * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' - * txid - A string of the transaction hash - * outputIndex - A number of corresponding transaction input - * height - The height of the block the transaction was included, will be -1 for mempool transactions - * confirmations - The number of confirmations, will equal 0 for mempool transactions - * - * @param {String} addressStr - The relevant address - * @param {Object} options - Additional options for query the outputs - * @param {Number} [options.start] - The relevant start block height - * @param {Number} [options.end] - The relevant end block height - * @param {Boolean} [options.queryMempool] - Include the mempool in the results - * @param {Function} callback - */ -AddressService.prototype.getInputs = function(addressStr, options, callback) { - - var self = this; - - var inputs = []; - - var addrObj = encoding.getAddressInfo(addressStr); - var hashBuffer = addrObj.hashBuffer; - var hashTypeBuffer = addrObj.hashTypeBuffer; - - var stream = this.createInputsStream(addressStr, options); - - stream.on('data', function(input) { - inputs.push(input); - if (inputs.length > self.maxInputsQueryLength) { - log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); - error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - stream.end(); - } - }); - - var error; - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('finish', function() { - if (error) { - return callback(error); - } - - if(options.queryMempool) { - self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { - if (err) { - return callback(err); - } - inputs = inputs.concat(mempoolInputs); - callback(null, inputs); - }); - } else { - callback(null, inputs); - } - - }); - - return stream; - -}; - -AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, hashTypeBuffer, callback) { - var self = this; - var mempoolInputs = []; - - var stream = self.mempoolIndex.createReadStream({ - gte: Buffer.concat([ - constants.MEMPREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN - ]), - lte: Buffer.concat([ - constants.MEMPREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MAX - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - - stream.on('data', function(data) { - var txid = data.value.slice(0, 32); - var inputIndex = data.value.readUInt32BE(32); - var timestamp = data.value.readDoubleBE(36); - var input = { - address: addressStr, - hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], - txid: txid.toString('hex'), //TODO use a buffer - inputIndex: inputIndex, - timestamp: timestamp, - height: -1, - confirmations: 0 - }; - mempoolInputs.push(input); - }); - - var error; - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('close', function() { - if (error) { - return callback(error); - } - callback(null, mempoolInputs); - }); - -}; - -AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, callback) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var spentIndexKey = Buffer.concat([ - constants.MEMPREFIXES.SPENTSMAP, - txidBuffer, - outputIndexBuffer - ]); - - this.mempoolIndex.get( - spentIndexKey, - function(err, mempoolValue) { - if (err) { - return callback(err); - } - var inputTxId = mempoolValue.slice(0, 32); - var inputIndex = mempoolValue.readUInt32BE(32); - callback(null, { - inputTxId: inputTxId.toString('hex'), - inputIndex: inputIndex - }); - } - ); -}; - -AddressService.prototype.createOutputsStream = function(addressStr, options) { - var outputStream = new OutputsTransformStream({ - address: new Address(addressStr, this.node.network), - tipHeight: this.node.services.db.tip.__height - }); - - var stream = this.createOutputsDBStream(addressStr, options) - .on('error', function(err) { - // Forward the error - outputStream.emit('error', err); - outputStream.end(); - }) - .pipe(outputStream); - - return stream; - -}; - -AddressService.prototype.createOutputsDBStream = function(addressStr, options) { - - var addrObj = encoding.getAddressInfo(addressStr); - var hashBuffer = addrObj.hashBuffer; - var hashTypeBuffer = addrObj.hashTypeBuffer; - var stream; - - if (options.start >= 0 && options.end >= 0) { - - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end, 0); - - var startBuffer = new Buffer(4); - // Because the key has additional data following it, we don't have an ability - // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number - // to be one value larger to include it. - var startAdjusted = options.start + 1; - startBuffer.writeUInt32BE(startAdjusted, 0); - - stream = this.node.services.db.store.createReadStream({ - gt: Buffer.concat([ - constants.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - endBuffer - ]), - lt: Buffer.concat([ - constants.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), - lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } - - return stream; - -}; - -/** - * Will give outputs for an address as an object with: - * address - The base58check encoded address - * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' - * txid - A string of the transaction hash - * outputIndex - A number of corresponding transaction output - * height - The height of the block the transaction was included, will be -1 for mempool transactions - * satoshis - The satoshis value of the output - * script - The script of the output as a hex string - * confirmations - The number of confirmations, will equal 0 for mempool transactions - * - * @param {String} addressStr - The relevant address - * @param {Object} options - Additional options for query the outputs - * @param {Number} [options.start] - The relevant start block height - * @param {Number} [options.end] - The relevant end block height - * @param {Boolean} [options.queryMempool] - Include the mempool in the results - * @param {Function} callback - */ -AddressService.prototype.getOutputs = function(addressStr, options, callback) { - var self = this; - $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); - $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - - var addrObj = encoding.getAddressInfo(addressStr); - var hashBuffer = addrObj.hashBuffer; - var hashTypeBuffer = addrObj.hashTypeBuffer; - if (!hashTypeBuffer) { - return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); - } - - var outputs = []; - var stream = this.createOutputsStream(addressStr, options); - - stream.on('data', function(data) { - outputs.push(data); - if (outputs.length > self.maxOutputsQueryLength) { - log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); - error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - stream.end(); - } - }); - - var error; - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('finish', function() { - if (error) { - return callback(error); - } - - if(options.queryMempool) { - self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { - if (err) { - return callback(err); - } - outputs = outputs.concat(mempoolOutputs); - callback(null, outputs); - }); - } else { - callback(null, outputs); - } - }); - - return stream; - -}; - -AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, hashTypeBuffer, callback) { - var self = this; - var mempoolOutputs = []; - - var stream = self.mempoolIndex.createReadStream({ - gte: Buffer.concat([ - constants.MEMPREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MIN - ]), - lte: Buffer.concat([ - constants.MEMPREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - constants.SPACER_MAX - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - - stream.on('data', function(data) { - // Format of data: - // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 - var txid = data.key.slice(22, 54); - var outputIndex = data.key.readUInt32BE(54); - var value = encoding.decodeOutputMempoolValue(data.value); - var output = { - address: addressStr, - hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], - txid: txid.toString('hex'), //TODO use a buffer - outputIndex: outputIndex, - height: -1, - timestamp: value.timestamp, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: 0 - }; - mempoolOutputs.push(output); - }); - - var error; - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('close', function() { - if (error) { - return callback(error); - } - callback(null, mempoolOutputs); - }); - -}; - -/** - * Will give unspent outputs for an address or an array of addresses. - * @param {Array|String} addresses - An array of addresses - * @param {Boolean} queryMempool - Include or exclude the mempool - * @param {Function} callback - */ -AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, callback) { - var self = this; - - if(!Array.isArray(addresses)) { - addresses = [addresses]; - } - - var utxos = []; - - async.eachSeries(addresses, function(address, next) { - self.getUnspentOutputsForAddress(address, queryMempool, function(err, unspents) { - if(err && err instanceof errors.NoOutputs) { - return next(); - } else if(err) { - return next(err); - } - - utxos = utxos.concat(unspents); - next(); - }); - }, function(err) { - callback(err, utxos); - }); -}; - -/** - * Will give unspent outputs for an address. - * @param {String} address - An address in base58check encoding - * @param {Boolean} queryMempool - Include or exclude the mempool - * @param {Function} callback - */ -AddressService.prototype.getUnspentOutputsForAddress = function(address, queryMempool, callback) { - - var self = this; - - this.getOutputs(address, {queryMempool: queryMempool}, function(err, outputs) { - if (err) { - return callback(err); - } else if(!outputs.length) { - return callback(new errors.NoOutputs('Address ' + address + ' has no outputs'), []); - } - - var opts = { - queryMempool: queryMempool - }; - - var isUnspent = function(output, callback) { - self.isUnspent(output, opts, callback); - }; - - async.filter(outputs, isUnspent, function(results) { - callback(null, results); - }); - }); -}; - -/** - * Will give the inverse of isSpent - * @param {Object} output - * @param {Object} options - * @param {Boolean} options.queryMempool - Include mempool in results - * @param {Function} callback - */ -AddressService.prototype.isUnspent = function(output, options, callback) { - $.checkArgument(_.isFunction(callback)); - this.isSpent(output, options, function(spent) { - callback(!spent); - }); -}; - -/** - * Will determine if an output is spent. - * @param {Object} output - An output as returned from getOutputs - * @param {Object} options - * @param {Boolean} options.queryMempool - Include mempool in results - * @param {Function} callback - */ -AddressService.prototype.isSpent = function(output, options, callback) { - $.checkArgument(_.isFunction(callback)); - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var self = this; - var txid = output.prevTxId ? output.prevTxId.toString('hex') : output.txid; - var spent = self.node.services.bitcoind.isSpent(txid, output.outputIndex); - if (!spent && queryMempool) { - var txidBuffer = new Buffer(txid, 'hex'); - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); - spent = self.mempoolSpentIndex[spentIndexSyncKey] ? true : false; - } - setImmediate(function() { - // TODO error should be the first argument? - callback(spent); - }); -}; - - -/** - * This will give the history for many addresses limited by a range of block heights (to limit - * the database lookup times) and/or paginated to limit the results length. - * - * The response format will be: - * { - * totalCount: 12 // the total number of items there are between the two heights - * items: [ - * { - * addresses: { - * '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX': { - * inputIndexes: [], - * outputIndexes: [0] - * } - * }, - * satoshis: 100, - * height: 300000, - * confirmations: 1, - * timestamp: 1442337090 // in seconds - * fees: 1000 // in satoshis - * tx: - * } - * ] - * } - * @param {Array} addresses - An array of addresses - * @param {Object} options - The options to limit the query - * @param {Number} [options.from] - The pagination "from" index - * @param {Number} [options.to] - The pagination "to" index - * @param {Number} [options.start] - The beginning block height (e.g. 1500 the most recent block height). - * @param {Number} [options.end] - The ending block height (e.g. 0 the older block height, results are inclusive). - * @param {Boolean} [options.queryMempool] - Include the mempool in the query - * @param {Function} callback - */ -AddressService.prototype.getAddressHistory = function(addresses, options, callback) { - var history = new AddressHistory({ - node: this.node, - options: options, - addresses: addresses - }); - history.get(callback); -}; - -/** - * This will give an object with: - * balance - confirmed balance - * unconfirmedBalance - unconfirmed balance - * totalReceived - satoshis received - * totalSpent - satoshis spent - * appearances - number of transactions - * unconfirmedAppearances - number of unconfirmed transactions - * txids - list of txids (unless noTxList is set) - * - * @param {String} address - * @param {Object} options - * @param {Boolean} [options.noTxList] - if set, txid array will not be included - * @param {Function} callback - */ -AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { - var self = this; - - var startTime = new Date(); - var address = new Address(addressArg); - - if (_.isUndefined(options.queryMempool)) { - options.queryMempool = true; - } - - async.waterfall([ - function(next) { - self._getAddressConfirmedSummary(address, options, next); - }, - function(result, next) { - self._getAddressMempoolSummary(address, options, result, next); - }, - function(result, next) { - self._setAndSortTxidsFromAppearanceIds(result, next); - } - ], function(err, result) { - if (err) { - return callback(err); - } - - var summary = self._transformAddressSummaryFromResult(result, options); - - var timeDelta = new Date() - startTime; - if (timeDelta > 5000) { - var seconds = Math.round(timeDelta / 1000); - log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); - } - - callback(null, summary); - - }); - -}; - -AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { - var self = this; - var baseResult = { - appearanceIds: {}, - totalReceived: 0, - balance: 0, - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - }; - - async.waterfall([ - function(next) { - self._getAddressConfirmedInputsSummary(address, baseResult, options, next); - }, - function(result, next) { - self._getAddressConfirmedOutputsSummary(address, result, options, next); - } - ], callback); - -}; - -AddressService.prototype._getAddressConfirmedInputsSummary = function(address, result, options, callback) { - $.checkArgument(address instanceof Address); - var self = this; - var error = null; - var count = 0; - - var inputsStream = self.createInputsStream(address, options); - inputsStream.on('data', function(input) { - var txid = input.txid; - result.appearanceIds[txid] = input.height; - - count++; - - if (count > self.maxInputsQueryLength) { - log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); - error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); - inputsStream.end(); - } - - }); - - inputsStream.on('error', function(err) { - error = err; - }); - - inputsStream.on('end', function() { - if (error) { - return callback(error); - } - callback(null, result); - }); -}; - -AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { - $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); - - var self = this; - var count = 0; - - var outputStream = self.createOutputsStream(address, options); - - outputStream.on('data', function(output) { - - var txid = output.txid; - var outputIndex = output.outputIndex; - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = output.height; - - if(!options.noBalance) { - - // Bitcoind's isSpent only works for confirmed transactions - var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - - if(!spentDB) { - result.balance += output.satoshis; - } - - if(options.queryMempool) { - // Check to see if this output is spent in the mempool and if so - // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(txid, 'hex'), // TODO: get buffer directly - outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - if(spentMempool) { - result.unconfirmedBalance -= output.satoshis; - } - } - } - - count++; - - if (count > self.maxOutputsQueryLength) { - log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); - error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); - outputStream.end(); - } - - }); - - var error = null; - - outputStream.on('error', function(err) { - error = err; - }); - - outputStream.on('end', function() { - if (error) { - return callback(error); - } - callback(null, result); - }); - -}; - -AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, callback) { - result.txids = Object.keys(result.appearanceIds); - result.txids.sort(function(a, b) { - return result.appearanceIds[a] - result.appearanceIds[b]; - }); - result.unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); - result.unconfirmedTxids.sort(function(a, b) { - return result.unconfirmedAppearanceIds[a] - result.unconfirmedAppearanceIds[b]; - }); - callback(null, result); -}; - -AddressService.prototype._getAddressMempoolSummary = function(address, options, result, callback) { - var self = this; - - // Skip if the options do not want to include the mempool - if (!options.queryMempool) { - return callback(null, result); - } - - var addressStr = address.toString(); - var hashBuffer = address.hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - var addressIndexKey = encoding.encodeMempoolAddressIndexKey(hashBuffer, hashTypeBuffer); - - if(!this.mempoolAddressIndex[addressIndexKey]) { - return callback(null, result); - } - - async.waterfall([ - function(next) { - self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { - if (err) { - return next(err); - } - for(var i = 0; i < mempoolInputs.length; i++) { - var input = mempoolInputs[i]; - result.unconfirmedAppearanceIds[input.txid] = input.timestamp; - } - next(null, result); - }); - - }, function(result, next) { - self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { - if (err) { - return next(err); - } - for(var i = 0; i < mempoolOutputs.length; i++) { - var output = mempoolOutputs[i]; - - result.unconfirmedAppearanceIds[output.txid] = output.timestamp; - - if(!options.noBalance) { - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(output.txid, 'hex'), // TODO: get buffer directly - output.outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; - // Only add this to the balance if it's not spent in the mempool already - if(!spentMempool) { - result.unconfirmedBalance += output.satoshis; - } - } - } - next(null, result); - }); - } - ], callback); -}; - -AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { - - var confirmedTxids = result.txids; - var unconfirmedTxids = result.unconfirmedTxids; - - var summary = { - totalReceived: result.totalReceived, - totalSpent: result.totalReceived - result.balance, - balance: result.balance, - appearances: confirmedTxids.length, - unconfirmedBalance: result.unconfirmedBalance, - unconfirmedAppearances: unconfirmedTxids.length - }; - - if (options.fullTxList) { - summary.appearanceIds = result.appearanceIds; - summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; - } else if (!options.noTxList) { - summary.txids = confirmedTxids.concat(unconfirmedTxids); - } - - return summary; - -}; - -module.exports = AddressService; diff --git a/lib/services/address/streams/inputs-transform.js b/lib/services/address/streams/inputs-transform.js deleted file mode 100644 index 8b8f71d3..00000000 --- a/lib/services/address/streams/inputs-transform.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict'; - -var Transform = require('stream').Transform; -var inherits = require('util').inherits; -var bitcore = require('bitcore-lib'); -var encodingUtil = require('../encoding'); -var $ = bitcore.util.preconditions; - -function InputsTransformStream(options) { - $.checkArgument(options.address instanceof bitcore.Address); - Transform.call(this, { - objectMode: true - }); - this._address = options.address; - this._addressStr = this._address.toString(); - this._tipHeight = options.tipHeight; -} -inherits(InputsTransformStream, Transform); - -InputsTransformStream.prototype._transform = function(chunk, encoding, callback) { - var self = this; - - var key = encodingUtil.decodeInputKey(chunk.key); - var value = encodingUtil.decodeInputValue(chunk.value); - - var input = { - address: this._addressStr, - hashType: this._address.type, - txid: value.txid.toString('hex'), - inputIndex: value.inputIndex, - height: key.height, - confirmations: this._tipHeight - key.height + 1 - }; - - self.push(input); - callback(); - -}; - -module.exports = InputsTransformStream; diff --git a/lib/services/address/streams/outputs-transform.js b/lib/services/address/streams/outputs-transform.js deleted file mode 100644 index b9c8e8d3..00000000 --- a/lib/services/address/streams/outputs-transform.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict'; - -var Transform = require('stream').Transform; -var inherits = require('util').inherits; -var bitcore = require('bitcore-lib'); -var encodingUtil = require('../encoding'); -var $ = bitcore.util.preconditions; - -function OutputsTransformStream(options) { - Transform.call(this, { - objectMode: true - }); - $.checkArgument(options.address instanceof bitcore.Address); - this._address = options.address; - this._addressStr = this._address.toString(); - this._tipHeight = options.tipHeight; -} -inherits(OutputsTransformStream, Transform); - -OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) { - var self = this; - - var key = encodingUtil.decodeOutputKey(chunk.key); - var value = encodingUtil.decodeOutputValue(chunk.value); - - var output = { - address: this._addressStr, - hashType: this._address.type, - txid: key.txid.toString('hex'), //TODO use a buffer - outputIndex: key.outputIndex, - height: key.height, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: this._tipHeight - key.height + 1 - }; - - self.push(output); - callback(); - -}; - -module.exports = OutputsTransformStream; diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index edc2dad6..d74d93ea 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1,14 +1,23 @@ 'use strict'; var fs = require('fs'); +var path = require('path'); +var spawn = require('child_process').spawn; var util = require('util'); -var bindings = require('bindings')('bitcoind.node'); var mkdirp = require('mkdirp'); var bitcore = require('bitcore-lib'); +var Address = bitcore.Address; +var zmq = require('zmq'); +var async = require('async'); +var BitcoinRPC = require('bitcoind-rpc'); var $ = bitcore.util.preconditions; +var _ = bitcore.deps._; + var index = require('../'); var log = index.log; +var errors = index.errors; var Service = require('../service'); +var Transaction = require('../transaction'); /** * Provides an interface to native bindings to [Bitcoin Core](https://github.com/bitcoin/bitcoin) @@ -31,7 +40,28 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; -Bitcoin.DEFAULT_CONFIG = 'whitelist=127.0.0.1\n' + 'txindex=1\n'; +Bitcoin.DEFAULT_CONFIG = 'whitelist=127.0.0.1\n' + 'txindex=1\n' + 'addressindex=1\n' + 'server=1\n'; + +/** + * Called by Node to determine the available API methods. + */ +Bitcoin.prototype.getAPIMethods = function() { + var methods = [ + ['getBlock', this, this.getBlock, 1], + ['getBlockHeader', this, this.getBlockHeader, 1], + ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], + ['getTransaction', this, this.getTransaction, 2], + ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], + ['sendTransaction', this, this.sendTransaction, 1], + ['estimateFee', this, this.estimateFee, 1], + ['getAddressTxids', this, this.getAddressTxids, 2], + ['getAddressBalance', this, this.getAddressBalance, 2], + ['getAddressUnspentOutputs', this, this.getAddressUnspentOutputs, 2], + ['getAddressHistory', this, this.getAddressHistory, 2], + ['getAddressSummary', this, this.getAddressSummary, 1] + ]; + return methods; +}; Bitcoin.prototype._loadConfiguration = function() { /* jshint maxstatements: 25 */ @@ -44,16 +74,6 @@ Bitcoin.prototype._loadConfiguration = function() { mkdirp.sync(this.node.datadir); } - if (!fs.existsSync(configPath)) { - var defaultConfig = Bitcoin.DEFAULT_CONFIG; - if(this.node.https && this.node.httpsOptions) { - defaultConfig += 'rpcssl=1\n'; - defaultConfig += 'rpcsslprivatekeyfile=' + this.node.httpsOptions.key + '\n'; - defaultConfig += 'rpcsslcertificatechainfile=' + this.node.httpsOptions.cert + '\n'; - } - fs.writeFileSync(configPath, defaultConfig); - } - var file = fs.readFileSync(configPath); var unparsed = file.toString().split('\n'); for(var i = 0; i < unparsed.length; i++) { @@ -72,11 +92,36 @@ Bitcoin.prototype._loadConfiguration = function() { $.checkState( this.configuration.txindex && this.configuration.txindex === 1, - 'Txindex option is required in order to use most of the features of bitcore-node. ' + + '"txindex" option is required in order to use transaction query features of bitcore-node. ' + 'Please add "txindex=1" to your configuration and reindex an existing database if ' + 'necessary with reindex=1' ); + $.checkState( + this.configuration.addressindex && this.configuration.addressindex === 1, + '"addressindex" option is required in order to use address query features of bitcore-node. ' + + 'Please add "addressindex=1" to your configuration and reindex an existing database if ' + + 'necessary with reindex=1' + ); + + $.checkState( + this.configuration.server && this.configuration.server === 1, + '"server" option is required to communicate to bitcoind from bitcore. ' + + 'Please add "server=1" to your configuration and restart' + ); + + $.checkState( + this.configuration.zmqpubhashtx, + '"zmqpubhashtx" option is required to get event updates from bitcoind. ' + + 'Please add "zmqpubhashtx=tcp://127.0.0.1:" to your configuration and restart' + ); + + $.checkState( + this.configuration.zmqpubhashtx, + '"zmqpubhashblock" option is required to get event updates from bitcoind. ' + + 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' + ); + if (this.configuration.reindex && this.configuration.reindex === 1) { log.warn('Reindex option is currently enabled. This means that bitcoind is undergoing a reindex. ' + 'The reindex flag will start the index from beginning every time the node is started, so it ' + @@ -87,42 +132,38 @@ Bitcoin.prototype._loadConfiguration = function() { }; -Bitcoin.prototype._onTipUpdate = function(result) { - if (result) { - // Emit and event that the tip was updated - this.height = result; - this.emit('tip', result); - - // TODO stopping status - if(!this.node.stopping) { - var percentage = this.syncPercentage(); - log.info('Bitcoin Height:', this.height, 'Percentage:', percentage); - } - - // Recursively wait until the next update - bindings.onTipUpdate(this._onTipUpdate.bind(this)); - } -}; - Bitcoin.prototype._registerEventHandlers = function() { var self = this; - // Set the height and emit a new tip - bindings.onTipUpdate(self._onTipUpdate.bind(this)); - - // Register callback function to handle transactions entering the mempool - bindings.startTxMon(function(txs) { - for(var i = 0; i < txs.length; i++) { - self.emit('tx', txs[i]); + this.zmqSubSocket.subscribe('hashblock'); + this.zmqSubSocket.subscribe('hashtx'); + + this.zmqSubSocket.on('message', function(topic, message) { + var topicString = topic.toString('utf8'); + if (topicString === 'hashtx') { + self.emit('tx', message.toString('hex')); + } else if (topicString === 'hashblock') { + self.tiphash = message.toString('hex'); + self.client.getBlock(self.tiphash, function(err, response) { + if (err) { + return log.error(err); + } + self.height = response.result.height; + $.checkState(self.height >= 0); + self.emit('tip', self.height); + }); + + if(!self.node.stopping) { + self.syncPercentage(function(err, percentage) { + if (err) { + return log.error(err); + } + log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + }); + } } }); - // Register callback function to handle transactions leaving the mempool - bindings.startTxMonLeave(function(txs) { - for(var i = 0; i < txs.length; i++) { - self.emit('txleave', txs[i]); - } - }); }; Bitcoin.prototype._onReady = function(result, callback) { @@ -130,19 +171,40 @@ Bitcoin.prototype._onReady = function(result, callback) { self._registerEventHandlers(); - var info = self.getInfo(); - self.height = info.blocks; - - self.getBlock(0, function(err, block) { + self.client.getInfo(function(err, response) { if (err) { return callback(err); } - self.genesisBuffer = block; - self.emit('ready', result); - log.info('Bitcoin Daemon Ready'); - callback(); + self.height = response.result.blocks; + + self.client.getBlockHash(0, function(err, response) { + if (err) { + return callback(err); + } + var blockhash = response.result; + self.getBlock(blockhash, function(err, block) { + if (err) { + return callback(err); + } + self.tiphash = block.hash; + self.genesisBuffer = block.toBuffer(); + self.emit('ready', result); + log.info('Bitcoin Daemon Ready'); + callback(); + }); + }); }); +}; +Bitcoin.prototype._getNetworkOption = function() { + var networkOption; + if (this.node.network === bitcore.Networks.testnet) { + if (this.node.network.regtestEnabled) { + networkOption = '--regtest'; + } + networkOption = '--testnet'; + } + return networkOption; }; /** @@ -152,58 +214,348 @@ Bitcoin.prototype._onReady = function(result, callback) { Bitcoin.prototype.start = function(callback) { var self = this; - this._loadConfiguration(); + self._loadConfiguration(); - var networkName = this.node.network.name; - if (this.node.network.regtestEnabled) { - networkName = 'regtest'; + var options = [ + '--conf=' + path.resolve(this.node.datadir, './bitcoin.conf'), + '--datadir=' + this.node.datadir, + ]; + + if (self._getNetworkOption()) { + options.push(self._getNetworkOption()); } - bindings.start({ - datadir: this.node.datadir, - network: networkName - }, function(err) { - if(err) { - return callback(err); + self.process = spawn('bitcoind', options, {stdio: 'inherit'}); + + self.process.on('error', function(err) { + log.error(err); + }); + + async.retry({times: 60, interval: 5000}, function(done) { + if (self.node.stopping) { + return done(new Error('Stopping while trying to connect to bitcoind.')); } - // Wait until the block chain is ready - bindings.onBlocksReady(function(err, result) { + + self.client = new BitcoinRPC({ + protocol: 'http', + host: '127.0.0.1', + port: self.configuration.rpcport, + user: self.configuration.rpcuser, + pass: self.configuration.rpcpassword + }); + + self.client.getInfo(function(err) { if (err) { - return callback(err); + if (!(err instanceof Error)) { + log.warn(err.message); + } + return done(new Error('Could not connect to bitcoind RPC')); } - if (self._reindex) { - var interval = setInterval(function() { - var percentSynced = bindings.syncPercentage(); - log.info("Bitcoin Core Daemon Reindex Percentage: " + percentSynced); - if (percentSynced >= 100) { + done(); + }); + + }, function ready(err, result) { + if (err) { + return callback(err); + } + + self.zmqSubSocket = zmq.socket('sub'); + + self.zmqSubSocket.on('monitor_error', function(err) { + log.error('Error in monitoring: %s, will restart monitoring in 5 seconds', err); + setTimeout(function() { + self.zmqSubSocket.monitor(500, 0); + }, 5000); + }); + + self.zmqSubSocket.monitor(500, 0); + self.zmqSubSocket.connect(self.configuration.zmqpubhashtx); + + if (self._reindex) { + var interval = setInterval(function() { + self.syncPercentage(function(err, percentSynced) { + if (err) { + return log.error(err); + } + log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); + if (Math.round(percentSynced) >= 100) { self._reindex = false; self._onReady(result, callback); clearInterval(interval); } - }, self._reindexWait); + }); + }, self._reindexWait); - } - else { - self._onReady(result, callback); - } - }); + } else { + self._onReady(result, callback); + } }); + }; /** * Helper to determine the state of the database. + * @param {Function} callback * @returns {Boolean} If the database is fully synced */ -Bitcoin.prototype.isSynced = function() { - return bindings.isSynced(); +Bitcoin.prototype.isSynced = function(callback) { + this.syncPercentage(function(err, percentage) { + if (err) { + return callback(err); + } + if (Math.round(percentage) >= 100) { + callback(null, true); + } else { + callback(null, false); + } + }); }; /** * Helper to determine the progress of the database. + * @param {Function} callback * @returns {Number} An estimated percentage of the syncronization status */ -Bitcoin.prototype.syncPercentage = function() { - return bindings.syncPercentage(); +Bitcoin.prototype.syncPercentage = function(callback) { + this.client.getBlockchainInfo(function(err, response) { + if (err) { + return callback(err); + } + var percentSynced = response.result.verificationprogress * 100; + callback(null, percentSynced); + }); +}; + +Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { + // TODO keep a cache and update the cache by a range of block heights + var addresses = [addressArg]; + if (Array.isArray(addressArg)) { + addresses = addressArg; + } + this.client.getAddressBalance({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); +}; + +Bitcoin.prototype.getAddressUnspentOutputs = function() { + // TODO add this rpc method to bitcoind +}; + +Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { + // TODO Keep a cache updated for queries + var addresses = [addressArg]; + if (Array.isArray(addressArg)) { + addresses = addressArg; + } + this.client.getAddressTxids({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + return callback(null, response.result); + }); +}; + +Bitcoin.prototype._getConfirmationsDetail = function(transaction) { + var confirmations = 0; + if (transaction.__height >= 0) { + confirmations = this.height - transaction.__height + 1; + } + return confirmations; +}; + +Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addressStrings) { + var result = { + addresses: {}, + satoshis: 0 + }; + + for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { + var input = transaction.inputs[inputIndex]; + if (!input.script) { + continue; + } + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress) { + var inputAddressString = inputAddress.toString(); + if (addressStrings.indexOf(inputAddressString) >= 0) { + if (!result.addresses[inputAddressString]) { + result.addresses[inputAddressString] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[inputAddressString].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; + } + } + } + + for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { + var output = transaction.outputs[outputIndex]; + if (!output.script) { + continue; + } + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress) { + var outputAddressString = outputAddress.toString(); + if (addressStrings.indexOf(outputAddressString) >= 0) { + if (!result.addresses[outputAddressString]) { + result.addresses[outputAddressString] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[outputAddressString].outputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; + } + } + } + + return result; + +}; + +/** + * Will expand into a detailed transaction from a txid + * @param {Object} txid - A bitcoin transaction id + * @param {Function} callback + */ +Bitcoin.prototype._getDetailedTransaction = function(txid, options, next) { + var self = this; + var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; + + self.getTransactionWithBlockInfo( + txid, + queryMempool, + function(err, transaction) { + if (err) { + return next(err); + } + + transaction.populateInputs(self, [], function(err) { + if (err) { + return next(err); + } + + var addressDetails = self._getAddressDetailsForTransaction(transaction, options.addressStrings); + + var details = { + addresses: addressDetails.addresses, + satoshis: addressDetails.satoshis, + height: transaction.__height, + confirmations: self._getConfirmationsDetail(transaction), + timestamp: transaction.__timestamp, + // TODO bitcore-lib should return null instead of throwing error on coinbase + fees: !transaction.isCoinbase() ? transaction.getFee() : null, + tx: transaction + }; + next(null, details); + }); + } + ); +}; + +Bitcoin.prototype._getAddressStrings = function(addresses) { + var addressStrings = []; + for (var i = 0; i < addresses.length; i++) { + var address = addresses[i]; + if (address instanceof bitcore.Address) { + addressStrings.push(address.toString()); + } else if (_.isString(address)) { + addressStrings.push(address); + } else { + throw new TypeError('Addresses are expected to be strings'); + } + } + return addressStrings; +}; + +Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { + var self = this; + var addresses = [addressArg]; + if (addresses.length > this.maxAddressesQuery) { + return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); + } + + var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; + var addressStrings = this._getAddressStrings(addresses); + + self.getAddressTxids(addresses, {}, function(err, txids) { + if (err) { + return callback(err); + } + async.mapSeries( + txids, + function(txid, next) { + self._getDetailedTransaction(txid, { + queryMempool: queryMempool, + addressStrings: addressStrings + }, next); + }, + function(err, transactions) { + if (err) { + return callback(err); + } + callback(null, { + totalCount: txids.length, + items: transactions + }); + } + ); + }); +}; + +Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { + // TODO: optional mempool + var self = this; + var summary = {}; + + if (_.isUndefined(options.queryMempool)) { + options.queryMempool = true; + } + + function getBalance(done) { + self.getAddressBalance(addressArg, options, function(err, data) { + if (err) { + return done(err); + } + summary.totalReceived = data.received; + summary.totalSpent = data.received - data.balance; + summary.balance = data.balance; + done(); + }); + } + + function getTxList(done) { + self.getAddressTxids(addressArg, options, function(err, txids) { + if (err) { + return done(err); + } + summary.txids = txids; + summary.appearances = txids.length; + done(); + }); + } + + var tasks = []; + if (!options.noBalance) { + tasks.push(getBalance); + } + if (!options.noTxList) { + tasks.push(getTxList); + } + + async.parallel(tasks, function(err) { + if (err) { + return callback(err); + } + callback(null, summary); + }); }; /** @@ -211,41 +563,80 @@ Bitcoin.prototype.syncPercentage = function() { * @param {String|Number} block - A block hash or block height number */ Bitcoin.prototype.getBlock = function(block, callback) { - return bindings.getBlock(block, callback); + // TODO apply performance patch to the RPC method for raw data + // TODO keep a cache of results + var self = this; + + function queryHeader(blockhash) { + self.client.getBlock(blockhash, false, function(err, response) { + if (err) { + return callback(err); + } + var block = bitcore.Block.fromString(response.result); + callback(null, block); + }); + } + + if (_.isNumber(block)) { + self.client.getBlockHash(block, function(err, response) { + if (err) { + return callback(err); + } + var blockhash = response.result; + queryHeader(blockhash); + }); + } else { + queryHeader(block); + } + }; -/** - * Will return the spent status of an output (not including the mempool) - * @param {String} txid - The transaction hash - * @param {Number} outputIndex - The output index in the transaction - * @returns {Boolean} If the output has been spent - */ -Bitcoin.prototype.isSpent = function(txid, outputIndex) { - return bindings.isSpent(txid, outputIndex); +Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { + var self = this; + self.client.getBlockHashes(high, low, function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); }; /** * Will return the block index information, the output will have the format: * { - * prevHash: '7194fcf33f58c96720f88f21ab28c34ebc5638c5f88d7838517deb27313b59de', - * hash: '7c5caf0af1bf16e3467b275a3b408bc1d251bff3c25be20cb727c47b66a7b216', + * prevHash: '000000004956cc2edd1a8caa05eacfa3c69f4c490bfc9ace820257834115ab35', + * nextHash: '0000000000629d100db387f37d0f37c51118f250fb0946310a8c37316cbc4028' + * hash: ' 00000000009e2958c15ff9290d571bf9459e93b19765c6801ddeccadbb160a1e', * chainWork: '0000000000000000000000000000000000000000000000000000000000000016', * height: 10 * } * @param {String|Number} block - A block hash or block height * @returns {Object} */ -Bitcoin.prototype.getBlockIndex = function(block) { - return bindings.getBlockIndex(block); -}; +Bitcoin.prototype.getBlockHeader = function(block, callback) { + // TODO keep a cache of queries + var self = this; -/** - * Will return if the block is a part of the main chain. - * @param {String} blockHash - * @returns {Boolean} - */ -Bitcoin.prototype.isMainChain = function(blockHash) { - return bindings.isMainChain(blockHash); + function queryHeader(blockhash) { + self.client.getBlockHeader(blockhash, function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); + } + + if (_.isNumber(block)) { + self.client.getBlockHash(block, function(err, response) { + if (err) { + return callback(err); + } + var blockhash = response.result; + queryHeader(blockhash); + }); + } else { + queryHeader(block); + } }; /** @@ -253,8 +644,13 @@ Bitcoin.prototype.isMainChain = function(blockHash) { * @param {Number} blocks - The number of blocks for the transaction to be confirmed. * @returns {Number} */ -Bitcoin.prototype.estimateFee = function(blocks) { - return bindings.estimateFee(blocks); +Bitcoin.prototype.estimateFee = function(blocks, callback) { + this.client.estimateFee(blocks, function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); }; /** @@ -263,8 +659,21 @@ Bitcoin.prototype.estimateFee = function(blocks) { * @param {String} transaction - The hex string of the transaction * @param {Boolean} allowAbsurdFees - Enable large fees */ -Bitcoin.prototype.sendTransaction = function(transaction, allowAbsurdFees) { - return bindings.sendTransaction(transaction, allowAbsurdFees); +Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { + var txString; + if (tx instanceof Transaction) { + txString = tx.serialize(); + } else { + txString = tx; + } + + this.client.sendTransaction(txString, allowAbsurdFees, function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); + }; /** @@ -274,7 +683,18 @@ Bitcoin.prototype.sendTransaction = function(transaction, allowAbsurdFees) { * @param {Function} callback */ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { - return bindings.getTransaction(txid, queryMempool, callback); + // TODO keep an LRU cache available of transactions + this.client.getRawTransaction(txid, function(err, response) { + if (err) { + return callback(err); + } + if (!response.result) { + return callback(new errors.Transaction.NotFound()); + } + var tx = Transaction(); + tx.fromString(response.result); + callback(null, tx); + }); }; /** @@ -290,41 +710,41 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { * @param {Function} callback */ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) { - return bindings.getTransactionWithBlockInfo(txid, queryMempool, callback); -}; - -/** - * Will return the entire mempool as an Array of transaction Buffers. - * @returns {Array} - */ -Bitcoin.prototype.getMempoolTransactions = function() { - return bindings.getMempoolTransactions(); -}; - -/** - * Will add a transaction to the mempool without any validation. This is used - * exclusively for testing purposes. - * @param {String} transaction - The hex string for the transaction - */ -Bitcoin.prototype.addMempoolUncheckedTransaction = function(transaction) { - return bindings.addMempoolUncheckedTransaction(transaction); + // TODO keep an LRU cache available of transactions + // TODO get information from txindex as an RPC method + this.client.getRawTransaction(txid, 1, function(err, response) { + if (err) { + return callback(err); + } + if (!response.result) { + return callback(new errors.Transaction.NotFound()); + } + var tx = Transaction(); + tx.fromString(response.result.hex); + tx.__blockHash = response.result.blockhash; + tx.__height = response.result.height; + tx.__timestamp = response.result.time; + callback(null, tx); + }); }; /** * Will get the best block hash for the chain. * @returns {String} */ -Bitcoin.prototype.getBestBlockHash = function() { - return bindings.getBestBlockHash(); +Bitcoin.prototype.getBestBlockHash = function(callback) { + // TODO keep an LRU cache available of transactions + this.client.getBestBlockHash(function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); }; -/** - * Will get the next block hash for a block hash. - * @param {String} hash - The starting block hash - * @returns {String} - */ -Bitcoin.prototype.getNextBlockHash = function(hash) { - return bindings.getNextBlockHash(hash); +Bitcoin.prototype.getInputForOutput = function(txid, index, options, callback) { + // TODO + setImmediate(callback); }; /** @@ -341,8 +761,13 @@ Bitcoin.prototype.getNextBlockHash = function(hash) { * errors: '' * } */ -Bitcoin.prototype.getInfo = function() { - return bindings.getInfo(); +Bitcoin.prototype.getInfo = function(callback) { + this.client.getInfo(function(err, response) { + if (err) { + return callback(err); + } + callback(null, response.result); + }); }; /** @@ -350,14 +775,18 @@ Bitcoin.prototype.getInfo = function() { * @param {Function} callback */ Bitcoin.prototype.stop = function(callback) { - return bindings.stop(function(err, status) { - if (err) { - return callback(err); - } else { - log.info(status); - return callback(); - } - }); + if (this.process) { + this.process.once('exit', function(err, status) { + if (err) { + return callback(err); + } else { + return callback(); + } + }); + this.process.kill('SIGHUP'); + } else { + callback(); + } }; module.exports = Bitcoin; diff --git a/lib/services/db.js b/lib/services/db.js deleted file mode 100644 index 679935ca..00000000 --- a/lib/services/db.js +++ /dev/null @@ -1,812 +0,0 @@ -'use strict'; - -var util = require('util'); -var fs = require('fs'); -var async = require('async'); -var levelup = require('levelup'); -var leveldown = require('leveldown'); -var mkdirp = require('mkdirp'); -var bitcore = require('bitcore-lib'); -var BufferUtil = bitcore.util.buffer; -var Networks = bitcore.Networks; -var Block = bitcore.Block; -var $ = bitcore.util.preconditions; -var index = require('../'); -var errors = index.errors; -var log = index.log; -var Transaction = require('../transaction'); -var Service = require('../service'); - -/** - * This service synchronizes a leveldb database with bitcoin block chain by connecting and - * disconnecting blocks to build new indexes that can be queried. Other services can extend - * the data that is indexed by implementing a `blockHandler` method. - * - * @param {Object} options - * @param {Node} options.node - A reference to the node - * @param {Node} options.store - A levelup backend store - */ -function DB(options) { - /* jshint maxstatements: 20 */ - - if (!(this instanceof DB)) { - return new DB(options); - } - if (!options) { - options = {}; - } - - Service.call(this, options); - - // Used to keep track of the version of the indexes - // to determine during an upgrade if a reindex is required - this.version = 2; - - this.tip = null; - this.genesis = null; - - $.checkState(this.node.network, 'Node is expected to have a "network" property'); - this.network = this.node.network; - - this._setDataPath(); - - this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; - this.maxTransactionLimit = options.maxTransactionLimit || DB.MAX_TRANSACTION_LIMIT; - - this.levelupStore = leveldown; - if (options.store) { - this.levelupStore = options.store; - } - - this.retryInterval = 60000; - - this.subscriptions = { - transaction: [], - block: [] - }; -} - -util.inherits(DB, Service); - -DB.dependencies = ['bitcoind']; - -DB.PREFIXES = { - VERSION: new Buffer('ff', 'hex'), - BLOCKS: new Buffer('01', 'hex'), - TIP: new Buffer('04', 'hex') -}; - -// The maximum number of transactions to query at once -// Used for populating previous inputs -DB.MAX_TRANSACTION_LIMIT = 5; - -// The default maxiumum number of files open for leveldb -DB.DEFAULT_MAX_OPEN_FILES = 200; - -/** - * This function will set `this.dataPath` based on `this.node.network`. - * @private - */ -DB.prototype._setDataPath = function() { - $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); - if (this.node.network === Networks.livenet) { - this.dataPath = this.node.datadir + '/bitcore-node.db'; - } else if (this.node.network === Networks.testnet) { - if (this.node.network.regtestEnabled) { - this.dataPath = this.node.datadir + '/regtest/bitcore-node.db'; - } else { - this.dataPath = this.node.datadir + '/testnet3/bitcore-node.db'; - } - } else { - throw new Error('Unknown network: ' + this.network); - } -}; - -DB.prototype._checkVersion = function(callback) { - var self = this; - var options = { - keyEncoding: 'binary', - valueEncoding: 'binary' - }; - self.store.get(DB.PREFIXES.TIP, options, function(err) { - if (err instanceof levelup.errors.NotFoundError) { - // The database is brand new and doesn't have a tip stored - // we can skip version checking - return callback(); - } else if (err) { - return callback(err); - } - self.store.get(DB.PREFIXES.VERSION, options, function(err, buffer) { - var version; - if (err instanceof levelup.errors.NotFoundError) { - // The initial version (1) of the database didn't store the version number - version = 1; - } else if (err) { - return callback(err); - } else { - version = buffer.readUInt32BE(); - } - if (self.version !== version) { - var helpUrl = 'https://github.com/bitpay/bitcore-node/blob/master/docs/services/db.md#how-to-reindex'; - return callback(new Error( - 'The version of the database "' + version + '" does not match the expected version "' + - self.version + '". A recreation of "' + self.dataPath + '" (can take several hours) is ' + - 'required or to switch versions of software to match. Please see ' + helpUrl + - ' for more information.' - )); - } - callback(); - }); - }); -}; - -DB.prototype._setVersion = function(callback) { - var versionBuffer = new Buffer(new Array(4)); - versionBuffer.writeUInt32BE(this.version); - this.store.put(DB.PREFIXES.VERSION, versionBuffer, callback); -}; - -/** - * Called by Node to start the service. - * @param {Function} callback - */ -DB.prototype.start = function(callback) { - - var self = this; - if (!fs.existsSync(this.dataPath)) { - mkdirp.sync(this.dataPath); - } - - this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles }); - this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); - - this.once('ready', function() { - log.info('Bitcoin Database Ready'); - - // Notify that there is a new tip - self.node.services.bitcoind.on('tip', function(height) { - if(!self.node.stopping) { - self.sync(); - } - }); - }); - - async.series([ - function(next) { - self._checkVersion(next); - }, - function(next) { - self._setVersion(next); - } - ], function(err) { - if (err) { - return callback(err); - } - self.loadTip(function(err) { - if (err) { - return callback(err); - } - - self.sync(); - self.emit('ready'); - setImmediate(callback); - }); - }); -}; - -/** - * Called by Node to stop the service - * @param {Function} callback - */ -DB.prototype.stop = function(callback) { - var self = this; - - // Wait until syncing stops and all db operations are completed before closing leveldb - async.whilst(function() { - return self.bitcoindSyncing; - }, function(next) { - setTimeout(next, 10); - }, function() { - self.store.close(callback); - }); -}; - -/** - * Will give information about the database from bitcoin. - * @param {Function} callback - */ -DB.prototype.getInfo = function(callback) { - var self = this; - setImmediate(function() { - var info = self.node.bitcoind.getInfo(); - callback(null, info); - }); -}; - -/** - * Closes the underlying store database - * @param {Function} callback - */ -DB.prototype.close = function(callback) { - this.store.close(callback); -}; - -/** - * This function is responsible for emitting `db/transaction` events. - * @param {Object} txInfo - The data from the bitcoind.on('tx') event - * @param {Buffer} txInfo.buffer - The transaction buffer - * @param {Boolean} txInfo.mempool - If the transaction was accepted in the mempool - * @param {String} txInfo.hash - The hash of the transaction - */ -DB.prototype.transactionHandler = function(txInfo) { - var tx = Transaction().fromBuffer(txInfo.buffer); - for (var i = 0; i < this.subscriptions.transaction.length; i++) { - this.subscriptions.transaction[i].emit('db/transaction', { - rejected: !txInfo.mempool, - tx: tx - }); - } -}; - -/** - * Called by Node to determine the available API methods. - */ -DB.prototype.getAPIMethods = function() { - var methods = [ - ['getBlock', this, this.getBlock, 1], - ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], - ['getTransaction', this, this.getTransaction, 2], - ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], - ['sendTransaction', this, this.sendTransaction, 1], - ['estimateFee', this, this.estimateFee, 1] - ]; - return methods; -}; - -DB.prototype.loadTip = function(callback) { - var self = this; - - var options = { - keyEncoding: 'binary', - valueEncoding: 'binary' - }; - - self.store.get(DB.PREFIXES.TIP, options, function(err, tipData) { - if(err && err instanceof levelup.errors.NotFoundError) { - self.tip = self.genesis; - self.tip.__height = 0; - self.connectBlock(self.genesis, function(err) { - if(err) { - return callback(err); - } - - self.emit('addblock', self.genesis); - callback(); - }); - return; - } else if(err) { - return callback(err); - } - - var hash = tipData.toString('hex'); - - var times = 0; - async.retry({times: 3, interval: self.retryInterval}, function(done) { - self.getBlock(hash, function(err, tip) { - if(err) { - times++; - log.warn('Bitcoind does not have our tip (' + hash + '). Bitcoind may have crashed and needs to catch up.'); - if(times < 3) { - log.warn('Retrying in ' + (self.retryInterval / 1000) + ' seconds.'); - } - return done(err); - } - - done(null, tip); - }); - }, function(err, tip) { - if(err) { - log.warn('Giving up after 3 tries. Please report this bug to https://github.com/bitpay/bitcore-node/issues'); - log.warn('Please reindex your database.'); - return callback(err); - } - - self.tip = tip; - var blockIndex = self.node.services.bitcoind.getBlockIndex(self.tip.hash); - if(!blockIndex) { - return callback(new Error('Could not get height for tip.')); - } - self.tip.__height = blockIndex.height; - callback(); - }); - }); -}; - -/** - * Will get a block from bitcoind and give a Bitcore Block - * @param {String|Number} hash - A block hash or block height - */ -DB.prototype.getBlock = function(hash, callback) { - this.node.services.bitcoind.getBlock(hash, function(err, blockBuffer) { - if (err) { - return callback(err); - } - callback(null, Block.fromBuffer(blockBuffer)); - }); -}; - -/** - * Get block hashes between two timestamps - * @param {Number} high - high timestamp, in seconds, inclusive - * @param {Number} low - low timestamp, in seconds, inclusive - * @param {Function} callback - */ -DB.prototype.getBlockHashesByTimestamp = function(high, low, callback) { - var self = this; - var hashes = []; - var lowKey; - var highKey; - - try { - lowKey = this._encodeBlockIndexKey(low); - highKey = this._encodeBlockIndexKey(high); - } catch(e) { - return callback(e); - } - - var stream = this.store.createReadStream({ - gte: lowKey, - lte: highKey, - reverse: true, - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - - stream.on('data', function(data) { - hashes.push(self._decodeBlockIndexValue(data.value)); - }); - - var error; - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('close', function() { - if (error) { - return callback(error); - } - callback(null, hashes); - }); - - return stream; -}; - -/** - * Will give a Bitcore Transaction from bitcoind by txid - * @param {String} txid - A transaction hash - * @param {Boolean} queryMempool - Include the mempool - * @param {Function} callback - */ -DB.prototype.getTransaction = function(txid, queryMempool, callback) { - this.node.services.bitcoind.getTransaction(txid, queryMempool, function(err, txBuffer) { - if (err) { - return callback(err); - } - if (!txBuffer) { - return callback(new errors.Transaction.NotFound()); - } - - callback(null, Transaction().fromBuffer(txBuffer)); - }); -}; - -/** - * Will give a Bitcore Transaction and populated information about the block included. - * @param {String} txid - A transaction hash - * @param {Boolean} queryMempool - Include the mempool - * @param {Function} callback - */ -DB.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) { - this.node.services.bitcoind.getTransactionWithBlockInfo(txid, queryMempool, function(err, obj) { - if (err) { - return callback(err); - } - - var tx = Transaction().fromBuffer(obj.buffer); - tx.__blockHash = obj.blockHash; - tx.__height = obj.height; - tx.__timestamp = obj.timestamp; - - callback(null, tx); - }); -}; - -/** - * Will send a transaction to the Bitcoin network. - * @param {Transaction} tx - An instance of a Bitcore Transaction - * @param {Function} callback - */ -DB.prototype.sendTransaction = function(tx, callback) { - var txString; - if (tx instanceof Transaction) { - txString = tx.serialize(); - } else { - txString = tx; - } - - try { - var txid = this.node.services.bitcoind.sendTransaction(txString); - return callback(null, txid); - } catch(err) { - return callback(err); - } -}; - -/** - * Will estimate fees for a transaction and give a result in - * satoshis per kilobyte. Similar to the bitcoind estimateFee method. - * @param {Number} blocks - The number of blocks for the transaction to be included. - * @param {Function} callback - */ -DB.prototype.estimateFee = function(blocks, callback) { - var self = this; - setImmediate(function() { - callback(null, self.node.services.bitcoind.estimateFee(blocks)); - }); -}; - -/** - * Called by the Bus to determine the available events. - */ -DB.prototype.getPublishEvents = function() { - return [ - { - name: 'db/transaction', - scope: this, - subscribe: this.subscribe.bind(this, 'transaction'), - unsubscribe: this.unsubscribe.bind(this, 'transaction') - }, - { - name: 'db/block', - scope: this, - subscribe: this.subscribe.bind(this, 'block'), - unsubscribe: this.unsubscribe.bind(this, 'block') - } - ]; -}; - -DB.prototype.subscribe = function(name, emitter) { - this.subscriptions[name].push(emitter); -}; - -DB.prototype.unsubscribe = function(name, emitter) { - var index = this.subscriptions[name].indexOf(emitter); - if (index > -1) { - this.subscriptions[name].splice(index, 1); - } -}; - -/** - * Will give the previous hash for a block. - * @param {String} blockHash - * @param {Function} callback - */ -DB.prototype.getPrevHash = function(blockHash, callback) { - var blockIndex = this.node.services.bitcoind.getBlockIndex(blockHash); - setImmediate(function() { - if (blockIndex) { - callback(null, blockIndex.prevHash); - } else { - callback(new Error('Could not get prevHash, block not found')); - } - }); -}; - -/** - * Connects a block to the database and add indexes - * @param {Block} block - The bitcore block - * @param {Function} callback - */ -DB.prototype.connectBlock = function(block, callback) { - log.debug('DB handling new chain block'); - this.runAllBlockHandlers(block, true, callback); -}; - -/** - * Disconnects a block from the database and removes indexes - * @param {Block} block - The bitcore block - * @param {Function} callback - */ -DB.prototype.disconnectBlock = function(block, callback) { - log.debug('DB removing chain block'); - this.runAllBlockHandlers(block, false, callback); -}; - -/** - * Will collect all database operations for a block from other services that implement - * `blockHandler` methods and then save operations to the database. - * @param {Block} block - The bitcore block - * @param {Boolean} add - If the block is being added/connected or removed/disconnected - * @param {Function} callback - */ -DB.prototype.runAllBlockHandlers = function(block, add, callback) { - var self = this; - var operations = []; - - // Notify block subscribers - for (var i = 0; i < this.subscriptions.block.length; i++) { - this.subscriptions.block[i].emit('db/block', block.hash); - } - - // Update tip - var tipHash = add ? new Buffer(block.hash, 'hex') : BufferUtil.reverse(block.header.prevHash); - operations.push({ - type: 'put', - key: DB.PREFIXES.TIP, - value: tipHash - }); - - // Update block index - operations.push({ - type: add ? 'put' : 'del', - key: this._encodeBlockIndexKey(block.header.timestamp), - value: this._encodeBlockIndexValue(block.hash) - }); - - async.eachSeries( - this.node.services, - function(mod, next) { - if(mod.blockHandler) { - $.checkArgument(typeof mod.blockHandler === 'function', 'blockHandler must be a function'); - - mod.blockHandler.call(mod, block, add, function(err, ops) { - if (err) { - return next(err); - } - if (ops) { - $.checkArgument(Array.isArray(ops), 'blockHandler for ' + mod.name + ' returned non-array'); - operations = operations.concat(ops); - } - next(); - }); - } else { - setImmediate(next); - } - }, - function(err) { - if (err) { - return callback(err); - } - - log.debug('Updating the database with operations', operations); - self.store.batch(operations, callback); - } - ); -}; - -DB.prototype._encodeBlockIndexKey = function(timestamp) { - $.checkArgument(timestamp >= 0 && timestamp <= 4294967295, 'timestamp out of bounds'); - var timestampBuffer = new Buffer(4); - timestampBuffer.writeUInt32BE(timestamp); - return Buffer.concat([DB.PREFIXES.BLOCKS, timestampBuffer]); -}; - -DB.prototype._encodeBlockIndexValue = function(hash) { - return new Buffer(hash, 'hex'); -}; - -DB.prototype._decodeBlockIndexValue = function(value) { - return value.toString('hex'); -}; - -/** - * This function will find the common ancestor between the current chain and a forked block, - * by moving backwards on both chains until there is a meeting point. - * @param {Block} block - The new tip that forks the current chain. - * @param {Function} done - A callback function that is called when complete. - */ -DB.prototype.findCommonAncestor = function(block, done) { - - var self = this; - - var mainPosition = self.tip.hash; - var forkPosition = block.hash; - - var mainHashesMap = {}; - var forkHashesMap = {}; - - mainHashesMap[mainPosition] = true; - forkHashesMap[forkPosition] = true; - - var commonAncestor = null; - - async.whilst( - function() { - return !commonAncestor; - }, - function(next) { - - if(mainPosition) { - var mainBlockIndex = self.node.services.bitcoind.getBlockIndex(mainPosition); - if(mainBlockIndex && mainBlockIndex.prevHash) { - mainHashesMap[mainBlockIndex.prevHash] = true; - mainPosition = mainBlockIndex.prevHash; - } else { - mainPosition = null; - } - } - - if(forkPosition) { - var forkBlockIndex = self.node.services.bitcoind.getBlockIndex(forkPosition); - if(forkBlockIndex && forkBlockIndex.prevHash) { - forkHashesMap[forkBlockIndex.prevHash] = true; - forkPosition = forkBlockIndex.prevHash; - } else { - forkPosition = null; - } - } - - if(forkPosition && mainHashesMap[forkPosition]) { - commonAncestor = forkPosition; - } - - if(mainPosition && forkHashesMap[mainPosition]) { - commonAncestor = mainPosition; - } - - if(!mainPosition && !forkPosition) { - return next(new Error('Unknown common ancestor')); - } - - setImmediate(next); - }, - function(err) { - done(err, commonAncestor); - } - ); -}; - -/** - * This function will attempt to rewind the chain to the common ancestor - * between the current chain and a forked block. - * @param {Block} block - The new tip that forks the current chain. - * @param {Function} done - A callback function that is called when complete. - */ -DB.prototype.syncRewind = function(block, done) { - - var self = this; - - self.findCommonAncestor(block, function(err, ancestorHash) { - if (err) { - return done(err); - } - log.warn('Reorg common ancestor found:', ancestorHash); - // Rewind the chain to the common ancestor - async.whilst( - function() { - // Wait until the tip equals the ancestor hash - return self.tip.hash !== ancestorHash; - }, - function(removeDone) { - - var tip = self.tip; - - // TODO: expose prevHash as a string from bitcore - var prevHash = BufferUtil.reverse(tip.header.prevHash).toString('hex'); - - self.getBlock(prevHash, function(err, previousTip) { - if (err) { - removeDone(err); - } - - // Undo the related indexes for this block - self.disconnectBlock(tip, function(err) { - if (err) { - return removeDone(err); - } - - // Set the new tip - previousTip.__height = self.tip.__height - 1; - self.tip = previousTip; - self.emit('removeblock', tip); - removeDone(); - }); - - }); - - }, done - ); - }); -}; - -/** - * This function will synchronize additional indexes for the chain based on - * the current active chain in the bitcoin daemon. In the event that there is - * a reorganization in the daemon, the chain will rewind to the last common - * ancestor and then resume syncing. - */ -DB.prototype.sync = function() { - var self = this; - - if (self.bitcoindSyncing || self.node.stopping || !self.tip) { - return; - } - - self.bitcoindSyncing = true; - - var height; - - async.whilst(function() { - height = self.tip.__height; - return height < self.node.services.bitcoind.height && !self.node.stopping; - }, function(done) { - self.node.services.bitcoind.getBlock(height + 1, function(err, blockBuffer) { - if (err) { - return done(err); - } - - var block = Block.fromBuffer(blockBuffer); - - // TODO: expose prevHash as a string from bitcore - var prevHash = BufferUtil.reverse(block.header.prevHash).toString('hex'); - - if (prevHash === self.tip.hash) { - - // This block appends to the current chain tip and we can - // immediately add it to the chain and create indexes. - - // Populate height - block.__height = self.tip.__height + 1; - - // Create indexes - self.connectBlock(block, function(err) { - if (err) { - return done(err); - } - self.tip = block; - log.debug('Chain added block to main chain'); - self.emit('addblock', block); - setImmediate(done); - }); - } else { - // This block doesn't progress the current tip, so we'll attempt - // to rewind the chain to the common ancestor of the block and - // then we can resume syncing. - log.warn('Beginning reorg! Current tip: ' + self.tip.hash + '; New tip: ' + block.hash); - self.syncRewind(block, function(err) { - if(err) { - return done(err); - } - - log.warn('Reorg complete. New tip is ' + self.tip.hash); - done(); - }); - } - }); - }, function(err) { - if (err) { - Error.captureStackTrace(err); - return self.node.emit('error', err); - } - - if(self.node.stopping) { - self.bitcoindSyncing = false; - return; - } - - if (self.node.services.bitcoind.isSynced()) { - self.bitcoindSyncing = false; - self.node.emit('synced'); - } else { - self.bitcoindSyncing = false; - } - - }); - -}; - -module.exports = DB; diff --git a/package.json b/package.json index 8b31e793..ef7a6b69 100644 --- a/package.json +++ b/package.json @@ -31,15 +31,8 @@ "bitcore-node": "./bin/bitcore-node" }, "scripts": { - "install": "./bin/install", - "build": "./bin/build", - "clean": "./bin/clean", - "package": "node bin/package.js", - "upload": "node bin/upload.js", - "start": "node bin/start.js", "test": "NODE_ENV=test mocha -R spec --recursive", - "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive", - "libbitcoind": "node bin/start-libbitcoind.js" + "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" }, "tags": [ "bitcoin", @@ -49,44 +42,28 @@ "async": "^1.3.0", "bindings": "^1.2.1", "bitcore-lib": "^0.13.13", + "bitcoind-rpc": "^0.3.0", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", - "leveldown": "bitpay/leveldown#bitpay-1.4.4", - "levelup": "^1.3.1", "liftoff": "^2.2.0", "memdown": "^1.0.0", "mkdirp": "0.5.0", - "nan": "^2.0.9", "npm": "^2.14.1", "semver": "^5.0.1", "socket.io": "bitpay/socket.io#bitpay-1.3.7", - "socket.io-client": "bitpay/socket.io-client#bitpay-1.3.7" + "socket.io-client": "bitpay/socket.io-client#bitpay-1.3.7", + "zmq": "^2.14.0" }, "devDependencies": { - "aws-sdk": "~2.0.0-rc.15", "benchmark": "1.0.0", - "bitcoin": "^2.3.2", - "bitcoind-rpc": "^0.3.0", "chai": "^3.0.0", "mocha": "~1.16.2", "proxyquire": "^1.3.1", "rimraf": "^2.4.2", - "sinon": "^1.15.4", - "bitcore-p2p": "~1.0.0" - }, - "engines": { - "node": "^0.12 || ^4.2" + "sinon": "^1.15.4" }, - "os": [ - "darwin", - "linux" - ], - "cpu": [ - "x64", - "arm" - ], "license": "MIT" } diff --git a/src/libbitcoind.cc b/src/libbitcoind.cc deleted file mode 100644 index 4627b6d3..00000000 --- a/src/libbitcoind.cc +++ /dev/null @@ -1,1614 +0,0 @@ -/** - * bitcoind.js - a binding for node.js which links to libbitcoind.so/dylib. - * Copyright (c) 2015, BitPay (MIT License) - * - * libbitcoind.cc: - * A bitcoind node.js binding. - */ - -#include "libbitcoind.h" - -using namespace std; -using namespace boost; -using namespace node; -using namespace v8; -using Nan::New; -using Nan::Null; -using Nan::Set; -using Nan::ThrowError; -using Nan::GetCurrentContext; -using Nan::GetFunction; -using v8::FunctionTemplate; - -/** - * Bitcoin Globals - */ - -// These global functions and variables are -// required to be defined/exposed here. - -extern void WaitForShutdown(boost::thread_group* threadGroup); -static termios orig_termios; -extern CTxMemPool mempool; -extern int64_t nTimeBestReceived; - -/** - * Node.js Internal Function Templates - */ - -static void -tx_notifier(uv_async_t *handle); - -static void -txleave_notifier(uv_async_t *handle); - -static void -async_tip_update(uv_work_t *req); - -static void -async_tip_update_after(uv_work_t *req); - -static void -async_start_node(uv_work_t *req); - -static void -async_start_node_after(uv_work_t *req); - -static void -async_blocks_ready(uv_work_t *req); - -static void -async_blocks_ready_after(uv_work_t *req); - -static void -async_stop_node(uv_work_t *req); - -static void -async_stop_node_after(uv_work_t *req); - -static int -start_node(void); - -static void -start_node_thread(void); - -static void -async_get_block(uv_work_t *req); - -static void -async_get_block_after(uv_work_t *req); - -static void -async_get_tx(uv_work_t *req); - -static void -async_get_tx_after(uv_work_t *req); - -static void -async_get_tx_and_info(uv_work_t *req); - -static void -async_get_tx_and_info_after(uv_work_t *req); - -static bool -queueTx(const CTransaction&); - -static bool -queueTxLeave(const CTransaction&); - -extern "C" void -init(Handle); - -/** - * Private Global Variables - * Used only by bitcoind functions. - */ -static std::vector txQueue; -static std::vector txQueueLeave; -static uv_async_t txmon_async; -static uv_async_t txmonleave_async; -static Eternal txmon_callback; -static Eternal txmonleave_callback; -static bool txmon_callback_available; -static bool txmonleave_callback_available; - -static volatile bool shutdown_complete = false; -static char *g_data_dir = NULL; -static bool g_rpc = false; -static bool g_testnet = false; -static bool g_regtest = false; -static bool g_txindex = false; - -static boost::thread_group threadGroup; - -/** - * Private Structs - * Used for async functions and necessary linked lists at points. - */ - -struct async_tip_update_data { - uv_work_t req; - size_t result; - Isolate* isolate; - Persistent callback; -}; - -/** - * async_node_data - * Where the uv async request data resides. - */ - -struct async_block_ready_data { - uv_work_t req; - std::string err_msg; - std::string result; - Isolate* isolate; - Persistent callback; -}; - -/** - * async_node_data - * Where the uv async request data resides. - */ - -struct async_node_data { - uv_work_t req; - std::string err_msg; - std::string result; - std::string datadir; - bool rpc; - bool testnet; - bool regtest; - bool txindex; - Isolate* isolate; - Persistent callback; -}; - -/** - * async_block_data - */ - -struct async_block_data { - uv_work_t req; - std::string err_msg; - uint256 hash; - int64_t height; - char* buffer; - uint32_t size; - CBlock cblock; - CBlockIndex* cblock_index; - Isolate* isolate; - Persistent callback; -}; - -/** - * async_tx_data - */ - -struct async_tx_data { - uv_work_t req; - std::string err_msg; - std::string txid; - std::string blockHash; - uint32_t nTime; - int64_t height; - bool queryMempool; - CTransaction ctx; - Isolate* isolate; - Persistent callback; -}; - -/** - * Helpers - */ - -static bool -set_cooked(void); - -/** - * SyncPercentage() - * bitcoind.syncPercentage() - * provides a float value >= indicating the progress of the blockchain sync - */ -NAN_METHOD(SyncPercentage) { - const CChainParams& chainParams = Params(); - float progress = 0; - progress = Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()); - info.GetReturnValue().Set(progress * 100); -}; - -NAN_METHOD(GetBestBlockHash) { - LOCK(cs_main); - info.GetReturnValue().Set(New(chainActive.Tip()->GetBlockHash().GetHex()).ToLocalChecked()); -} - -NAN_METHOD(GetNextBlockHash) { - - if (info.Length() < 1 || !info[0]->IsString()) { - return ThrowError("Usage: bitcoind.getNextBlockHash(blockhash)"); - } - - CBlockIndex* pblockindex; - v8::String::Utf8Value param1(info[0]->ToString()); - std::string *hash = new std::string(*param1); - uint256 shash = uint256S(*hash); - pblockindex = mapBlockIndex[shash]; - CBlockIndex* pnextblockindex = chainActive.Next(pblockindex); - if (pnextblockindex) { - uint256 nexthash = pnextblockindex->GetBlockHash(); - std::string rethash = nexthash.ToString(); - info.GetReturnValue().Set(New(rethash).ToLocalChecked()); - } else { - info.GetReturnValue().Set(Null()); - } - -} - -/** - * IsSynced() - * bitcoind.isSynced() - * returns a boolean of bitcoin is fully synced - */ -NAN_METHOD(IsSynced) { - bool isDownloading = IsInitialBlockDownload(); - info.GetReturnValue().Set(New(!isDownloading)); -}; - -NAN_METHOD(StartTxMon) { - Isolate* isolate = info.GetIsolate(); - Local callback = Local::Cast(info[0]); - Eternal cb(isolate, callback); - txmon_callback = cb; - txmon_callback_available = true; - - CNodeSignals& nodeSignals = GetNodeSignals(); - nodeSignals.TxToMemPool.connect(&queueTx); - - uv_async_init(uv_default_loop(), &txmon_async, tx_notifier); - - info.GetReturnValue().Set(Null()); -}; - -NAN_METHOD(StartTxMonLeave) { - Isolate* isolate = info.GetIsolate(); - Local callback = Local::Cast(info[0]); - Eternal cb(isolate, callback); - txmonleave_callback = cb; - txmonleave_callback_available = true; - - CNodeSignals& nodeSignals = GetNodeSignals(); - nodeSignals.TxLeaveMemPool.connect(&queueTxLeave); - - uv_async_init(uv_default_loop(), &txmonleave_async, txleave_notifier); - - info.GetReturnValue().Set(Null()); -}; - -static void -tx_notifier(uv_async_t *handle) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - Local results = Array::New(isolate); - int arrayIndex = 0; - - LOCK(cs_main); - BOOST_FOREACH(const CTransaction& tx, txQueue) { - - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << tx; - std::string stx = ssTx.str(); - Nan::MaybeLocal txBuffer = Nan::CopyBuffer((char *)stx.c_str(), stx.size()); - - uint256 hash = tx.GetHash(); - - Local obj = New(); - - Nan::Set(obj, New("buffer").ToLocalChecked(), txBuffer.ToLocalChecked()); - Nan::Set(obj, New("hash").ToLocalChecked(), New(hash.GetHex()).ToLocalChecked()); - Nan::Set(obj, New("mempool").ToLocalChecked(), New(true)); - - results->Set(arrayIndex, obj); - arrayIndex++; - } - - const unsigned argc = 1; - Local argv[argc] = { - Local::New(isolate, results) - }; - - Local cb = txmon_callback.Get(isolate); - - cb->Call(isolate->GetCurrentContext()->Global(), argc, argv); - - txQueue.clear(); - -} -static bool -queueTx(const CTransaction& tx) { - LOCK(cs_main); - txQueue.push_back(tx); - uv_async_send(&txmon_async); - return true; -} - -static void -txleave_notifier(uv_async_t *handle) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - Local results = Array::New(isolate); - int arrayIndex = 0; - - LOCK(cs_main); - BOOST_FOREACH(const CTransaction& tx, txQueueLeave) { - - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << tx; - std::string stx = ssTx.str(); - Nan::MaybeLocal txBuffer = Nan::CopyBuffer((char *)stx.c_str(), stx.size()); - - uint256 hash = tx.GetHash(); - - Local obj = New(); - - Nan::Set(obj, New("buffer").ToLocalChecked(), txBuffer.ToLocalChecked()); - Nan::Set(obj, New("hash").ToLocalChecked(), New(hash.GetHex()).ToLocalChecked()); - - results->Set(arrayIndex, obj); - arrayIndex++; - } - - const unsigned argc = 1; - Local argv[argc] = { - Local::New(isolate, results) - }; - - Local cb = txmonleave_callback.Get(isolate); - - cb->Call(isolate->GetCurrentContext()->Global(), argc, argv); - - txQueueLeave.clear(); - -} -static bool -queueTxLeave(const CTransaction& tx) { - LOCK(cs_main); - txQueueLeave.push_back(tx); - uv_async_send(&txmonleave_async); - return true; -} - -/** - * Functions - */ - -NAN_METHOD(OnTipUpdate) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - - async_tip_update_data *req = new async_tip_update_data(); - - Local callback = Local::Cast(info[0]); - req->callback.Reset(isolate, callback); - req->req.data = req; - req->isolate = isolate; - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_tip_update, - (uv_after_work_cb)async_tip_update_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -static void -async_tip_update(uv_work_t *req) { - async_tip_update_data *data = reinterpret_cast(req->data); - - size_t lastHeight = chainActive.Height(); - - while(lastHeight == (size_t)chainActive.Height() && !shutdown_complete) { - usleep(1E6); - } - - data->result = chainActive.Height(); - -} - -static void -async_tip_update_after(uv_work_t *r) { - async_tip_update_data *req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - Local cb = Local::New(isolate, req->callback); - - Nan::TryCatch try_catch; - Local result = Undefined(isolate); - - if (!shutdown_complete) { - result = New(req->result); - } - Local argv[1] = { - Local::New(isolate, result) - }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - - req->callback.Reset(); -} - -NAN_METHOD(OnBlocksReady) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - - async_block_ready_data *req = new async_block_ready_data(); - req->err_msg = std::string(""); - req->result = std::string(""); - req->req.data = req; - req->isolate = isolate; - - Local callback = Local::Cast(info[0]); - req->callback.Reset(isolate, callback); - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_blocks_ready, - (uv_after_work_cb)async_blocks_ready_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -/** - * async_start_node() - * Call start_node() and start all our boost threads. - */ - -static void -async_blocks_ready(uv_work_t *req) { - async_block_ready_data *data = reinterpret_cast(req->data); - data->result = std::string(""); - - while(!chainActive.Tip()) { - usleep(1E6); - } - - CBlockIndex* tip = chainActive.Tip(); - uint256 tipHash = tip->GetBlockHash(); - - // Wait to be able to query for blocks by hash - while(mapBlockIndex.count(tipHash) == 0) { - usleep(1E6); - } - - // Wait for chainActive to be able to get the hash - // for the genesis block for querying blocks by height - while(chainActive[0] == NULL) { - usleep(1E6); - } - - //If the wallet is enabled, then we should make sure we can load it -#ifdef ENABLE_WALLET - while(pwalletMain == NULL || RPCIsInWarmup(NULL)) { - usleep(1E6); - } -#endif - - // Wait until we can get a lock on cs_main - // And therefore ready to be able to quickly - // query for transactions from the mempool. - LOCK(cs_main); - { - return; - } - -} - -static void -async_blocks_ready_after(uv_work_t *r) { - async_block_ready_data* req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - Local argv[2] = { - v8::Null(isolate), - Local::New(isolate, New(req->result).ToLocalChecked()) - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - - req->callback.Reset(); -} - -/** - * StartBitcoind() - * bitcoind.start(callback) - * Start the bitcoind node with AppInit2() on a separate thread. - */ -NAN_METHOD(StartBitcoind) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - - Local callback; - std::string datadir = std::string(""); - bool rpc = false; - bool testnet = false; - bool regtest = false; - bool txindex = false; - - if (info.Length() >= 2 && info[0]->IsObject() && info[1]->IsFunction()) { - Local options = Local::Cast(info[0]); - if (options->Get(New("datadir").ToLocalChecked())->IsString()) { - String::Utf8Value datadir_(options->Get(New("datadir").ToLocalChecked())->ToString()); - datadir = std::string(*datadir_); - } - if (options->Get(New("rpc").ToLocalChecked())->IsBoolean()) { - rpc = options->Get(New("rpc").ToLocalChecked())->ToBoolean()->IsTrue(); - } - if (options->Get(New("network").ToLocalChecked())->IsString()) { - String::Utf8Value network_(options->Get(New("network").ToLocalChecked())->ToString()); - std::string network = std::string(*network_); - if (network == "testnet") { - testnet = true; - } else if (network == "regtest") { - regtest = true; - } - } - if (options->Get(New("txindex").ToLocalChecked())->IsBoolean()) { - txindex = options->Get(New("txindex").ToLocalChecked())->ToBoolean()->IsTrue(); - } - callback = Local::Cast(info[1]); - } else if (info.Length() >= 2 - && (info[0]->IsUndefined() || info[0]->IsNull()) - && info[1]->IsFunction()) { - callback = Local::Cast(info[1]); - } else if (info.Length() >= 1 && info[0]->IsFunction()) { - callback = Local::Cast(info[0]); - } else { - return ThrowError( - "Usage: bitcoind.start(callback)"); - } - - // - // Run bitcoind's StartNode() on a separate thread. - // - - async_node_data *req = new async_node_data(); - req->err_msg = std::string(""); - req->result = std::string(""); - req->datadir = datadir; - req->rpc = rpc; - req->testnet = testnet; - req->regtest = regtest; - req->txindex = txindex; - - req->isolate = isolate; - req->callback.Reset(isolate, callback); - req->req.data = req; - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_start_node, - (uv_after_work_cb)async_start_node_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -/** - * async_start_node() - * Call start_node() and start all our boost threads. - */ - -static void -async_start_node(uv_work_t *req) { - async_node_data *data = reinterpret_cast(req->data); - if (data->datadir != "") { - g_data_dir = (char *)data->datadir.c_str(); - } else { - g_data_dir = (char *)malloc(sizeof(char) * 512); - snprintf(g_data_dir, sizeof(char) * 512, "%s/.bitcoind.js", getenv("HOME")); - } - g_rpc = (bool)data->rpc; - g_testnet = (bool)data->testnet; - g_regtest = (bool)data->regtest; - g_txindex = (bool)data->txindex; - tcgetattr(STDIN_FILENO, &orig_termios); - start_node(); - data->result = std::string("bitcoind opened."); -} - -/** - * async_start_node_after() - * Execute our callback. - */ - -static void -async_start_node_after(uv_work_t *r) { - async_node_data *req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - Local argv[2] = { - v8::Null(isolate), - Local::New(isolate, New(req->result).ToLocalChecked()) - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - - req->callback.Reset(); -} - -/** - * start_node(void) - * Start AppInit2() on a separate thread, wait for - * Unfortunately, we need to wait for the initialization - * to unhook the signal handlers so we can use them - * from node.js in javascript. - */ - -static int -start_node(void) { - SetupEnvironment(); - - noui_connect(); - - new boost::thread(boost::bind(&start_node_thread)); - return 0; -} - -static void -start_node_thread(void) { - CScheduler scheduler; - - // Workaround for AppInit2() arg parsing. Not ideal, but it works. - int argc = 0; - char **argv = (char **)malloc((4 + 1) * sizeof(char **)); - - argv[argc] = (char *)"bitcoind"; - argc++; - - if (g_data_dir) { - const int argl = 9 + strlen(g_data_dir) + 1; - char *arg = (char *)malloc(sizeof(char) * argl); - int w = snprintf(arg, argl, "-datadir=%s", g_data_dir); - if (w >= 10 && w <= argl) { - arg[w] = '\0'; - argv[argc] = arg; - argc++; - } else { - if (set_cooked()) { - fprintf(stderr, "bitcoind.js: Bad -datadir value.\n"); - } - } - } - - if (g_rpc) { - argv[argc] = (char *)"-server"; - argc++; - } - - if (g_testnet) { - argv[argc] = (char *)"-testnet"; - argc++; - } - - if (g_regtest) { - argv[argc] = (char *)"-regtest"; - argc++; - } - - argv[argc] = (char *)"-txindex"; - argc++; - - argv[argc] = NULL; - - bool fRet = false; - try { - ParseParameters((const int)argc, (const char **)argv); - - if (!boost::filesystem::is_directory(GetDataDir(false))) { - if (set_cooked()) { - fprintf(stderr, - "bitcoind.js: Specified data directory \"%s\" does not exist.\n", - mapArgs["-datadir"].c_str()); - } - shutdown_complete = true; - _exit(1); - return; - } - - try { - ReadConfigFile(mapArgs, mapMultiArgs); - } catch(std::exception &e) { - if (set_cooked()) { - fprintf(stderr, - "bitcoind.js: Error reading configuration file: %s\n", e.what()); - } - shutdown_complete = true; - _exit(1); - return; - } - - if (!SelectParamsFromCommandLine()) { - if (set_cooked()) { - fprintf(stderr, - "bitcoind.js: Invalid combination of -regtest and -testnet.\n"); - } - shutdown_complete = true; - _exit(1); - return; - } - - CreatePidFile(GetPidFile(), getpid()); - - fRet = AppInit2(threadGroup, scheduler); - - } catch (std::exception& e) { - if (set_cooked()) { - fprintf(stderr, "bitcoind.js: AppInit2(): std::exception\n"); - } - } catch (...) { - if (set_cooked()) { - fprintf(stderr, "bitcoind.js: AppInit2(): other exception\n"); - } - } - - if (!fRet) - { - threadGroup.interrupt_all(); - } else { - WaitForShutdown(&threadGroup); - } - Shutdown(); - shutdown_complete = true; - -} - -/** - * StopBitcoind() - * bitcoind.stop(callback) - */ - -NAN_METHOD(StopBitcoind) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - - if (info.Length() < 1 || !info[0]->IsFunction()) { - return ThrowError( - "Usage: bitcoind.stop(callback)"); - } - - Local callback = Local::Cast(info[0]); - - // - // Run bitcoind's StartShutdown() on a separate thread. - // - - async_node_data *req = new async_node_data(); - req->err_msg = std::string(""); - req->result = std::string(""); - req->callback.Reset(isolate, callback); - req->req.data = req; - req->isolate = isolate; - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_stop_node, - (uv_after_work_cb)async_stop_node_after); - - assert(status == 0); - info.GetReturnValue().Set(Null()); - -} - -/** - * async_stop_node() - * Call StartShutdown() to join the boost threads, which will call Shutdown() - * and set shutdown_complete to true to notify the main node.js thread. - */ - -static void -async_stop_node(uv_work_t *req) { - async_node_data *data = reinterpret_cast(req->data); - - StartShutdown(); - - while(!shutdown_complete) { - usleep(1E6); - } - data->result = std::string("bitcoind shutdown."); -} - -/** - * async_stop_node_after() - * Execute our callback. - */ - -static void -async_stop_node_after(uv_work_t *r) { - async_node_data* req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - Local argv[2] = { - Local::New(isolate, Null()), - Local::New(isolate, New(req->result).ToLocalChecked()) - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - req->callback.Reset(); -} - -/** - * GetBlock() - * bitcoind.getBlock([blockhash,blockheight], callback) - * Read any block from disk asynchronously. - */ - -NAN_METHOD(GetBlock) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - if (info.Length() < 2 - || (!info[0]->IsString() && !info[0]->IsNumber()) - || !info[1]->IsFunction()) { - return ThrowError( - "Usage: bitcoind.getBlock([blockhash,blockheight], callback)"); - } - - async_block_data *req = new async_block_data(); - - if (info[0]->IsNumber()) { - int64_t height = info[0]->IntegerValue(); - req->err_msg = std::string(""); - req->height = height; - } else { - std::string hash = *Nan::Utf8String(info[0]); - req->err_msg = std::string(""); - req->hash = uint256S(hash); - req->height = -1; - } - - Local callback = Local::Cast(info[1]); - req->req.data = req; - req->isolate = isolate; - req->callback.Reset(isolate, callback); - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_get_block, - (uv_after_work_cb)async_get_block_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -static void -async_get_block(uv_work_t *req) { - async_block_data* data = reinterpret_cast(req->data); - - CBlockIndex* pblockindex; - - if (data->height != -1) { - pblockindex = chainActive[data->height]; - if (pblockindex == NULL) { - data->err_msg = std::string("Block not found."); - return; - } - } else { - if (mapBlockIndex.count(data->hash) == 0) { - data->err_msg = std::string("Block not found."); - return; - } else { - pblockindex = mapBlockIndex[data->hash]; - } - } - - const CDiskBlockPos& pos = pblockindex->GetBlockPos(); - - // We can read directly from the file, and pass that, we don't need to - // deserialize the entire block only for it to then be serialized - // and then deserialized again in JavaScript - - // Open history file to read - CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) { - data->err_msg = std::string("ReadBlockFromDisk: OpenBlockFile failed"); - return; - } - - // Get the actual file, seeked position and rewind a uint32_t - FILE* blockFile = filein.release(); - long int filePos = ftell(blockFile); - fseek(blockFile, filePos - sizeof(uint32_t), SEEK_SET); - - // Read the size of the block - uint32_t size = 0; - fread(&size, sizeof(uint32_t), 1, blockFile); - - // Read block - char* buffer = (char *)malloc(sizeof(char) * size); - fread((void *)buffer, sizeof(char), size, blockFile); - fclose(blockFile); - - data->buffer = buffer; - data->size = size; - data->cblock_index = pblockindex; - -} - -static void -async_get_block_after(uv_work_t *r) { - async_block_data* req = reinterpret_cast(r->data); - Isolate *isolate = req->isolate; - HandleScope scope(isolate); - - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - - Nan::MaybeLocal rawNodeBuffer = Nan::NewBuffer(req->buffer, req->size); - - Local argv[2] = { - Local::New(isolate, Null()), - rawNodeBuffer.ToLocalChecked() - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - - req->callback.Reset(); -} - -/** - * GetTransaction() - * bitcoind.getTransaction(txid, queryMempool, callback) - * Read any transaction from disk asynchronously. - */ - -NAN_METHOD(GetTransaction) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - if (info.Length() < 3 - || !info[0]->IsString() - || !info[1]->IsBoolean() - || !info[2]->IsFunction()) { - return ThrowError( - "Usage: daemon.getTransaction(txid, queryMempool, callback)"); - } - - std::string txid = *Nan::Utf8String(info[0]); - - bool queryMempool = info[1]->BooleanValue(); - Local callback = Local::Cast(info[2]); - - async_tx_data *req = new async_tx_data(); - - req->err_msg = std::string(""); - req->txid = txid; - req->queryMempool = queryMempool; - req->isolate = isolate; - req->req.data = req; - req->callback.Reset(isolate, callback); - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_get_tx, - (uv_after_work_cb)async_get_tx_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -static void -async_get_tx(uv_work_t *req) { - async_tx_data* data = reinterpret_cast(req->data); - - uint256 blockhash; - uint256 hash = uint256S(data->txid); - CTransaction ctx; - - if (data->queryMempool) { - LOCK(cs_main); - { - if (mempool.lookup(hash, ctx)) - { - data->ctx = ctx; - return; - } - } - } - - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - - if (file.IsNull()) { - data->err_msg = std::string("%s: OpenBlockFile failed", __func__); - return; - } - - const int HEADER_SIZE = sizeof(int32_t) + sizeof(uint32_t) * 3 + sizeof(char) * 64; - - try { - fseek(file.Get(), postx.nTxOffset + HEADER_SIZE, SEEK_CUR); - file >> ctx; - data->ctx = ctx; - } catch (const std::exception& e) { - data->err_msg = std::string("Deserialize or I/O error - %s", __func__); - return; - } - - } - -} - -static void -async_get_tx_after(uv_work_t *r) { - async_tx_data* req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - - CTransaction ctx = req->ctx; - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - - if (!ctx.IsNull()) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << ctx; - std::string stx = ssTx.str(); - Nan::MaybeLocal result = Nan::CopyBuffer((char *)stx.c_str(), stx.size()); - Local argv[2] = { - Local::New(isolate, Null()), - result.ToLocalChecked() - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - - } else { - Local argv[2] = { - Local::New(isolate, Null()), - Local::New(isolate, Null()) - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - - } - - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - - req->callback.Reset(); -} - -/** - * GetTransactionWithBlockInfo() - * bitcoind.getTransactionWithBlockInfo(txid, queryMempool, callback) - * Read any transaction from disk asynchronously with block timestamp and height. - */ - -NAN_METHOD(GetTransactionWithBlockInfo) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - if (info.Length() < 3 - || !info[0]->IsString() - || !info[1]->IsBoolean() - || !info[2]->IsFunction()) { - return ThrowError( - "Usage: bitcoind.getTransactionWithBlockInfo(txid, queryMempool, callback)"); - } - - String::Utf8Value txid_(info[0]->ToString()); - bool queryMempool = info[1]->BooleanValue(); - Local callback = Local::Cast(info[2]); - - async_tx_data *req = new async_tx_data(); - - req->err_msg = std::string(""); - req->txid = std::string(""); - - std::string txid = std::string(*txid_); - - req->txid = txid; - req->queryMempool = queryMempool; - req->req.data = req; - req->isolate = isolate; - req->callback.Reset(isolate, callback); - - int status = uv_queue_work(uv_default_loop(), - &req->req, async_get_tx_and_info, - (uv_after_work_cb)async_get_tx_and_info_after); - - assert(status == 0); - - info.GetReturnValue().Set(Null()); -} - -static void -async_get_tx_and_info(uv_work_t *req) { - async_tx_data* data = reinterpret_cast(req->data); - - uint256 hash = uint256S(data->txid); - uint256 blockHash; - CTransaction ctx; - - if (data->queryMempool) { - LOCK(mempool.cs); - map::const_iterator i = mempool.mapTx.find(hash); - if (i != mempool.mapTx.end()) { - data->ctx = i->second.GetTx(); - data->nTime = i->second.GetTime(); - data->height = -1; - return; - } - } - - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - - if (file.IsNull()) { - data->err_msg = std::string("%s: OpenBlockFile failed", __func__); - return; - } - - CBlockHeader blockHeader; - - try { - // Read header first to get block timestamp and hash - file >> blockHeader; - blockHash = blockHeader.GetHash(); - data->blockHash = blockHash.GetHex(); - data->nTime = blockHeader.nTime; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> ctx; - data->ctx = ctx; - } catch (const std::exception& e) { - data->err_msg = std::string("Deserialize or I/O error - %s", __func__); - return; - } - - // get block height - CBlockIndex* blockIndex; - - if (mapBlockIndex.count(blockHash) == 0) { - data->height = -1; - } else { - blockIndex = mapBlockIndex[blockHash]; - if (!chainActive.Contains(blockIndex)) { - data->height = -1; - } else { - data->height = blockIndex->nHeight; - } - } - - } - -} - -static void -async_get_tx_and_info_after(uv_work_t *r) { - async_tx_data* req = reinterpret_cast(r->data); - Isolate* isolate = req->isolate; - HandleScope scope(isolate); - - CTransaction ctx = req->ctx; - Nan::TryCatch try_catch; - Local cb = Local::New(isolate, req->callback); - Local obj = New(); - - if (req->err_msg != "") { - Local err = Exception::Error(New(req->err_msg).ToLocalChecked()); - Local argv[1] = { err }; - cb->Call(isolate->GetCurrentContext()->Global(), 1, argv); - } else { - - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << ctx; - std::string stx = ssTx.str(); - Nan::MaybeLocal rawNodeBuffer = Nan::CopyBuffer((char *)stx.c_str(), stx.size()); - - Nan::Set(obj, New("blockHash").ToLocalChecked(), New(req->blockHash).ToLocalChecked()); - Nan::Set(obj, New("height").ToLocalChecked(), New(req->height)); - Nan::Set(obj, New("timestamp").ToLocalChecked(), New(req->nTime)); - Nan::Set(obj, New("buffer").ToLocalChecked(), rawNodeBuffer.ToLocalChecked()); - - Local argv[2] = { - Local::New(isolate, Null()), - obj - }; - cb->Call(isolate->GetCurrentContext()->Global(), 2, argv); - } - if (try_catch.HasCaught()) { - Nan::FatalException(try_catch); - } - req->callback.Reset(); -} - -/** - * IsSpent() - * bitcoind.isSpent() - * Determine if an outpoint is spent - */ -NAN_METHOD(IsSpent) { - if (info.Length() > 2) { - return ThrowError( - "Usage: bitcoind.isSpent(txid, outputIndex)"); - } - - String::Utf8Value arg(info[0]->ToString()); - std::string argStr = std::string(*arg); - const uint256 txid = uint256S(argStr); - int outputIndex = info[1]->IntegerValue(); - - { - LOCK(mempool.cs); - CCoinsView dummy; - CCoinsViewCache view(&dummy); - - CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); - view.SetBackend(viewMemPool); - - if (view.HaveCoins(txid)) { - const CCoins* coins = view.AccessCoins(txid); - if (coins && coins->IsAvailable(outputIndex)) { - info.GetReturnValue().Set(New(false)); - return; - } - } - } - info.GetReturnValue().Set(New(true)); -}; - -/** - * GetBlockIndex() - * bitcoind.getBlockIndex() - * Get index information about a block by hash including: - * - the total amount of work (expected number of hashes) in the chain up to - * and including this block. - * - the previous hash of the block - */ -NAN_METHOD(GetBlockIndex) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - CBlockIndex* blockIndex; - - if (info[0]->IsNumber()) { - int64_t height = info[0]->IntegerValue(); - blockIndex = chainActive[height]; - - if (blockIndex == NULL) { - info.GetReturnValue().Set(Null()); - return; - } - - } else { - String::Utf8Value hash_(info[0]->ToString()); - std::string hashStr = std::string(*hash_); - uint256 hash = uint256S(hashStr); - if (mapBlockIndex.count(hash) == 0) { - info.GetReturnValue().Set(Null()); - } else { - blockIndex = mapBlockIndex[hash]; - } - } - - Local obj = New(); - - arith_uint256 cw = blockIndex->nChainWork; - CBlockIndex* prevBlockIndex = blockIndex->pprev; - if (&prevBlockIndex->phashBlock != 0) { - const uint256* prevHash = prevBlockIndex->phashBlock; - Nan::Set(obj, New("prevHash").ToLocalChecked(), New(prevHash->GetHex()).ToLocalChecked()); - } else { - Nan::Set(obj, New("prevHash").ToLocalChecked(), Null()); - } - - Nan::Set(obj, New("hash").ToLocalChecked(), New(blockIndex->phashBlock->GetHex()).ToLocalChecked()); - Nan::Set(obj, New("chainWork").ToLocalChecked(), New(cw.GetHex()).ToLocalChecked()); - - Nan::Set(obj, New("height").ToLocalChecked(), New(blockIndex->nHeight)); - - info.GetReturnValue().Set(obj); -}; - - -/** - * IsMainChain() - * bitcoind.isMainChain() - * - * @param {string} - block hash - * @returns {boolean} - True if the block is in the main chain. False if it is an orphan. - */ -NAN_METHOD(IsMainChain) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - CBlockIndex* blockIndex; - - String::Utf8Value hash_(info[0]->ToString()); - std::string hashStr = std::string(*hash_); - uint256 hash = uint256S(hashStr); - if (mapBlockIndex.count(hash) == 0) { - info.GetReturnValue().Set(Null()); - } else { - blockIndex = mapBlockIndex[hash]; - } - - if (chainActive.Contains(blockIndex)) { - info.GetReturnValue().Set(New(true)); - } else { - info.GetReturnValue().Set(New(false)); - } -} - -/** - * GetInfo() - * bitcoind.getInfo() - * Get miscellaneous information - */ - -NAN_METHOD(GetInfo) { - if (info.Length() > 0) { - return ThrowError( - "Usage: bitcoind.getInfo()"); - } - - Local obj = New(); - - proxyType proxy; - GetProxy(NET_IPV4, proxy); - - Nan::Set(obj, New("version").ToLocalChecked(), New(CLIENT_VERSION)); - Nan::Set(obj, New("protocolversion").ToLocalChecked(), New(PROTOCOL_VERSION)); - Nan::Set(obj, New("blocks").ToLocalChecked(), New((int)chainActive.Height())->ToInt32()); - Nan::Set(obj, New("timeoffset").ToLocalChecked(), New(GetTimeOffset())); - Nan::Set(obj, New("connections").ToLocalChecked(), New((int)vNodes.size())->ToInt32()); - Nan::Set(obj, New("difficulty").ToLocalChecked(), New((double)GetDifficulty())); - Nan::Set(obj, New("testnet").ToLocalChecked(), New(Params().NetworkIDString() == "test")); - Nan::Set(obj, New("network").ToLocalChecked(), New(Params().NetworkIDString()).ToLocalChecked()); - Nan::Set(obj, New("relayfee").ToLocalChecked(), New(::minRelayTxFee.GetFeePerK())); // double - Nan::Set(obj, New("errors").ToLocalChecked(), New(GetWarnings("statusbar")).ToLocalChecked()); - - info.GetReturnValue().Set(obj); -} - -/** - * Estimate Fee - * @blocks {number} - The number of blocks until confirmed - */ - -NAN_METHOD(EstimateFee) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - int nBlocks = info[0]->NumberValue(); - if (nBlocks < 1) { - nBlocks = 1; - } - - CFeeRate feeRate = mempool.estimateFee(nBlocks); - - if (feeRate == CFeeRate(0)) { - info.GetReturnValue().Set(New(-1.0)); - return; - } - - CAmount nFee = feeRate.GetFeePerK(); - - info.GetReturnValue().Set(New(nFee)); - -} - -/** - * Send Transaction - * bitcoind.sendTransaction() - * Will add a transaction to the mempool and broadcast to connected peers. - * @param {string} - The serialized hex string of the transaction. - * @param {boolean} - Skip absurdly high fee checks - */ -NAN_METHOD(SendTransaction) { - Isolate* isolate = Isolate::GetCurrent(); - HandleScope scope(isolate); - - LOCK(cs_main); - - // Decode the transaction - v8::String::Utf8Value param1(info[0]->ToString()); - std::string *input = new std::string(*param1); - CTransaction tx; - if (!DecodeHexTx(tx, *input)) { - return ThrowError("TX decode failed"); - } - uint256 hashTx = tx.GetHash(); - - // Skip absurdly high fee check - bool allowAbsurdFees = false; - if (info.Length() > 1) { - allowAbsurdFees = info[1]->BooleanValue(); - } - - CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); - bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; - if (!fHaveMempool && !fHaveChain) { - CValidationState state; - bool fMissingInputs; - - // Attempt to add the transaction to the mempool - if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, !allowAbsurdFees)) { - if (state.IsInvalid()) { - return ThrowError((boost::lexical_cast(state.GetRejectCode()) + ": " + state.GetRejectReason()).c_str()); - } else { - if (fMissingInputs) { - return ThrowError("Missing inputs"); - } - return ThrowError(state.GetRejectReason().c_str()); - } - } - } else if (fHaveChain) { - return ThrowError("transaction already in block chain"); - } - - // Relay the transaction connect peers - RelayTransaction(tx); - - info.GetReturnValue().Set(Local::New(isolate, New(hashTx.GetHex()).ToLocalChecked())); -} - -/** - * GetMempoolTransactions - * bitcoind.getMempoolTransactions() - * Will return an array of transaction buffers. - */ -NAN_METHOD(GetMempoolTransactions) { - Isolate* isolate = info.GetIsolate(); - HandleScope scope(isolate); - - Local transactions = Array::New(isolate); - int arrayIndex = 0; - - { - LOCK(mempool.cs); - - // Iterate through the entire mempool - std::map mapTx = mempool.mapTx; - - for(std::map::iterator it = mapTx.begin(); - it != mapTx.end(); - it++) { - CTxMemPoolEntry entry = it->second; - const CTransaction tx = entry.GetTx(); - CDataStream dataStreamTx(SER_NETWORK, PROTOCOL_VERSION); - dataStreamTx << tx; - std::string txString = dataStreamTx.str(); - Nan::MaybeLocal txBuffer = Nan::CopyBuffer((char *)txString.c_str(), txString.size()); - transactions->Set(arrayIndex, txBuffer.ToLocalChecked()); - arrayIndex++; - } - } - - info.GetReturnValue().Set(transactions); - -} - -/** - * AddMempoolUncheckedTransaction - */ -NAN_METHOD(AddMempoolUncheckedTransaction) { - v8::String::Utf8Value param1(info[0]->ToString()); - std::string *input = new std::string(*param1); - - CTransaction tx; - if (!DecodeHexTx(tx, *input)) { - return ThrowError("could not decode tx"); - } - bool added = mempool.addUnchecked(tx.GetHash(), CTxMemPoolEntry(tx, 0, 0, 0.0, 1)); - info.GetReturnValue().Set(New(added)); - -} - -/** - * Helpers - */ - -static bool -set_cooked(void) { - uv_tty_t tty; - tty.mode = 1; - tty.orig_termios = orig_termios; - - if (!uv_tty_set_mode(&tty, 0)) { - printf("\x1b[H\x1b[J"); - return true; - } - - return false; -} - -/** - * Init() - * Initialize the singleton object known as bitcoind. - */ -NAN_MODULE_INIT(init) { - Nan::Set(target, New("start").ToLocalChecked(), GetFunction(New(StartBitcoind)).ToLocalChecked()); - Nan::Set(target, New("onBlocksReady").ToLocalChecked(), GetFunction(New(OnBlocksReady)).ToLocalChecked()); - Nan::Set(target, New("onTipUpdate").ToLocalChecked(), GetFunction(New(OnTipUpdate)).ToLocalChecked()); - Nan::Set(target, New("stop").ToLocalChecked(), GetFunction(New(StopBitcoind)).ToLocalChecked()); - Nan::Set(target, New("getBlock").ToLocalChecked(), GetFunction(New(GetBlock)).ToLocalChecked()); - Nan::Set(target, New("getTransaction").ToLocalChecked(), GetFunction(New(GetTransaction)).ToLocalChecked()); - Nan::Set(target, New("getTransactionWithBlockInfo").ToLocalChecked(), GetFunction(New(GetTransactionWithBlockInfo)).ToLocalChecked()); - Nan::Set(target, New("getInfo").ToLocalChecked(), GetFunction(New(GetInfo)).ToLocalChecked()); - Nan::Set(target, New("isSpent").ToLocalChecked(), GetFunction(New(IsSpent)).ToLocalChecked()); - Nan::Set(target, New("getBlockIndex").ToLocalChecked(), GetFunction(New(GetBlockIndex)).ToLocalChecked()); - Nan::Set(target, New("isMainChain").ToLocalChecked(), GetFunction(New(IsMainChain)).ToLocalChecked()); - Nan::Set(target, New("getMempoolTransactions").ToLocalChecked(), GetFunction(New(GetMempoolTransactions)).ToLocalChecked()); - Nan::Set(target, New("addMempoolUncheckedTransaction").ToLocalChecked(), GetFunction(New(AddMempoolUncheckedTransaction)).ToLocalChecked()); - Nan::Set(target, New("sendTransaction").ToLocalChecked(), GetFunction(New(SendTransaction)).ToLocalChecked()); - Nan::Set(target, New("estimateFee").ToLocalChecked(), GetFunction(New(EstimateFee)).ToLocalChecked()); - Nan::Set(target, New("startTxMon").ToLocalChecked(), GetFunction(New(StartTxMon)).ToLocalChecked()); - Nan::Set(target, New("startTxMonLeave").ToLocalChecked(), GetFunction(New(StartTxMonLeave)).ToLocalChecked()); - Nan::Set(target, New("syncPercentage").ToLocalChecked(), GetFunction(New(SyncPercentage)).ToLocalChecked()); - Nan::Set(target, New("isSynced").ToLocalChecked(), GetFunction(New(IsSynced)).ToLocalChecked()); - Nan::Set(target, New("getBestBlockHash").ToLocalChecked(), GetFunction(New(GetBestBlockHash)).ToLocalChecked()); - Nan::Set(target, New("getNextBlockHash").ToLocalChecked(), GetFunction(New(GetNextBlockHash)).ToLocalChecked()); -} - -NODE_MODULE(libbitcoind, init); diff --git a/src/libbitcoind.h b/src/libbitcoind.h deleted file mode 100644 index 90d2ca97..00000000 --- a/src/libbitcoind.h +++ /dev/null @@ -1,19 +0,0 @@ -#include "main.h" -#include "addrman.h" -#include "alert.h" -#include "base58.h" -#include "init.h" -#include "noui.h" -#include "rpcserver.h" -#include "txdb.h" -#include -#include -#include -#include "nan.h" -#include "scheduler.h" -#include "core_io.h" -#include "script/bitcoinconsensus.h" -#include "consensus/validation.h" -#ifdef ENABLE_WALLET -#include "wallet/wallet.h" -#endif From b69d8483521a1dae9f713e99b31e0e333a418f7c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 25 Mar 2016 14:17:22 -0400 Subject: [PATCH 061/299] bitcoind: add lru caching for results --- lib/services/bitcoind.js | 265 ++++++++++++++++++++++++++------------- package.json | 3 +- 2 files changed, 181 insertions(+), 87 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index d74d93ea..b6cd6fdf 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -9,6 +9,7 @@ var bitcore = require('bitcore-lib'); var Address = bitcore.Address; var zmq = require('zmq'); var async = require('async'); +var LRU = require('lru-cache'); var BitcoinRPC = require('bitcoind-rpc'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; @@ -33,6 +34,18 @@ function Bitcoin(options) { this._reindex = false; this._reindexWait = 1000; Service.call(this, options); + + // caches valid until there is a new block + this.txidsCache = LRU(50000); + this.balanceCache = LRU(50000); + this.summaryCache = LRU(50000); + + // caches valid indefinetly + this.transactionCache = LRU(100000); + this.transactionInfoCache = LRU(100000); + this.blockCache = LRU(144); + this.blockHeaderCache = LRU(288); + $.checkState(this.node.datadir, 'Node is missing datadir property'); } @@ -132,6 +145,12 @@ Bitcoin.prototype._loadConfiguration = function() { }; +Bitcoin.prototype._resetCaches = function() { + this.txidsCache.reset(); + this.balanceCache.reset(); + this.summaryCache.reset(); +}; + Bitcoin.prototype._registerEventHandlers = function() { var self = this; @@ -143,6 +162,7 @@ Bitcoin.prototype._registerEventHandlers = function() { if (topicString === 'hashtx') { self.emit('tx', message.toString('hex')); } else if (topicString === 'hashblock') { + self._resetCaches(); self.tiphash = message.toString('hex'); self.client.getBlock(self.tiphash, function(err, response) { if (err) { @@ -327,17 +347,26 @@ Bitcoin.prototype.syncPercentage = function(callback) { }; Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { - // TODO keep a cache and update the cache by a range of block heights + var self = this; var addresses = [addressArg]; if (Array.isArray(addressArg)) { addresses = addressArg; } - this.client.getAddressBalance({addresses: addresses}, function(err, response) { - if (err) { - return callback(err); - } - callback(null, response.result); - }); + var cacheKey = addresses.join(''); + var balance = self.balanceCache.get(cacheKey); + if (balance) { + return setImmediate(function() { + callback(null, balance); + }); + } else { + this.client.getAddressBalance({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + self.balanceCache.set(cacheKey, response.result); + callback(null, response.result); + }); + } }; Bitcoin.prototype.getAddressUnspentOutputs = function() { @@ -345,17 +374,27 @@ Bitcoin.prototype.getAddressUnspentOutputs = function() { }; Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { - // TODO Keep a cache updated for queries + var self = this; var addresses = [addressArg]; if (Array.isArray(addressArg)) { addresses = addressArg; } - this.client.getAddressTxids({addresses: addresses}, function(err, response) { - if (err) { - return callback(err); - } - return callback(null, response.result); - }); + var cacheKey = addresses.join(''); + var txids = self.txidsCache.get(cacheKey); + if (txids) { + return setImmediate(function() { + callback(null, txids); + }); + } else { + self.client.getAddressTxids({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + response.result.reverse(); + self.txidsCache.set(cacheKey, response.result); + return callback(null, response.result); + }); + } }; Bitcoin.prototype._getConfirmationsDetail = function(transaction) { @@ -475,6 +514,19 @@ Bitcoin.prototype._getAddressStrings = function(addresses) { return addressStrings; }; +Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { + var totalCount = fullTxids.length; + var txids; + if (from >= 0 && to >= 0) { + var fromOffset = totalCount - from; + var toOffset = totalCount - to; + txids = fullTxids.slice(toOffset, fromOffset); + } else { + txids = fullTxids; + } + return txids; +}; + Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var self = this; var addresses = [addressArg]; @@ -489,6 +541,10 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { if (err) { return callback(err); } + + var totalCount = txids.length; + txids = self._paginateTxids(txids, options.from, options.to); + async.mapSeries( txids, function(txid, next) { @@ -502,7 +558,7 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { return callback(err); } callback(null, { - totalCount: txids.length, + totalCount: totalCount, items: transactions }); } @@ -514,48 +570,65 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { // TODO: optional mempool var self = this; var summary = {}; + var summaryTxids = []; + + var addresses = [addressArg]; + if (Array.isArray(addressArg)) { + addresses = addressArg; + } + + var cacheKey = addresses.join(''); if (_.isUndefined(options.queryMempool)) { options.queryMempool = true; } - function getBalance(done) { - self.getAddressBalance(addressArg, options, function(err, data) { - if (err) { - return done(err); + function querySummary() { + async.parallel([ + function getTxList(done) { + self.getAddressTxids(addressArg, options, function(err, txids) { + if (err) { + return done(err); + } + summaryTxids = txids; + summary.appearances = txids.length; + done(); + }); + }, + function getBalance(done) { + self.getAddressBalance(addressArg, options, function(err, data) { + if (err) { + return done(err); + } + summary.totalReceived = data.received; + summary.totalSpent = data.received - data.balance; + summary.balance = data.balance; + done(); + }); } - summary.totalReceived = data.received; - summary.totalSpent = data.received - data.balance; - summary.balance = data.balance; - done(); - }); - } - - function getTxList(done) { - self.getAddressTxids(addressArg, options, function(err, txids) { + ], function(err) { if (err) { - return done(err); + return callback(err); } - summary.txids = txids; - summary.appearances = txids.length; - done(); + self.summaryCache.set(cacheKey, summary); + if (!options.noTxList) { + summary.txids = summaryTxids; + } + callback(null, summary); }); } - var tasks = []; - if (!options.noBalance) { - tasks.push(getBalance); - } - if (!options.noTxList) { - tasks.push(getTxList); + if (options.noTxList) { + var summaryCache = self.summaryCache.get(cacheKey); + if (summaryCache) { + callback(null, summaryCache); + } else { + querySummary(); + } + } else { + querySummary(); } - async.parallel(tasks, function(err) { - if (err) { - return callback(err); - } - callback(null, summary); - }); }; /** @@ -564,29 +637,36 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { */ Bitcoin.prototype.getBlock = function(block, callback) { // TODO apply performance patch to the RPC method for raw data - // TODO keep a cache of results var self = this; - function queryHeader(blockhash) { + function queryBlock(blockhash) { self.client.getBlock(blockhash, false, function(err, response) { if (err) { return callback(err); } - var block = bitcore.Block.fromString(response.result); - callback(null, block); + var blockCache = bitcore.Block.fromString(response.result); + self.blockCache.set(block, blockCache); + callback(null, blockCache); }); } - if (_.isNumber(block)) { - self.client.getBlockHash(block, function(err, response) { - if (err) { - return callback(err); - } - var blockhash = response.result; - queryHeader(blockhash); + var cachedBlock = self.blockCache.get(block); + if (cachedBlock) { + return setImmediate(function() { + callback(null, cachedBlock); }); } else { - queryHeader(block); + if (_.isNumber(block)) { + self.client.getBlockHash(block, function(err, response) { + if (err) { + return callback(err); + } + var blockhash = response.result; + queryBlock(blockhash); + }); + } else { + queryBlock(block); + } } }; @@ -614,7 +694,6 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { * @returns {Object} */ Bitcoin.prototype.getBlockHeader = function(block, callback) { - // TODO keep a cache of queries var self = this; function queryHeader(blockhash) { @@ -683,18 +762,26 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { * @param {Function} callback */ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { - // TODO keep an LRU cache available of transactions - this.client.getRawTransaction(txid, function(err, response) { - if (err) { - return callback(err); - } - if (!response.result) { - return callback(new errors.Transaction.NotFound()); - } - var tx = Transaction(); - tx.fromString(response.result); - callback(null, tx); - }); + var self = this; + var tx = self.transactionCache.get(txid); + if (tx) { + return setImmediate(function() { + callback(null, tx); + }); + } else { + self.client.getRawTransaction(txid, function(err, response) { + if (err) { + return callback(err); + } + if (!response.result) { + return callback(new errors.Transaction.NotFound()); + } + var tx = Transaction(); + tx.fromString(response.result); + self.transactionCache.set(txid, tx); + callback(null, tx); + }); + } }; /** @@ -710,22 +797,29 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { * @param {Function} callback */ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) { - // TODO keep an LRU cache available of transactions - // TODO get information from txindex as an RPC method - this.client.getRawTransaction(txid, 1, function(err, response) { - if (err) { - return callback(err); - } - if (!response.result) { - return callback(new errors.Transaction.NotFound()); - } - var tx = Transaction(); - tx.fromString(response.result.hex); - tx.__blockHash = response.result.blockhash; - tx.__height = response.result.height; - tx.__timestamp = response.result.time; - callback(null, tx); - }); + var self = this; + var tx = self.transactionInfoCache.get(txid); + if (tx) { + return setImmediate(function() { + callback(null, tx); + }); + } else { + self.client.getRawTransaction(txid, 1, function(err, response) { + if (err) { + return callback(err); + } + if (!response.result) { + return callback(new errors.Transaction.NotFound()); + } + var tx = Transaction(); + tx.fromString(response.result.hex); + tx.__blockHash = response.result.blockhash; + tx.__height = response.result.height; + tx.__timestamp = response.result.time; + self.transactionInfoCache.set(txid, tx); + callback(null, tx); + }); + } }; /** @@ -733,7 +827,6 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal * @returns {String} */ Bitcoin.prototype.getBestBlockHash = function(callback) { - // TODO keep an LRU cache available of transactions this.client.getBestBlockHash(function(err, response) { if (err) { return callback(err); diff --git a/package.json b/package.json index ef7a6b69..63dc3a15 100644 --- a/package.json +++ b/package.json @@ -41,14 +41,15 @@ "dependencies": { "async": "^1.3.0", "bindings": "^1.2.1", - "bitcore-lib": "^0.13.13", "bitcoind-rpc": "^0.3.0", + "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", "liftoff": "^2.2.0", + "lru-cache": "^4.0.1", "memdown": "^1.0.0", "mkdirp": "0.5.0", "npm": "^2.14.1", From af573b765babb3127ed9e1cb0941be74c70511f1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 25 Mar 2016 15:09:44 -0400 Subject: [PATCH 062/299] bitcoind: fix txid pagination --- lib/services/bitcoind.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b6cd6fdf..95bada78 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -515,12 +515,9 @@ Bitcoin.prototype._getAddressStrings = function(addresses) { }; Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { - var totalCount = fullTxids.length; var txids; if (from >= 0 && to >= 0) { - var fromOffset = totalCount - from; - var toOffset = totalCount - to; - txids = fullTxids.slice(toOffset, fromOffset); + txids = fullTxids.slice(from, to); } else { txids = fullTxids; } From 7d7dfe329dd68b289d9825817587eb12e78f9ff9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 31 Mar 2016 10:15:29 -0400 Subject: [PATCH 063/299] bitcoind: variable name fixes --- lib/services/bitcoind.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 95bada78..4c70f910 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -130,7 +130,7 @@ Bitcoin.prototype._loadConfiguration = function() { ); $.checkState( - this.configuration.zmqpubhashtx, + this.configuration.zmqpubhashblock, '"zmqpubhashblock" option is required to get event updates from bitcoind. ' + 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' ); @@ -632,7 +632,7 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { * Will retrieve a block as a Node.js Buffer from disk. * @param {String|Number} block - A block hash or block height number */ -Bitcoin.prototype.getBlock = function(block, callback) { +Bitcoin.prototype.getBlock = function(blockArg, callback) { // TODO apply performance patch to the RPC method for raw data var self = this; @@ -641,20 +641,20 @@ Bitcoin.prototype.getBlock = function(block, callback) { if (err) { return callback(err); } - var blockCache = bitcore.Block.fromString(response.result); - self.blockCache.set(block, blockCache); - callback(null, blockCache); + var blockObj = bitcore.Block.fromString(response.result); + self.blockCache.set(blockArg, blockObj); + callback(null, blockObj); }); } - var cachedBlock = self.blockCache.get(block); + var cachedBlock = self.blockCache.get(blockArg); if (cachedBlock) { return setImmediate(function() { callback(null, cachedBlock); }); } else { - if (_.isNumber(block)) { - self.client.getBlockHash(block, function(err, response) { + if (_.isNumber(blockArg)) { + self.client.getBlockHash(blockArg, function(err, response) { if (err) { return callback(err); } @@ -662,7 +662,7 @@ Bitcoin.prototype.getBlock = function(block, callback) { queryBlock(blockhash); }); } else { - queryBlock(block); + queryBlock(blockArg); } } From ab70aa666ef459c34e9c9fc31e633274c2e46f02 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 10:32:07 -0400 Subject: [PATCH 064/299] bitcoind: add address utxos --- lib/services/bitcoind.js | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4c70f910..edf0537e 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -36,6 +36,7 @@ function Bitcoin(options) { Service.call(this, options); // caches valid until there is a new block + this.utxosCache = LRU(50000); this.txidsCache = LRU(50000); this.balanceCache = LRU(50000); this.summaryCache = LRU(50000); @@ -146,6 +147,7 @@ Bitcoin.prototype._loadConfiguration = function() { }; Bitcoin.prototype._resetCaches = function() { + this.utxosCache.reset(); this.txidsCache.reset(); this.balanceCache.reset(); this.summaryCache.reset(); @@ -369,8 +371,27 @@ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { } }; -Bitcoin.prototype.getAddressUnspentOutputs = function() { - // TODO add this rpc method to bitcoind +Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callback) { + var self = this; + var addresses = [addressArg]; + if (Array.isArray(addressArg)) { + addresses = addressArg; + } + var cacheKey = addresses.join(''); + var utxos = self.utxosCache.get(cacheKey); + if (utxos) { + return setImmediate(function() { + callback(null, utxos); + }); + } else { + self.client.getAddressUtxos({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + self.utxosCache.set(cacheKey, response.result); + callback(null, response.result); + }); + } }; Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { From 60333bcb0ead979112f7e2323e30da8da006ee0e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 13:50:31 -0400 Subject: [PATCH 065/299] bitcoind: add mempool to address txid results --- lib/services/bitcoind.js | 87 ++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index edf0537e..1ef76e90 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -394,28 +394,69 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb } }; +Bitcoin.prototype._getBalanceFromMempool = function(deltas) { + var satoshis = 0; + for (var i = 0; i < deltas.length; i++) { + satoshis += deltas[i].satoshis; + } + return satoshis; +}; + +Bitcoin.prototype._getTxidsFromMempool = function(deltas) { + var mempoolTxids = []; + var mempoolTxidsKnown = {}; + for (var i = 0; i < deltas.length; i++) { + var txid = deltas[i].txid; + if (!mempoolTxidsKnown[txid]) { + mempoolTxids.push(txid); + mempoolTxidsKnown[txid] = true; + } + } + return mempoolTxids; +}; + Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { var self = this; + var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var addresses = [addressArg]; if (Array.isArray(addressArg)) { addresses = addressArg; } var cacheKey = addresses.join(''); + var mempoolTxids = []; var txids = self.txidsCache.get(cacheKey); - if (txids) { - return setImmediate(function() { - callback(null, txids); - }); - } else { - self.client.getAddressTxids({addresses: addresses}, function(err, response) { + + function finish() { + if (txids) { + var allTxids = mempoolTxids.reverse().concat(txids); + return setImmediate(function() { + callback(null, allTxids); + }); + } else { + self.client.getAddressTxids({addresses: addresses}, function(err, response) { + if (err) { + return callback(err); + } + response.result.reverse(); + self.txidsCache.set(cacheKey, response.result); + var allTxids = mempoolTxids.reverse().concat(response.result); + return callback(null, allTxids); + }); + } + } + + if (queryMempool) { + self.client.getAddressMempool({addresses: addresses}, function(err, response) { if (err) { return callback(err); } - response.result.reverse(); - self.txidsCache.set(cacheKey, response.result); - return callback(null, response.result); + mempoolTxids = self._getTxidsFromMempool(response.result); + finish(); }); + } else { + finish(); } + }; Bitcoin.prototype._getConfirmationsDetail = function(transaction) { @@ -585,10 +626,11 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { }; Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { - // TODO: optional mempool var self = this; var summary = {}; + var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var summaryTxids = []; + var mempoolTxids = []; var addresses = [addressArg]; if (Array.isArray(addressArg)) { @@ -597,14 +639,10 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var cacheKey = addresses.join(''); - if (_.isUndefined(options.queryMempool)) { - options.queryMempool = true; - } - function querySummary() { async.parallel([ function getTxList(done) { - self.getAddressTxids(addressArg, options, function(err, txids) { + self.getAddressTxids(addressArg, {queryMempool: false}, function(err, txids) { if (err) { return done(err); } @@ -623,14 +661,29 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { summary.balance = data.balance; done(); }); - } + }, + function getMempool(done) { + if (!queryMempool) { + return done(); + } + self.client.getAddressMempool({'addresses': [addressArg]}, function(err, response) { + if (err) { + return done(err); + } + mempoolTxids = self._getTxidsFromMempool(response.result); + summary.unconfirmedAppearances = mempoolTxids.length; + summary.unconfirmedBalance = self._getBalanceFromMempool(response.result); + done(); + }); + }, ], function(err) { if (err) { return callback(err); } self.summaryCache.set(cacheKey, summary); if (!options.noTxList) { - summary.txids = summaryTxids; + var allTxids = mempoolTxids.reverse().concat(summaryTxids); + summary.txids = allTxids; } callback(null, summary); }); From b473b65207f4d6e70ec514d57b3487e367b53256 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 14:50:49 -0400 Subject: [PATCH 066/299] bitcoind: fix tx event to include tx buffer --- lib/services/bitcoind.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 1ef76e90..0f12f10b 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -125,9 +125,9 @@ Bitcoin.prototype._loadConfiguration = function() { ); $.checkState( - this.configuration.zmqpubhashtx, - '"zmqpubhashtx" option is required to get event updates from bitcoind. ' + - 'Please add "zmqpubhashtx=tcp://127.0.0.1:" to your configuration and restart' + this.configuration.zmqpubrawtx, + '"zmqpubrawtx" option is required to get event updates from bitcoind. ' + + 'Please add "zmqpubrawtx=tcp://127.0.0.1:" to your configuration and restart' ); $.checkState( @@ -157,12 +157,12 @@ Bitcoin.prototype._registerEventHandlers = function() { var self = this; this.zmqSubSocket.subscribe('hashblock'); - this.zmqSubSocket.subscribe('hashtx'); + this.zmqSubSocket.subscribe('rawtx'); this.zmqSubSocket.on('message', function(topic, message) { var topicString = topic.toString('utf8'); - if (topicString === 'hashtx') { - self.emit('tx', message.toString('hex')); + if (topicString === 'rawtx') { + self.emit('tx', message); } else if (topicString === 'hashblock') { self._resetCaches(); self.tiphash = message.toString('hex'); @@ -291,7 +291,7 @@ Bitcoin.prototype.start = function(callback) { }); self.zmqSubSocket.monitor(500, 0); - self.zmqSubSocket.connect(self.configuration.zmqpubhashtx); + self.zmqSubSocket.connect(self.configuration.zmqpubrawtx); if (self._reindex) { var interval = setInterval(function() { From 9409374fbe513a8525f0fca1f9da55db91e32900 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 16:12:19 -0400 Subject: [PATCH 067/299] bitcoind: fix multiple addresses for address history --- lib/services/bitcoind.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 0f12f10b..8f864345 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -589,6 +589,9 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var self = this; var addresses = [addressArg]; + if (_.isArray(addressArg)) { + addresses = addressArg; + } if (addresses.length > this.maxAddressesQuery) { return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } From 5932b34a1ffb0baa8d18c13b7ae94e669cb8876b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Apr 2016 11:43:02 -0400 Subject: [PATCH 068/299] bitcoind: set height when starting --- lib/services/bitcoind.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 8f864345..765a8550 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -266,14 +266,22 @@ Bitcoin.prototype.start = function(callback) { pass: self.configuration.rpcpassword }); - self.client.getInfo(function(err) { + self.client.getBestBlockHash(function(err, response) { if (err) { if (!(err instanceof Error)) { log.warn(err.message); } return done(new Error('Could not connect to bitcoind RPC')); } - done(); + self.client.getBlock(response.result, function(err, response) { + if (err) { + return done(err); + } + self.height = response.result.height; + $.checkState(self.height >= 0); + self.emit('tip', self.height); + done(); + }); }); }, function ready(err, result) { From 0f24dd5f49ffbbe6d569227fe1830575fd5c65cc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Apr 2016 18:02:22 -0400 Subject: [PATCH 069/299] config: update configuration options for exec path - config options for bitcoind to specify exec path of bitcoind - config options to connect to multiple bitcoind processes - systemd and upstart preferred methods to daemonize --- lib/node.js | 3 - lib/scaffold/default-base-config.js | 9 +- lib/scaffold/default-config.js | 11 +- lib/scaffold/start.js | 42 --- lib/services/bitcoind.js | 397 +++++++++++++++++++--------- package.json | 2 - 6 files changed, 286 insertions(+), 178 deletions(-) diff --git a/lib/node.js b/lib/node.js index 737cd307..be4ec189 100644 --- a/lib/node.js +++ b/lib/node.js @@ -28,7 +28,6 @@ var errors = require('./errors'); * * @param {Object} config - The configuration of the node * @param {Array} config.services - The array of services - * @param {String} config.datadir - The directory for data (e.g. bitcoind datadir) * @param {Number} config.port - The HTTP port for services * @param {Boolean} config.https - Enable https * @param {Object} config.httpsOptions - Options for https @@ -52,8 +51,6 @@ function Node(config) { $.checkArgument(Array.isArray(config.services)); this._unloadedServices = config.services; } - $.checkState(config.datadir, 'Node config expects "datadir"'); - this.datadir = config.datadir; this.port = config.port; this.https = config.https; this.httpsOptions = config.httpsOptions; diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index 6bb0babc..6cc5b316 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -15,10 +15,15 @@ function getDefaultBaseConfig(options) { return { path: process.cwd(), config: { - datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), network: options.network || 'livenet', port: 3001, - services: ['bitcoind', 'db', 'address', 'web'] + services: ['bitcoind', 'web'], + servicesConfig: { + bitcoind: { + datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), + exec: path.resolve(__dirname, '../../dist/bitcoind') + } + } } }; } diff --git a/lib/scaffold/default-config.js b/lib/scaffold/default-config.js index 83d36f7a..145b6a5c 100644 --- a/lib/scaffold/default-config.js +++ b/lib/scaffold/default-config.js @@ -24,17 +24,22 @@ function getDefaultConfig(options) { mkdirp.sync(defaultPath); } - var defaultServices = ['bitcoind', 'db', 'address', 'web']; + var defaultServices = ['bitcoind', 'web']; if (options.additionalServices) { defaultServices = defaultServices.concat(options.additionalServices); } if (!fs.existsSync(defaultConfigFile)) { var defaultConfig = { - datadir: path.resolve(defaultPath, './data'), network: 'livenet', port: 3001, - services: defaultServices + services: defaultServices, + servicesConfig: { + bitcoind: { + datadir: path.resolve(defaultPath, './data'), + exec: path.resolve(__dirname, '../../dist/bitcoind') + } + } }; fs.writeFileSync(defaultConfigFile, JSON.stringify(defaultConfig, null, 2)); } diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index f9ac430a..2f2bcfad 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -201,11 +201,6 @@ function start(options) { } fullConfig.services = start.setupServices(require, servicesPath, options.config); - fullConfig.datadir = path.resolve(options.path, options.config.datadir); - - if (fullConfig.daemon) { - start.spawnChildProcess(fullConfig.datadir, process); - } var node = new BitcoreNode(fullConfig); @@ -237,46 +232,9 @@ function start(options) { } -/** - * This function will fork the passed in process and exit the parent process - * in order to daemonize the process. If there is already a daemon for this pid (process), - * then the function just returns. Stdout and stderr both append to one file, 'bitcore-node.log' - * located in the datadir. - * @param {String} datadir - The data directory where the bitcoin blockchain and config live. - * @param {Object} _process - The process that needs to fork a child and then, itself, exit. - */ -function spawnChildProcess(datadir, _process) { - - if (_process.env.__bitcore_node) { - return _process.pid; - } - - var args = [].concat(_process.argv); - args.shift(); - var script = args.shift(); - var env = _process.env; - var cwd = _process.cwd(); - env.__bitcore_node = true; - - var stderr = fs.openSync(datadir + '/bitcore-node.log', 'a+'); - var stdout = stderr; - - var cp_opt = { - stdio: ['ignore', stdout, stderr], - env: env, - cwd: cwd, - detached: true - }; - - var child = child_process.spawn(_process.execPath, [script].concat(args), cp_opt); - child.unref(); - return _process.exit(); -} - module.exports = start; module.exports.registerExitHandlers = registerExitHandlers; module.exports.exitHandler = exitHandler; module.exports.registerSyncHandlers = registerSyncHandlers; module.exports.setupServices = setupServices; -module.exports.spawnChildProcess = spawnChildProcess; module.exports.cleanShutdown = cleanShutdown; diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 765a8550..a0298cd6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -6,7 +6,6 @@ var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); var bitcore = require('bitcore-lib'); -var Address = bitcore.Address; var zmq = require('zmq'); var async = require('async'); var LRU = require('lru-cache'); @@ -27,12 +26,11 @@ var Transaction = require('../transaction'); * @param {Node} options.node - A reference to the node */ function Bitcoin(options) { + /* jshint maxstatements: 20 */ if (!(this instanceof Bitcoin)) { return new Bitcoin(options); } - this._reindex = false; - this._reindexWait = 1000; Service.call(this, options); // caches valid until there is a new block @@ -41,15 +39,32 @@ function Bitcoin(options) { this.balanceCache = LRU(50000); this.summaryCache = LRU(50000); - // caches valid indefinetly + // caches valid indefinitely this.transactionCache = LRU(100000); this.transactionInfoCache = LRU(100000); this.blockCache = LRU(144); this.blockHeaderCache = LRU(288); + this.zmqKnownTransactions = LRU(50); + + this.options = options; + + // bitcoind child process + this.spawn = false; + + // available bitcoind nodes + this.nodes = []; + this.nodesIndex = 0; + Object.defineProperty(this, 'client', { + get: function() { + var client = this.nodes[this.nodesIndex].client; + this.nodesIndex = (this.nodesIndex + 1) % this.nodes.length; + return client; + }, + enumerable: true, + configurable: false + }); - $.checkState(this.node.datadir, 'Node is missing datadir property'); } - util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; @@ -77,15 +92,24 @@ Bitcoin.prototype.getAPIMethods = function() { return methods; }; -Bitcoin.prototype._loadConfiguration = function() { +Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ - $.checkArgument(this.node.datadir, 'Please specify "datadir" in configuration options'); - var configPath = this.node.datadir + '/bitcoin.conf'; - this.configuration = {}; + $.checkArgument(this.options.spawn, 'Please specify "spawn" in bitcoind config options'); + $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in bitcoind config options'); + $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in bitcoind config options'); + + var spawnOptions = this.options.spawn; + var configPath = spawnOptions.datadir + '/bitcoin.conf'; - if (!fs.existsSync(this.node.datadir)) { - mkdirp.sync(this.node.datadir); + this.spawn = {}; + this.spawn.datadir = this.options.spawn.datadir; + this.spawn.exec = this.options.spawn.exec; + this.spawn.configPath = configPath; + this.spawn.config = {}; + + if (!fs.existsSync(spawnOptions.datadir)) { + mkdirp.sync(spawnOptions.datadir); } var file = fs.readFileSync(configPath); @@ -100,48 +124,50 @@ Bitcoin.prototype._loadConfiguration = function() { } else { value = option[1]; } - this.configuration[option[0]] = value; + this.spawn.config[option[0]] = value; } } + var spawnConfig = this.spawn.config; + $.checkState( - this.configuration.txindex && this.configuration.txindex === 1, + spawnConfig.txindex && spawnConfig.txindex === 1, '"txindex" option is required in order to use transaction query features of bitcore-node. ' + 'Please add "txindex=1" to your configuration and reindex an existing database if ' + 'necessary with reindex=1' ); $.checkState( - this.configuration.addressindex && this.configuration.addressindex === 1, + spawnConfig.addressindex && spawnConfig.addressindex === 1, '"addressindex" option is required in order to use address query features of bitcore-node. ' + 'Please add "addressindex=1" to your configuration and reindex an existing database if ' + 'necessary with reindex=1' ); $.checkState( - this.configuration.server && this.configuration.server === 1, + spawnConfig.server && spawnConfig.server === 1, '"server" option is required to communicate to bitcoind from bitcore. ' + 'Please add "server=1" to your configuration and restart' ); $.checkState( - this.configuration.zmqpubrawtx, + spawnConfig.zmqpubrawtx, '"zmqpubrawtx" option is required to get event updates from bitcoind. ' + 'Please add "zmqpubrawtx=tcp://127.0.0.1:" to your configuration and restart' ); $.checkState( - this.configuration.zmqpubhashblock, + spawnConfig.zmqpubhashblock, '"zmqpubhashblock" option is required to get event updates from bitcoind. ' + 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' ); - if (this.configuration.reindex && this.configuration.reindex === 1) { + if (spawnConfig.reindex && spawnConfig.reindex === 1) { log.warn('Reindex option is currently enabled. This means that bitcoind is undergoing a reindex. ' + - 'The reindex flag will start the index from beginning every time the node is started, so it ' + - 'should be removed after the reindex has been initiated. Once the reindex is complete, the rest ' + - 'of bitcore-node services will start.'); - this._reindex = true; + 'The reindex flag will start the index from beginning every time the node is started, so it ' + + 'should be removed after the reindex has been initiated. Once the reindex is complete, the rest ' + + 'of bitcore-node services will start.'); + node._reindex = true; } }; @@ -153,67 +179,37 @@ Bitcoin.prototype._resetCaches = function() { this.summaryCache.reset(); }; -Bitcoin.prototype._registerEventHandlers = function() { +Bitcoin.prototype._initChain = function(callback) { var self = this; - this.zmqSubSocket.subscribe('hashblock'); - this.zmqSubSocket.subscribe('rawtx'); - - this.zmqSubSocket.on('message', function(topic, message) { - var topicString = topic.toString('utf8'); - if (topicString === 'rawtx') { - self.emit('tx', message); - } else if (topicString === 'hashblock') { - self._resetCaches(); - self.tiphash = message.toString('hex'); - self.client.getBlock(self.tiphash, function(err, response) { - if (err) { - return log.error(err); - } - self.height = response.result.height; - $.checkState(self.height >= 0); - self.emit('tip', self.height); - }); - - if(!self.node.stopping) { - self.syncPercentage(function(err, percentage) { - if (err) { - return log.error(err); - } - log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); - }); - } - } - }); - -}; - -Bitcoin.prototype._onReady = function(result, callback) { - var self = this; - - self._registerEventHandlers(); - - self.client.getInfo(function(err, response) { + self.client.getBestBlockHash(function(err, response) { if (err) { return callback(err); } - self.height = response.result.blocks; - self.client.getBlockHash(0, function(err, response) { + self.client.getBlock(response.result, function(err, response) { if (err) { return callback(err); } - var blockhash = response.result; - self.getBlock(blockhash, function(err, block) { + + self.height = response.result.height; + + self.client.getBlockHash(0, function(err, response) { if (err) { return callback(err); } - self.tiphash = block.hash; - self.genesisBuffer = block.toBuffer(); - self.emit('ready', result); - log.info('Bitcoin Daemon Ready'); - callback(); + var blockhash = response.result; + self.getBlock(blockhash, function(err, block) { + if (err) { + return callback(err); + } + self.genesisBuffer = block.toBuffer(); + self.emit('ready'); + log.info('Bitcoin Daemon Ready'); + callback(); + }); }); + }); }); }; @@ -229,27 +225,139 @@ Bitcoin.prototype._getNetworkOption = function() { return networkOption; }; -/** - * Called by Node to start the service - * @param {Function} callback - */ -Bitcoin.prototype.start = function(callback) { +Bitcoin.prototype._zmqBlockHandler = function(node, message) { + var self = this; + var hex = message.toString('hex'); + if (hex !== self.tiphash) { + self._resetCaches(); + self.tiphash = message.toString('hex'); + node.client.getBlock(self.tiphash, function(err, response) { + if (err) { + return log.error(err); + } + self.height = response.result.height; + $.checkState(self.height >= 0); + self.emit('tip', self.height); + }); + + if(!self.node.stopping) { + self.syncPercentage(function(err, percentage) { + if (err) { + return log.error(err); + } + log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + }); + } + } +}; + +Bitcoin.prototype._zmqTransactionHandler = function(node, message) { + var self = this; + var id = message.toString('binary'); + if (!self.zmqKnownTransactions[id]) { + self.zmqKnownTransactions[id] = true; + self.emit('tx', message); + } +}; + +Bitcoin.prototype._subscribeZmqEvents = function(node) { + var self = this; + node.zmqSubSocket.subscribe('hashblock'); + node.zmqSubSocket.subscribe('rawtx'); + node.zmqSubSocket.on('message', function(topic, message) { + var topicString = topic.toString('utf8'); + if (topicString === 'rawtx') { + self._zmqTransactionHandler(node, message); + } else if (topicString === 'hashblock') { + self._zmqBlockHandler(node, message); + } + }); +}; + +Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { + var self = this; + node.zmqSubSocket = zmq.socket('sub'); + + node.zmqSubSocket.on('monitor_error', function(err) { + log.error('Error in monitoring: %s, will restart monitoring in 5 seconds', err); + setTimeout(function() { + self.zmqSubSocket.monitor(500, 0); + }, 5000); + }); + + node.zmqSubSocket.monitor(500, 0); + node.zmqSubSocket.connect(zmqUrl); +}; + +Bitcoin.prototype._checkReindex = function(node, callback) { var self = this; + if (node._reindex) { + var interval = setInterval(function() { + node.client.syncPercentage(function(err, percentSynced) { + if (err) { + return log.error(err); + } + log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); + if (Math.round(percentSynced) >= 100) { + node._reindex = false; + self._subscribeZmqEvents(node); + callback(); + clearInterval(interval); + } + }); + }, self._reindexWait); - self._loadConfiguration(); + } else { + self._subscribeZmqEvents(node); + callback(); + } +}; + +Bitcoin.prototype._loadTipFromNode = function(node, callback) { + var self = this; + node.client.getBestBlockHash(function(err, response) { + if (err) { + if (!(err instanceof Error)) { + log.warn(err.message); + } + return callback(new Error('Could not connect to bitcoind RPC')); + } + node.client.getBlock(response.result, function(err, response) { + if (err) { + return done(err); + } + self.height = response.result.height; + $.checkState(self.height >= 0); + self.emit('tip', self.height); + callback(); + }); + }); +}; + +Bitcoin.prototype._spawnChildProcess = function(callback) { + var self = this; + + var node = {}; + node._reindex = false; + node._reindexWait = 1000; + + try { + self._loadSpawnConfiguration(node); + } catch(e) { + return callback(e); + } var options = [ - '--conf=' + path.resolve(this.node.datadir, './bitcoin.conf'), - '--datadir=' + this.node.datadir, + '--conf=' + path.resolve(this.spawn.configPath), + '--datadir=' + this.spawn.datadir, ]; if (self._getNetworkOption()) { options.push(self._getNetworkOption()); } + self.spawn.process = spawn(this.spawn.exec, options, {stdio: 'inherit'}); - self.process = spawn('bitcoind', options, {stdio: 'inherit'}); - - self.process.on('error', function(err) { + self.spawn.process.on('error', function(err) { log.error(err); }); @@ -258,67 +366,104 @@ Bitcoin.prototype.start = function(callback) { return done(new Error('Stopping while trying to connect to bitcoind.')); } - self.client = new BitcoinRPC({ + node.client = new BitcoinRPC({ protocol: 'http', host: '127.0.0.1', - port: self.configuration.rpcport, - user: self.configuration.rpcuser, - pass: self.configuration.rpcpassword + port: self.spawn.config.rpcport, + user: self.spawn.config.rpcuser, + pass: self.spawn.config.rpcpassword }); - self.client.getBestBlockHash(function(err, response) { + self._loadTipFromNode(node, done); + + }, function(err) { + if (err) { + return callback(err); + } + + self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); + + self._checkReindex(node, function() { if (err) { - if (!(err instanceof Error)) { - log.warn(err.message); - } - return done(new Error('Could not connect to bitcoind RPC')); + return callback(err); } - self.client.getBlock(response.result, function(err, response) { - if (err) { - return done(err); - } - self.height = response.result.height; - $.checkState(self.height >= 0); - self.emit('tip', self.height); - done(); - }); + callback(null, node); + }); + + }); +}; + +Bitcoin.prototype._connectProcess = function(config, callback) { + var self = this; + var node = {}; + + async.retry({times: 60, interval: 5000}, function(done) { + if (self.node.stopping) { + return done(new Error('Stopping while trying to connect to bitcoind.')); + } + + node.client = new BitcoinRPC({ + protocol: config.rpcprotocol || 'http', + host: config.rpchost || '127.0.0.1', + port: config.rpcport, + user: config.rpcuser, + pass: config.rpcpassword }); - }, function ready(err, result) { + self._loadTipFromNode(node, done); + + }, function(err) { if (err) { return callback(err); } - self.zmqSubSocket = zmq.socket('sub'); + self._initZmqSubSocket(node, config.zmqpubrawtx); - self.zmqSubSocket.on('monitor_error', function(err) { - log.error('Error in monitoring: %s, will restart monitoring in 5 seconds', err); - setTimeout(function() { - self.zmqSubSocket.monitor(500, 0); - }, 5000); - }); + callback(null, node); + }); +}; - self.zmqSubSocket.monitor(500, 0); - self.zmqSubSocket.connect(self.configuration.zmqpubrawtx); +/** + * Called by Node to start the service + * @param {Function} callback + */ +Bitcoin.prototype.start = function(callback) { + var self = this; - if (self._reindex) { - var interval = setInterval(function() { - self.syncPercentage(function(err, percentSynced) { + async.series([ + function(next) { + if (self.options.spawn) { + self._spawnChildProcess(function(err, node) { if (err) { - return log.error(err); + return next(err); } - log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); - if (Math.round(percentSynced) >= 100) { - self._reindex = false; - self._onReady(result, callback); - clearInterval(interval); + self.nodes.push(node); + next(); + }); + } else { + next(); + } + }, + function(next) { + if (self.options.connect) { + async.map(self.options.connect, self._connectProcess.bind(self), function(err, nodes) { + if (err) { + return callback(err); } + for(var i = 0; i < nodes.length; i++) { + self.nodes.push(nodes[i]); + } + next(); }); - }, self._reindexWait); - - } else { - self._onReady(result, callback); + } else { + next(); + } + } + ], function(err) { + if (err) { + return callback(err); } + self._initChain(callback); }); }; @@ -950,15 +1095,15 @@ Bitcoin.prototype.getInfo = function(callback) { * @param {Function} callback */ Bitcoin.prototype.stop = function(callback) { - if (this.process) { - this.process.once('exit', function(err, status) { + if (this.spawn && this.spawn.process) { + this.spawn.process.once('exit', function(err, status) { if (err) { return callback(err); } else { return callback(); } }); - this.process.kill('SIGHUP'); + this.spawn.process.kill('SIGHUP'); } else { callback(); } diff --git a/package.json b/package.json index 63dc3a15..56627575 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ ], "dependencies": { "async": "^1.3.0", - "bindings": "^1.2.1", "bitcoind-rpc": "^0.3.0", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", @@ -50,7 +49,6 @@ "express": "^4.13.3", "liftoff": "^2.2.0", "lru-cache": "^4.0.1", - "memdown": "^1.0.0", "mkdirp": "0.5.0", "npm": "^2.14.1", "semver": "^5.0.1", From 18310268a5991fe69f8ab70f520c5c8e853a5997 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 10:14:34 -0400 Subject: [PATCH 070/299] node: log intro with config path --- lib/node.js | 9 +++++++++ lib/scaffold/default-base-config.js | 2 +- lib/scaffold/default-config.js | 2 +- lib/scaffold/start.js | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/node.js b/lib/node.js index be4ec189..893dba1c 100644 --- a/lib/node.js +++ b/lib/node.js @@ -40,6 +40,7 @@ function Node(config) { if(!(this instanceof Node)) { return new Node(config); } + this.configPath = config.path; this.errors = errors; this.log = log; this.network = null; @@ -215,6 +216,12 @@ Node.prototype._startService = function(serviceInfo, callback) { }; +Node.prototype._logTitle = function() { + console.log('\n\n\n\n\n\n\n\n\n\n\n\n'); + log.info('Using config:', this.configPath); +}; + + /** * Will start all running services in the order based on the dependency chain. * @param {Function} callback - Called when all services are started @@ -223,6 +230,8 @@ Node.prototype.start = function(callback) { var self = this; var servicesOrder = this.getServiceOrder(); + self._logTitle(); + async.eachSeries( servicesOrder, function(service, next) { diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index 6cc5b316..eae48369 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -21,7 +21,7 @@ function getDefaultBaseConfig(options) { servicesConfig: { bitcoind: { datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), - exec: path.resolve(__dirname, '../../dist/bitcoind') + exec: path.resolve(__dirname, '../../bin/bitcoind') } } } diff --git a/lib/scaffold/default-config.js b/lib/scaffold/default-config.js index 145b6a5c..8a477300 100644 --- a/lib/scaffold/default-config.js +++ b/lib/scaffold/default-config.js @@ -37,7 +37,7 @@ function getDefaultConfig(options) { servicesConfig: { bitcoind: { datadir: path.resolve(defaultPath, './data'), - exec: path.resolve(__dirname, '../../dist/bitcoind') + exec: path.resolve(__dirname, '../../bin/bitcoind') } } }; diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 2f2bcfad..47c61307 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -200,6 +200,7 @@ function start(options) { servicesPath = options.path; // defaults to the same directory } + fullConfig.path = path.resolve(options.path, './bitcore-node.json'); fullConfig.services = start.setupServices(require, servicesPath, options.config); var node = new BitcoreNode(fullConfig); From 7c6e5cf7b1a913fc4e33b67d282b4797b2c93b53 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 10:14:43 -0400 Subject: [PATCH 071/299] bitcoind: only cache transaction with height if confirmations >= 6 --- lib/services/bitcoind.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index a0298cd6..8a1e9edb 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -42,6 +42,7 @@ function Bitcoin(options) { // caches valid indefinitely this.transactionCache = LRU(100000); this.transactionInfoCache = LRU(100000); + this.transactionInfoCacheConfirmations = 6; this.blockCache = LRU(144); this.blockHeaderCache = LRU(288); this.zmqKnownTransactions = LRU(50); @@ -1043,7 +1044,10 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal tx.__blockHash = response.result.blockhash; tx.__height = response.result.height; tx.__timestamp = response.result.time; - self.transactionInfoCache.set(txid, tx); + var confirmations = self._getConfirmationsDetail(tx); + if (confirmations >= self.transactionInfoCacheConfirmations) { + self.transactionInfoCache.set(txid, tx); + } callback(null, tx); }); } From c116353b8da8cfeb0bb675e4ea28e228fe66e142 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 14:44:35 -0400 Subject: [PATCH 072/299] build: start of install script --- package.json | 2 +- scripts/install | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 scripts/install diff --git a/package.json b/package.json index 56627575..8ec46e56 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", "version": "2.1.1-dev", - "lastBuild": "2.1.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", @@ -31,6 +30,7 @@ "bitcore-node": "./bin/bitcore-node" }, "scripts": { + "install": "./scripts/install", "test": "NODE_ENV=test mocha -R spec --recursive", "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" }, diff --git a/scripts/install b/scripts/install new file mode 100755 index 00000000..496ca5d2 --- /dev/null +++ b/scripts/install @@ -0,0 +1,51 @@ +#!/bin/bash + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." +platform=`uname -a | awk '{print tolower($1)}'` +arch=`uname -m` +version="0.12.0-bitcore" +url="https://github.com/" + +cd "${root_dir}/bin" + +if [ "${platform}" == "linux" ]; then + if [ "${arch}" == "x86_64" ]; then + tarball_name="bitcoin-${version}-linux64.tar.gz" + elif [ "${arch}" == "x86_32" ]; then + tarball_name="bitcoin-${version}-linux32.tar.gz" + fi +elif [ "${platform}" == "darwin" ]; then + tarball_name="bitcoin-${version}-osx64.tar.gz" +else + echo "Bitcoin binary distribution not available for platform and architecture" + exit -1 +fi + +binary_url="${url}/${tarball_name}" + +echo "Downloading bitcoin: ${binary_url}" + +is_curl=true +if hash curl 2>/dev/null; then + curl --fail -I $binary_url >/dev/null 2>&1 +else + is_curl=false + wget --server-response --spider $binary_url >/dev/null 2>&1 +fi + +if test $? -eq 0; then + if [ "${is_curl}" = true ]; then + curl $binary_url > $tarball_name + else + wget $binary_url + fi + if test -e "${tarball_name}"; then + echo "Unpacking bitcoin distribution" + tar -xvzf $tarball_name + if test $? -eq 0; then + exit 0 + fi + fi +fi +echo "Bitcoin binary distribution could not be downloaded" +exit -1 From 31da32ecfd2e88913d0bcaf6ee75204772c74ac1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 14:46:19 -0400 Subject: [PATCH 073/299] test: updated p2p integration test --- integration/data/bitcoin.conf | 8 +++-- integration/p2p.js | 35 +++++++++---------- lib/services/bitcoind.js | 65 ++++++++++++++++++++++++----------- 3 files changed, 67 insertions(+), 41 deletions(-) diff --git a/integration/data/bitcoin.conf b/integration/data/bitcoin.conf index ac6a7812..84aafd8c 100644 --- a/integration/data/bitcoin.conf +++ b/integration/data/bitcoin.conf @@ -1,9 +1,11 @@ server=1 whitelist=127.0.0.1 -rpcssl=1 -rpcsslcertificatechainfile=../bitcoind.crt -rpcsslprivatekeyfile=../bitcoind_no_pass.key txindex=1 +addressindex=1 +timestampindex=1 +zmqpubrawtx=tcp://127.0.0.1:30332 +zmqpubhashblock=tcp://127.0.0.1:30332 rpcallowip=127.0.0.1 +rpcport=30331 rpcuser=bitcoin rpcpassword=local321 diff --git a/integration/p2p.js b/integration/p2p.js index b578b908..3e30e464 100644 --- a/integration/p2p.js +++ b/integration/p2p.js @@ -3,10 +3,6 @@ var index = require('..'); var log = index.log; -if (process.env.BITCORENODE_ENV !== 'test') { - log.info('Please set the environment variable BITCORENODE_ENV=test and make sure bindings are compiled for testing'); - process.exit(); -} var p2p = require('bitcore-p2p'); var Peer = p2p.Peer; var Messages = p2p.Messages; @@ -58,18 +54,20 @@ describe('P2P Functionality', function() { var regtestNetwork = bitcore.Networks.get('regtest'); var datadir = __dirname + '/data'; - rimraf(datadir + '/regtest', function(err) {; - + rimraf(datadir + '/regtest', function(err) { if (err) { throw err; } + // enable regtest + bitcore.Networks.enableRegtest(); bitcoind = require('../').services.Bitcoin({ - node: { + spawn: { datadir: datadir, - network: { - name: 'regtest' - } + exec: 'bitcoind' + }, + node: { + network: bitcore.Networks.testnet } }); @@ -79,13 +77,16 @@ describe('P2P Functionality', function() { log.info('Waiting for Bitcoin Core to initialize...'); - bitcoind.start(function() { + bitcoind.start(function(err) { + if (err) { + throw err; + } log.info('Bitcoind started'); client = new BitcoinRPC({ - protocol: 'https', + protocol: 'http', host: '127.0.0.1', - port: 18332, + port: 30331, user: 'bitcoin', pass: 'local321', rejectUnauthorized: false @@ -186,13 +187,11 @@ describe('P2P Functionality', function() { var usedTxs = {}; - bitcoind.on('tx', function(result) { - var txFromResult = new Transaction().fromBuffer(result.buffer); + bitcoind.on('tx', function(buffer) { + var txFromResult = new Transaction().fromBuffer(buffer); var tx = usedTxs[txFromResult.id]; should.exist(tx); - result.buffer.toString('hex').should.equal(tx.serialize()); - result.hash.should.equal(tx.hash); - result.mempool.should.equal(true); + buffer.toString('hex').should.equal(tx.serialize()); delete usedTxs[tx.id]; if (Object.keys(usedTxs).length === 0) { done(); diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 8a1e9edb..3803ffa6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -27,6 +27,7 @@ var Transaction = require('../transaction'); */ function Bitcoin(options) { /* jshint maxstatements: 20 */ + var self = this; if (!(this instanceof Bitcoin)) { return new Bitcoin(options); } @@ -46,6 +47,8 @@ function Bitcoin(options) { this.blockCache = LRU(144); this.blockHeaderCache = LRU(288); this.zmqKnownTransactions = LRU(50); + this.zmqLastBlock = 0; + this.zmqUpdateTipTimeout = false; this.options = options; @@ -57,8 +60,8 @@ function Bitcoin(options) { this.nodesIndex = 0; Object.defineProperty(this, 'client', { get: function() { - var client = this.nodes[this.nodesIndex].client; - this.nodesIndex = (this.nodesIndex + 1) % this.nodes.length; + var client = self.nodes[self.nodesIndex].client; + self.nodesIndex = (self.nodesIndex + 1) % self.nodes.length; return client; }, enumerable: true, @@ -218,38 +221,57 @@ Bitcoin.prototype._initChain = function(callback) { Bitcoin.prototype._getNetworkOption = function() { var networkOption; if (this.node.network === bitcore.Networks.testnet) { + networkOption = '--testnet'; if (this.node.network.regtestEnabled) { networkOption = '--regtest'; } - networkOption = '--testnet'; } return networkOption; }; Bitcoin.prototype._zmqBlockHandler = function(node, message) { var self = this; - var hex = message.toString('hex'); - if (hex !== self.tiphash) { - self._resetCaches(); - self.tiphash = message.toString('hex'); - node.client.getBlock(self.tiphash, function(err, response) { - if (err) { - return log.error(err); - } - self.height = response.result.height; - $.checkState(self.height >= 0); - self.emit('tip', self.height); - }); - if(!self.node.stopping) { - self.syncPercentage(function(err, percentage) { + function updateChain() { + var hex = message.toString('hex'); + if (hex !== self.tiphash) { + self._resetCaches(); + self.tiphash = message.toString('hex'); + node.client.getBlock(self.tiphash, function(err, response) { if (err) { return log.error(err); } - log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + self.height = response.result.height; + self.zmqLastBlock = new Date(); + $.checkState(self.height >= 0); + self.emit('tip', self.height); }); + + if(!self.node.stopping) { + self.syncPercentage(function(err, percentage) { + if (err) { + return log.error(err); + } + log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + }); + } } } + + // Prevent a rapid succession of updates with regtest + if (self.node.network.regtestEnabled) { + if (new Date() - self.zmqLastBlock > 1000) { + updateChain(); + } else { + clearTimeout(self.zmqUpdateTipTimeout); + self.zmqUpdateTipTimeout = setTimeout(function() { + updateChain(); + }, 1000); + } + } else { + updateChain(); + } + }; Bitcoin.prototype._zmqTransactionHandler = function(node, message) { @@ -325,7 +347,7 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) { } node.client.getBlock(response.result, function(err, response) { if (err) { - return done(err); + return callback(err); } self.height = response.result.height; $.checkState(self.height >= 0); @@ -464,6 +486,9 @@ Bitcoin.prototype.start = function(callback) { if (err) { return callback(err); } + if (self.nodes.length === 0) { + return callback(new Error('Bitcoin configuration options "spawn" or "connect" are expected')); + } self._initChain(callback); }); @@ -1107,7 +1132,7 @@ Bitcoin.prototype.stop = function(callback) { return callback(); } }); - this.spawn.process.kill('SIGHUP'); + this.spawn.process.kill('SIGINT'); } else { callback(); } From c4649c9b133840935b43d01134f8e11089dc6fd7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 15:00:09 -0400 Subject: [PATCH 074/299] test: mark last zmq block before rpc calls --- lib/services/bitcoind.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 3803ffa6..bf74ac9a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -242,7 +242,6 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { return log.error(err); } self.height = response.result.height; - self.zmqLastBlock = new Date(); $.checkState(self.height >= 0); self.emit('tip', self.height); }); @@ -261,6 +260,7 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { // Prevent a rapid succession of updates with regtest if (self.node.network.regtestEnabled) { if (new Date() - self.zmqLastBlock > 1000) { + self.zmqLastBlock = new Date(); updateChain(); } else { clearTimeout(self.zmqUpdateTipTimeout); From 1fb552a9728496b84a7d08f815a797d121695550 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 15:37:30 -0400 Subject: [PATCH 075/299] build: download bitcoin binary distribution --- .gitignore | 10 +--------- scripts/install | 10 ++++++---- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index d72450d4..e94d8aba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,7 @@ node_modules/ node_modules/* coverage/* -out/ -out/* -build/ -build/* .lock-wscript -Makefile.gyp *.swp *.Makefile *.target.gyp.mk @@ -19,15 +14,12 @@ Makefile.gyp *.filters *.user *.project -test.js **/*.dylib **/*.so **/*.old **/*.files **/*.config **/*.creator -libbitcoind -libbitcoind* -libbitcoind.includes *.log .DS_Store +bin/bitcoin* \ No newline at end of file diff --git a/scripts/install b/scripts/install index 496ca5d2..132775f3 100755 --- a/scripts/install +++ b/scripts/install @@ -3,8 +3,9 @@ root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` -version="0.12.0-bitcore" -url="https://github.com/" +version="0.12.0" +url="https://github.com/braydonf/bitcoin/releases/download" +tag="v0.12.0-bitcore-beta1" cd "${root_dir}/bin" @@ -21,7 +22,7 @@ else exit -1 fi -binary_url="${url}/${tarball_name}" +binary_url="${url}/${tag}/${tarball_name}" echo "Downloading bitcoin: ${binary_url}" @@ -35,7 +36,7 @@ fi if test $? -eq 0; then if [ "${is_curl}" = true ]; then - curl $binary_url > $tarball_name + curl -L $binary_url > $tarball_name else wget $binary_url fi @@ -43,6 +44,7 @@ if test $? -eq 0; then echo "Unpacking bitcoin distribution" tar -xvzf $tarball_name if test $? -eq 0; then + ln -s "bitcoin-${version}/bin/bitcoind" exit 0 fi fi From 7c344b5f24520af8738b21c82d9f6f2b50a77c56 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 16:22:54 -0400 Subject: [PATCH 076/299] build: updates for npm install --- integration/p2p.js | 3 ++- lib/transaction.js | 6 ++++-- package.json | 3 ++- scripts/install | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/integration/p2p.js b/integration/p2p.js index 3e30e464..47c45a86 100644 --- a/integration/p2p.js +++ b/integration/p2p.js @@ -1,5 +1,6 @@ 'use strict'; +var path = require('path'); var index = require('..'); var log = index.log; @@ -64,7 +65,7 @@ describe('P2P Functionality', function() { bitcoind = require('../').services.Bitcoin({ spawn: { datadir: datadir, - exec: 'bitcoind' + exec: path.resolve(__dirname, '../bin/bitcoind') }, node: { network: bitcore.Networks.testnet diff --git a/lib/transaction.js b/lib/transaction.js index f55df37b..1f826c45 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -1,10 +1,12 @@ 'use strict'; var async = require('async'); -var levelup = require('levelup'); var bitcore = require('bitcore-lib'); var Transaction = bitcore.Transaction; +var index = require('./'); +var errors = index.errors; + var MAX_TRANSACTION_LIMIT = 5; Transaction.prototype.populateInputs = function(db, poolTransactions, callback) { @@ -30,7 +32,7 @@ Transaction.prototype._populateInput = function(db, input, poolTransactions, cal } var txid = input.prevTxId.toString('hex'); db.getTransaction(txid, true, function(err, prevTx) { - if(err instanceof levelup.errors.NotFoundError) { + if(err instanceof errors.Transaction.NotFoundError) { // Check the pool for transaction for(var i = 0; i < poolTransactions.length; i++) { if(txid === poolTransactions[i].hash) { diff --git a/package.json b/package.json index 8ec46e56..0cdfbf4b 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "^0.3.0", + "bitcoind-rpc": "braydonf/bitcoind-rpc#8d27a545f4e7de5a8faca5de6bdbb1a6c1e41f5c", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", @@ -58,6 +58,7 @@ }, "devDependencies": { "benchmark": "1.0.0", + "bitcore-p2p": "^1.1.0", "chai": "^3.0.0", "mocha": "~1.16.2", "proxyquire": "^1.3.1", diff --git a/scripts/install b/scripts/install index 132775f3..3ded019c 100755 --- a/scripts/install +++ b/scripts/install @@ -44,7 +44,7 @@ if test $? -eq 0; then echo "Unpacking bitcoin distribution" tar -xvzf $tarball_name if test $? -eq 0; then - ln -s "bitcoin-${version}/bin/bitcoind" + ln -sf "bitcoin-${version}/bin/bitcoind" exit 0 fi fi From 67b8ec215231a4c88102fbbcb86b2abc7f6f7d2b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 16:31:42 -0400 Subject: [PATCH 077/299] build: update travis with zmq --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index edcb8607..b370ed84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: node_js env: - - BITCORENODE_ENV=test BITCORENODE_ASSUME_YES=true CXX=g++-4.8 CC=gcc-4.8 + - CXX=g++-4.8 CC=gcc-4.8 addons: apt: sources: @@ -9,14 +9,13 @@ addons: packages: - g++-4.8 - gcc-4.8 + - libzmq3-dev node_js: - "v0.12.7" - "v4" script: + - _mocha -R spec integration/p2p.js - _mocha -R spec integration/regtest.js - _mocha -R spec integration/regtest-node.js - - _mocha -R spec integration/p2p.js - _mocha -R spec --recursive -cache: - directories: - - cache + From 962e7b87f8703ca8fc3fa950ec731fc6ee122f3a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 18:59:28 -0400 Subject: [PATCH 078/299] test: update regtest --- integration/p2p.js | 19 +-- integration/regtest.js | 344 +++++++++++++++------------------------ lib/services/bitcoind.js | 30 +++- 3 files changed, 157 insertions(+), 236 deletions(-) diff --git a/integration/p2p.js b/integration/p2p.js index 47c45a86..8bde448d 100644 --- a/integration/p2p.js +++ b/integration/p2p.js @@ -37,21 +37,8 @@ describe('P2P Functionality', function() { before(function(done) { this.timeout(100000); - // Add the regtest network - bitcore.Networks.remove(bitcore.Networks.testnet); - bitcore.Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); - + // enable regtest + bitcore.Networks.enableRegtest(); var regtestNetwork = bitcore.Networks.get('regtest'); var datadir = __dirname + '/data'; @@ -60,8 +47,6 @@ describe('P2P Functionality', function() { throw err; } - // enable regtest - bitcore.Networks.enableRegtest(); bitcoind = require('../').services.Bitcoin({ spawn: { datadir: datadir, diff --git a/integration/regtest.js b/integration/regtest.js index e7dba7a8..98118ec1 100644 --- a/integration/regtest.js +++ b/integration/regtest.js @@ -5,14 +5,10 @@ // functionality by including the wallet in the build. // To run the tests: $ mocha -R spec integration/regtest.js +var path = require('path'); var index = require('..'); var log = index.log; -if (process.env.BITCORENODE_ENV !== 'test') { - log.info('Please set the environment variable BITCORENODE_ENV=test and make sure bindings are compiled for testing'); - process.exit(); -} - var chai = require('chai'); var bitcore = require('bitcore-lib'); var BN = bitcore.crypto.BN; @@ -39,19 +35,8 @@ describe('Daemon Binding Functionality', function() { this.timeout(30000); // Add the regtest network - bitcore.Networks.remove(bitcore.Networks.testnet); - bitcore.Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); + bitcore.Networks.enableRegtest(); + var regtestNetwork = bitcore.Networks.get('regtest'); var datadir = __dirname + '/data'; @@ -62,11 +47,12 @@ describe('Daemon Binding Functionality', function() { } bitcoind = require('../').services.Bitcoin({ - node: { + spawn: { datadir: datadir, - network: { - name: 'regtest' - } + exec: path.resolve(__dirname, '../bin/bitcoind') + }, + node: { + network: regtestNetwork } }); @@ -80,9 +66,9 @@ describe('Daemon Binding Functionality', function() { log.info('Bitcoind started'); client = new BitcoinRPC({ - protocol: 'https', + protocol: 'http', host: '127.0.0.1', - port: 18332, + port: 30331, user: 'bitcoin', pass: 'local321', rejectUnauthorized: false @@ -154,12 +140,11 @@ describe('Daemon Binding Functionality', function() { [0,1,2,3,5,6,7,8,9].forEach(function(i) { it('generated block ' + i, function(done) { - bitcoind.getBlock(blockHashes[i], function(err, response) { + bitcoind.getBlock(blockHashes[i], function(err, block) { if (err) { throw err; } - should.exist(response); - var block = bitcore.Block.fromBuffer(response); + should.exist(block); block.hash.should.equal(blockHashes[i]); done(); }); @@ -173,12 +158,11 @@ describe('Daemon Binding Functionality', function() { it('generated block ' + i, function(done) { // add the genesis block var height = i + 1; - bitcoind.getBlock(i + 1, function(err, response) { + bitcoind.getBlock(i + 1, function(err, block) { if (err) { throw err; } - should.exist(response); - var block = bitcore.Block.fromBuffer(response); + should.exist(block); block.hash.should.equal(blockHashes[i]); done(); }); @@ -223,26 +207,45 @@ describe('Daemon Binding Functionality', function() { }); - describe('get block index', function() { + describe('get block header', function() { var expectedWork = new BN(6); [1,2,3,4,5,6,7,8,9].forEach(function(i) { - it('generate block ' + i, function() { - var blockIndex = bitcoind.getBlockIndex(blockHashes[i]); - should.exist(blockIndex); - should.exist(blockIndex.chainWork); - var work = new BN(blockIndex.chainWork, 'hex'); - work.cmp(expectedWork).should.equal(0); - expectedWork = expectedWork.add(new BN(2)); - should.exist(blockIndex.prevHash); - blockIndex.hash.should.equal(blockHashes[i]); - blockIndex.prevHash.should.equal(blockHashes[i - 1]); - blockIndex.height.should.equal(i + 1); + it('generate block ' + i, function(done) { + bitcoind.getBlockHeader(blockHashes[i], function(err, blockIndex) { + if (err) { + return done(err); + } + should.exist(blockIndex); + should.exist(blockIndex.chainwork); + var work = new BN(blockIndex.chainwork, 'hex'); + work.cmp(expectedWork).should.equal(0); + expectedWork = expectedWork.add(new BN(2)); + should.exist(blockIndex.previousblockhash); + blockIndex.hash.should.equal(blockHashes[i]); + blockIndex.previousblockhash.should.equal(blockHashes[i - 1]); + blockIndex.height.should.equal(i + 1); + done(); + }); + }); + }); + it('will get null prevHash for the genesis block', function(done) { + bitcoind.getBlockHeader(0, function(err, header) { + if (err) { + return done(err); + } + should.exist(header); + should.equal(header.previousblockhash, undefined); + done(); }); }); - it('will get null prevHash for the genesis block', function() { - var blockIndex = bitcoind.getBlockIndex(0); - should.exist(blockIndex); - should.equal(blockIndex.prevHash, null); + it('will get null for block not found', function(done) { + bitcoind.getBlockHeader('notahash', function(err, header) { + if(err) { + return done(err); + } + should.equal(header, null); + done(); + }); }); }); @@ -250,36 +253,33 @@ describe('Daemon Binding Functionality', function() { var expectedWork = new BN(6); [2,3,4,5,6,7,8,9].forEach(function(i) { it('generate block ' + i, function() { - var blockIndex = bitcoind.getBlockIndex(i); - should.exist(blockIndex); - should.exist(blockIndex.chainWork); - var work = new BN(blockIndex.chainWork, 'hex'); - work.cmp(expectedWork).should.equal(0); - expectedWork = expectedWork.add(new BN(2)); - should.exist(blockIndex.prevHash); - blockIndex.hash.should.equal(blockHashes[i - 1]); - blockIndex.prevHash.should.equal(blockHashes[i - 2]); - blockIndex.height.should.equal(i); + bitcoind.getBlockHeader(i, function(err, header) { + should.exist(header); + should.exist(header.chainwork); + var work = new BN(header.chainwork, 'hex'); + work.cmp(expectedWork).should.equal(0); + expectedWork = expectedWork.add(new BN(2)); + should.exist(header.previousblockhash); + header.hash.should.equal(blockHashes[i - 1]); + header.previousblockhash.should.equal(blockHashes[i - 2]); + header.height.should.equal(i); + }); }); }); it('will get null with number greater than tip', function(done) { - var index = bitcoind.getBlockIndex(100000); - should.equal(index, null); - done(); - }); - }); - - describe('isMainChain', function() { - [1,2,3,4,5,6,7,8,9].forEach(function(i) { - it('block ' + i + ' is on the main chain', function() { - bitcoind.isMainChain(blockHashes[i]).should.equal(true); + bitcoind.getBlockHeader(100000, function(err, header) { + if (err) { + return done(err); + } + should.equal(header, null); + done(); }); }); }); describe('send transaction functionality', function() { - it('will not error and return the transaction hash', function() { + it('will not error and return the transaction hash', function(done) { // create and sign the transaction var tx = bitcore.Transaction(); @@ -289,31 +289,40 @@ describe('Daemon Binding Functionality', function() { tx.sign(bitcore.PrivateKey.fromWIF(utxos[0].privateKeyWIF)); // test sending the transaction - var hash = bitcoind.sendTransaction(tx.serialize()); - hash.should.equal(tx.hash); - }); + bitcoind.sendTransaction(tx.serialize(), function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal(tx.hash); + done(); + }); - it('will throw an error if an unsigned transaction is sent', function() { + }); + it('will throw an error if an unsigned transaction is sent', function(done) { var tx = bitcore.Transaction(); tx.from(utxos[1]); tx.change(privateKey.toAddress()); tx.to(destKey.toAddress(), utxos[1].amount * 1e8 - 1000); - (function() { - bitcoind.sendTransaction(tx.uncheckedSerialize()); - }).should.throw('\x10: mandatory-script-verify-flag-failed (Operation not valid with the current stack size)'); + bitcoind.sendTransaction(tx.uncheckedSerialize(), function(err, hash) { + should.exist(err); + should.not.exist(hash); + done(); + }); }); - it('will throw an error for unexpected types', function() { + it('will throw an error for unexpected types (tx decode failed)', function(done) { var garbage = new Buffer('abcdef', 'hex'); - (function() { - bitcoind.sendTransaction(garbage); - }).should.throw('TX decode failed'); - - var num = 23; - (function() { - bitcoind.sendTransaction(num); - }).should.throw('TX decode failed'); + bitcoind.sendTransaction(garbage, function(err, hash) { + should.exist(err); + should.not.exist(hash); + var num = 23; + bitcoind.sendTransaction(num, function(err, hash) { + should.exist(err); + should.not.exist(hash); + done(); + }); + }); }); it('will emit "tx" events', function(done) { @@ -325,21 +334,29 @@ describe('Daemon Binding Functionality', function() { var serialized = tx.serialize(); - bitcoind.once('tx', function(result) { - result.buffer.toString('hex').should.equal(serialized); - result.hash.should.equal(tx.hash); - result.mempool.should.equal(true); + bitcoind.once('tx', function(buffer) { + buffer.toString('hex').should.equal(serialized); done(); }); - bitcoind.sendTransaction(serialized); + bitcoind.sendTransaction(serialized, function(err, hash) { + if (err) { + return done(err); + } + should.exist(hash); + }); }); }); describe('fee estimation', function() { - it('will estimate fees', function() { - var fees = bitcoind.estimateFee(); - fees.should.equal(-1); + it('will estimate fees', function(done) { + bitcoind.estimateFee(1, function(err, fees) { + if (err) { + return done(err); + } + fees.should.equal(-1); + done(); + }); }); }); @@ -347,8 +364,7 @@ describe('Daemon Binding Functionality', function() { it('will get an event when the tip is new', function(done) { this.timeout(4000); bitcoind.on('tip', function(height) { - if (height == 151) { - height.should.equal(151); + if (height === 151) { done(); } }); @@ -360,134 +376,40 @@ describe('Daemon Binding Functionality', function() { }); }); - describe('transactions leaving the mempool', function() { - it('receive event when transaction leaves', function(done) { - - // add transaction to build a new block - var tx = bitcore.Transaction(); - tx.from(utxos[4]); - tx.change(privateKey.toAddress()); - tx.to(destKey.toAddress(), utxos[4].amount * 1e8 - 1000); - tx.sign(bitcore.PrivateKey.fromWIF(utxos[4].privateKeyWIF)); - bitcoind.sendTransaction(tx.serialize()); - - bitcoind.once('txleave', function(txInfo) { - txInfo.hash.should.equal(tx.hash); - done(); - }); - - client.generate(1, function(err, response) { + describe('get transaction with block info', function() { + it('should include tx buffer, height and timestamp', function(done) { + bitcoind.getTransactionWithBlockInfo(utxos[0].txid, true, function(err, tx) { if (err) { - throw err; + return done(err); } + should.exist(tx.__height); + tx.__height.should.be.a('number'); + should.exist(tx.__timestamp); + should.exist(tx.__blockHash); + done(); }); }); }); - describe('mempool functionality', function() { - - var fromAddress = 'mszYqVnqKoQx4jcTdJXxwKAissE3Jbrrc1'; - var utxo1 = { - address: fromAddress, - txId: 'a477af6b2667c29670467e4e0728b685ee07b240235771862318e29ddbe58458', - outputIndex: 0, - script: bitcore.Script.buildPublicKeyHashOut(fromAddress).toString(), - satoshis: 100000 - }; - var toAddress = 'mrU9pEmAx26HcbKVrABvgL7AwA5fjNFoDc'; - var changeAddress = 'mgBCJAsvzgT2qNNeXsoECg2uPKrUsZ76up'; - var changeAddressP2SH = '2N7T3TAetJrSCruQ39aNrJvYLhG1LJosujf'; - var privateKey1 = 'cSBnVM4xvxarwGQuAfQFwqDg9k5tErHUHzgWsEfD4zdwUasvqRVY'; - var private1 = '6ce7e97e317d2af16c33db0b9270ec047a91bff3eff8558afb5014afb2bb5976'; - var private2 = 'c9b26b0f771a0d2dad88a44de90f05f416b3b385ff1d989343005546a0032890'; - var tx = new bitcore.Transaction(); - tx.from(utxo1); - tx.to(toAddress, 50000); - tx.change(changeAddress); - tx.sign(privateKey1); - - var tx2; - var tx2Key; - - before(function() { - tx2 = bitcore.Transaction(); - tx2.from(utxos[3]); - tx2.change(privateKey.toAddress()); - tx2.to(destKey.toAddress(), utxos[3].amount * 1e8 - 1000); - tx2Key = bitcore.PrivateKey.fromWIF(utxos[3].privateKeyWIF); - tx2.sign(tx2Key); - }); - - it('will add an unchecked transaction', function() { - var added = bitcoind.addMempoolUncheckedTransaction(tx.serialize()); - added.should.equal(true); - bitcoind.getTransaction(tx.hash, true, function(err, txBuffer) { - if(err) { - throw err; + describe('#getInfo', function() { + it('will get information', function(done) { + bitcoind.getInfo(function(err, info) { + if (err) { + return done(err); } - var expected = tx.toBuffer().toString('hex'); - txBuffer.toString('hex').should.equal(expected); - }); - - }); - - it('get one transaction', function() { - var transactions = bitcoind.getMempoolTransactions(); - transactions[0].toString('hex').should.equal(tx.serialize()); - }); - - it('get multiple transactions', function() { - bitcoind.sendTransaction(tx2.serialize()); - var transactions = bitcoind.getMempoolTransactions(); - var expected = [tx.serialize(), tx2.serialize()]; - expected.should.contain(transactions[0].toString('hex')); - expected.should.contain(transactions[1].toString('hex')); - }); - - }); - - describe('get transaction with block info', function() { - it('should include tx buffer, height and timestamp', function(done) { - bitcoind.getTransactionWithBlockInfo(utxos[0].txid, true, function(err, data) { - should.not.exist(err); - should.exist(data.height); - data.height.should.be.a('number'); - should.exist(data.timestamp); - should.exist(data.buffer); + info.network.should.equal('regtest'); + should.exist(info); + should.exist(info.version); + should.exist(info.blocks); + should.exist(info.timeoffset); + should.exist(info.connections); + should.exist(info.difficulty); + should.exist(info.testnet); + should.exist(info.relayfee); + should.exist(info.errors); done(); }); }); }); - describe('get next block hash', function() { - it('will get next block hash', function() { - var nextBlockHash = bitcoind.getNextBlockHash(blockHashes[0]); - nextBlockHash.should.equal(blockHashes[1]); - var nextnextBlockHash = bitcoind.getNextBlockHash(nextBlockHash); - nextnextBlockHash.should.equal(blockHashes[2]); - }); - - it('will get a null response if the tip hash is provided', function() { - var bestBlockHash = bitcoind.getBestBlockHash(); - var nextBlockHash = bitcoind.getNextBlockHash(bestBlockHash); - should.not.exist(nextBlockHash); - }); - }); - - describe('#getInfo', function() { - it('will get information', function() { - var info = bitcoind.getInfo(); - info.network.should.equal('regtest'); - should.exist(info); - should.exist(info.version); - should.exist(info.blocks); - should.exist(info.timeoffset); - should.exist(info.connections); - should.exist(info.difficulty); - should.exist(info.testnet); - should.exist(info.relayfee); - should.exist(info.errors); - }); - }); - }); diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index bf74ac9a..6d9c1a06 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -951,16 +951,21 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { function queryHeader(blockhash) { self.client.getBlockHeader(blockhash, function(err, response) { - if (err) { + if (err && response.error.code === -5) { + return callback(null, null); + } else if (err) { return callback(err); } + // TODO format response prevHash instead of previousblockhash, etc. callback(null, response.result); }); } if (_.isNumber(block)) { self.client.getBlockHash(block, function(err, response) { - if (err) { + if (err && response.error.code === -8) { + return callback(null, null); + } else if (err) { return callback(err); } var blockhash = response.result; @@ -998,8 +1003,12 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { } else { txString = tx; } + if (_.isFunction(allowAbsurdFees) && _.isUndefined(callback)) { + callback = allowAbsurdFees; + allowAbsurdFees = false; + } - this.client.sendTransaction(txString, allowAbsurdFees, function(err, response) { + this.client.sendRawTransaction(txString, allowAbsurdFees, function(err, response) { if (err) { return callback(err); } @@ -1023,12 +1032,11 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { }); } else { self.client.getRawTransaction(txid, function(err, response) { - if (err) { + if (err && response.error.code === -5) { + return callback(null, null); + } else if (err) { return callback(err); } - if (!response.result) { - return callback(new errors.Transaction.NotFound()); - } var tx = Transaction(); tx.fromString(response.result); self.transactionCache.set(txid, tx); @@ -1111,11 +1119,17 @@ Bitcoin.prototype.getInputForOutput = function(txid, index, options, callback) { * } */ Bitcoin.prototype.getInfo = function(callback) { + var self = this; this.client.getInfo(function(err, response) { if (err) { return callback(err); } - callback(null, response.result); + var result = response.result; + result.network = self.node.network.name; + if (self.node.network.regtestEnabled) { + result.network = 'regtest'; + } + callback(null, result); }); }; From 88a82719caddeba998b36aeda414f030cb9e1bd9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 09:54:45 -0400 Subject: [PATCH 079/299] test: moved integration -> regtest --- .travis.yml | 6 +++--- integration/regtest.js => regtest/bitcoind.js | 0 {integration => regtest}/data/.gitignore | 0 {integration => regtest}/data/bitcoin.conf | 0 {integration => regtest}/data/bitcoind.crt | 0 {integration => regtest}/data/bitcoind_no_pass.key | 0 integration/regtest-node.js => regtest/node.js | 0 {integration => regtest}/p2p.js | 0 8 files changed, 3 insertions(+), 3 deletions(-) rename integration/regtest.js => regtest/bitcoind.js (100%) rename {integration => regtest}/data/.gitignore (100%) rename {integration => regtest}/data/bitcoin.conf (100%) rename {integration => regtest}/data/bitcoind.crt (100%) rename {integration => regtest}/data/bitcoind_no_pass.key (100%) rename integration/regtest-node.js => regtest/node.js (100%) rename {integration => regtest}/p2p.js (100%) diff --git a/.travis.yml b/.travis.yml index b370ed84..1192bff0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ node_js: - "v0.12.7" - "v4" script: - - _mocha -R spec integration/p2p.js - - _mocha -R spec integration/regtest.js - - _mocha -R spec integration/regtest-node.js + - _mocha -R spec regtest/p2p.js + - _mocha -R spec regtest/bitcoind.js + - _mocha -R spec regtest/node.js - _mocha -R spec --recursive diff --git a/integration/regtest.js b/regtest/bitcoind.js similarity index 100% rename from integration/regtest.js rename to regtest/bitcoind.js diff --git a/integration/data/.gitignore b/regtest/data/.gitignore similarity index 100% rename from integration/data/.gitignore rename to regtest/data/.gitignore diff --git a/integration/data/bitcoin.conf b/regtest/data/bitcoin.conf similarity index 100% rename from integration/data/bitcoin.conf rename to regtest/data/bitcoin.conf diff --git a/integration/data/bitcoind.crt b/regtest/data/bitcoind.crt similarity index 100% rename from integration/data/bitcoind.crt rename to regtest/data/bitcoind.crt diff --git a/integration/data/bitcoind_no_pass.key b/regtest/data/bitcoind_no_pass.key similarity index 100% rename from integration/data/bitcoind_no_pass.key rename to regtest/data/bitcoind_no_pass.key diff --git a/integration/regtest-node.js b/regtest/node.js similarity index 100% rename from integration/regtest-node.js rename to regtest/node.js diff --git a/integration/p2p.js b/regtest/p2p.js similarity index 100% rename from integration/p2p.js rename to regtest/p2p.js From 3ead5928a72f42ffcdb06f7f235983c920ff802e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 09:56:08 -0400 Subject: [PATCH 080/299] test: update titles and docs for regtests --- regtest/bitcoind.js | 7 ++----- regtest/node.js | 5 +---- regtest/p2p.js | 2 ++ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 98118ec1..74d9ffa5 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -1,9 +1,6 @@ 'use strict'; -// These tests require bitcore-node Bitcoin Core bindings to be compiled with -// the environment variable BITCORENODE_ENV=test. This enables the use of regtest -// functionality by including the wallet in the build. -// To run the tests: $ mocha -R spec integration/regtest.js +// To run the tests: $ mocha -R spec regtest/bitcoind.js var path = require('path'); var index = require('..'); @@ -29,7 +26,7 @@ var coinbasePrivateKey; var privateKey = bitcore.PrivateKey(); var destKey = bitcore.PrivateKey(); -describe('Daemon Binding Functionality', function() { +describe('Bitcoind Functionality', function() { before(function(done) { this.timeout(30000); diff --git a/regtest/node.js b/regtest/node.js index 9523f79a..fa031a38 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -1,9 +1,6 @@ 'use strict'; -// These tests require bitcore-node Bitcoin Core bindings to be compiled with -// the environment variable BITCORENODE_ENV=test. This enables the use of regtest -// functionality by including the wallet in the build. -// To run the tests: $ mocha -R spec integration/regtest-node.js +// To run the tests: $ mocha -R spec regtest/node.js var index = require('..'); var async = require('async'); diff --git a/regtest/p2p.js b/regtest/p2p.js index 8bde448d..acf718d1 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -1,5 +1,7 @@ 'use strict'; +// To run the tests: $ mocha -R spec regtest/p2p.js + var path = require('path'); var index = require('..'); var log = index.log; From 82232c0f5563fc9acc653237f1e67bbd12a3f8da Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 10:35:01 -0400 Subject: [PATCH 081/299] bitcoind: wrap rpc errors as instances of error --- lib/services/bitcoind.js | 73 ++++++++++++++++++++++------------------ lib/transaction.js | 2 +- regtest/bitcoind.js | 15 +++++++++ 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 6d9c1a06..8319b57d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -183,24 +183,30 @@ Bitcoin.prototype._resetCaches = function() { this.summaryCache.reset(); }; +Bitcoin.prototype._wrapRPCError = function(errObj) { + var err = new Error(errObj.message); + err.code = errObj.code; + return err; +}; + Bitcoin.prototype._initChain = function(callback) { var self = this; self.client.getBestBlockHash(function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } self.client.getBlock(response.result, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } self.height = response.result.height; self.client.getBlockHash(0, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var blockhash = response.result; self.getBlock(blockhash, function(err, block) { @@ -239,7 +245,7 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { self.tiphash = message.toString('hex'); node.client.getBlock(self.tiphash, function(err, response) { if (err) { - return log.error(err); + return log.error(self._wrapRPCError(err)); } self.height = response.result.height; $.checkState(self.height >= 0); @@ -318,7 +324,7 @@ Bitcoin.prototype._checkReindex = function(node, callback) { var interval = setInterval(function() { node.client.syncPercentage(function(err, percentSynced) { if (err) { - return log.error(err); + return log.error(self._wrapRPCError(err)); } log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); if (Math.round(percentSynced) >= 100) { @@ -339,15 +345,15 @@ Bitcoin.prototype._checkReindex = function(node, callback) { Bitcoin.prototype._loadTipFromNode = function(node, callback) { var self = this; node.client.getBestBlockHash(function(err, response) { - if (err) { - if (!(err instanceof Error)) { - log.warn(err.message); - } - return callback(new Error('Could not connect to bitcoind RPC')); + if (err && err.code === -28) { + log.warn(err.message); + return callback(self._wrapRPCError(err)); + } else if (err) { + return callback(self._wrapRPCError(err)); } node.client.getBlock(response.result, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } self.height = response.result.height; $.checkState(self.height >= 0); @@ -518,9 +524,10 @@ Bitcoin.prototype.isSynced = function(callback) { * @returns {Number} An estimated percentage of the syncronization status */ Bitcoin.prototype.syncPercentage = function(callback) { + var self = this; this.client.getBlockchainInfo(function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var percentSynced = response.result.verificationprogress * 100; callback(null, percentSynced); @@ -542,7 +549,7 @@ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { } else { this.client.getAddressBalance({addresses: addresses}, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } self.balanceCache.set(cacheKey, response.result); callback(null, response.result); @@ -565,7 +572,7 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb } else { self.client.getAddressUtxos({addresses: addresses}, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } self.utxosCache.set(cacheKey, response.result); callback(null, response.result); @@ -614,7 +621,7 @@ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { } else { self.client.getAddressTxids({addresses: addresses}, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } response.result.reverse(); self.txidsCache.set(cacheKey, response.result); @@ -627,7 +634,7 @@ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { if (queryMempool) { self.client.getAddressMempool({addresses: addresses}, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } mempoolTxids = self._getTxidsFromMempool(response.result); finish(); @@ -850,7 +857,7 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { } self.client.getAddressMempool({'addresses': [addressArg]}, function(err, response) { if (err) { - return done(err); + return done(self._wrapRPCError(err)); } mempoolTxids = self._getTxidsFromMempool(response.result); summary.unconfirmedAppearances = mempoolTxids.length; @@ -895,7 +902,7 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { function queryBlock(blockhash) { self.client.getBlock(blockhash, false, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var blockObj = bitcore.Block.fromString(response.result); self.blockCache.set(blockArg, blockObj); @@ -912,7 +919,7 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { if (_.isNumber(blockArg)) { self.client.getBlockHash(blockArg, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var blockhash = response.result; queryBlock(blockhash); @@ -928,7 +935,7 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { var self = this; self.client.getBlockHashes(high, low, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } callback(null, response.result); }); @@ -954,7 +961,7 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { if (err && response.error.code === -5) { return callback(null, null); } else if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } // TODO format response prevHash instead of previousblockhash, etc. callback(null, response.result); @@ -966,7 +973,7 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { if (err && response.error.code === -8) { return callback(null, null); } else if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var blockhash = response.result; queryHeader(blockhash); @@ -982,9 +989,10 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { * @returns {Number} */ Bitcoin.prototype.estimateFee = function(blocks, callback) { + var self = this; this.client.estimateFee(blocks, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } callback(null, response.result); }); @@ -997,6 +1005,7 @@ Bitcoin.prototype.estimateFee = function(blocks, callback) { * @param {Boolean} allowAbsurdFees - Enable large fees */ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { + var self = this; var txString; if (tx instanceof Transaction) { txString = tx.serialize(); @@ -1010,7 +1019,7 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { this.client.sendRawTransaction(txString, allowAbsurdFees, function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } callback(null, response.result); }); @@ -1035,7 +1044,7 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { if (err && response.error.code === -5) { return callback(null, null); } else if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var tx = Transaction(); tx.fromString(response.result); @@ -1066,11 +1075,10 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal }); } else { self.client.getRawTransaction(txid, 1, function(err, response) { - if (err) { - return callback(err); - } - if (!response.result) { - return callback(new errors.Transaction.NotFound()); + if (err && response.error.code === -5) { + return callback(null, null); + } else if (err) { + return callback(self._wrapRPCError(err)); } var tx = Transaction(); tx.fromString(response.result.hex); @@ -1091,9 +1099,10 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal * @returns {String} */ Bitcoin.prototype.getBestBlockHash = function(callback) { + var self = this; this.client.getBestBlockHash(function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } callback(null, response.result); }); @@ -1122,7 +1131,7 @@ Bitcoin.prototype.getInfo = function(callback) { var self = this; this.client.getInfo(function(err, response) { if (err) { - return callback(err); + return callback(self._wrapRPCError(err)); } var result = response.result; result.network = self.node.network.name; diff --git a/lib/transaction.js b/lib/transaction.js index 1f826c45..1acb350d 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -32,7 +32,7 @@ Transaction.prototype._populateInput = function(db, input, poolTransactions, cal } var txid = input.prevTxId.toString('hex'); db.getTransaction(txid, true, function(err, prevTx) { - if(err instanceof errors.Transaction.NotFoundError) { + if(!prevTx) { // Check the pool for transaction for(var i = 0; i < poolTransactions.length; i++) { if(txid === poolTransactions[i].hash) { diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 74d9ffa5..9b7a8f05 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -149,6 +149,19 @@ describe('Bitcoind Functionality', function() { }); }); + describe('get errors as error instances', function() { + it('will wrap an rpc into a javascript error', function(done) { + bitcoind.client.getBlock(1000000000, function(err, response) { + var error = bitcoind._wrapRPCError(err); + (error instanceof Error).should.equal(true); + error.message.should.equal(err.message); + error.code.should.equal(err.code); + should.exist(error.stack); + done(); + }); + }); + }); + describe('get blocks by height', function() { [0,1,2,3,4,5,6,7,8,9].forEach(function(i) { @@ -303,6 +316,7 @@ describe('Bitcoind Functionality', function() { tx.to(destKey.toAddress(), utxos[1].amount * 1e8 - 1000); bitcoind.sendTransaction(tx.uncheckedSerialize(), function(err, hash) { should.exist(err); + (err instanceof Error).should.equal(true); should.not.exist(hash); done(); }); @@ -316,6 +330,7 @@ describe('Bitcoind Functionality', function() { var num = 23; bitcoind.sendTransaction(num, function(err, hash) { should.exist(err); + (err instanceof Error).should.equal(true); should.not.exist(hash); done(); }); From 88872734deaba1e03ebbbc189d922bc30a6ff412 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 10:44:34 -0400 Subject: [PATCH 082/299] bitcoind: add missing api methods to export --- lib/services/bitcoind.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 8319b57d..e989a8fc 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -83,6 +83,10 @@ Bitcoin.prototype.getAPIMethods = function() { ['getBlock', this, this.getBlock, 1], ['getBlockHeader', this, this.getBlockHeader, 1], ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], + ['getBestBlockHash', this, this.getBestBlockHash, 0], + ['getInfo', this, this.getInfo, 0], + ['syncPercentage', this, this.syncPercentage, 0], + ['isSynced', this, this.isSynced, 0], ['getTransaction', this, this.getTransaction, 2], ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], ['sendTransaction', this, this.sendTransaction, 1], From 4662ca0850dee6996608aa55d90f6fe3a5d3c5de Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 11:58:59 -0400 Subject: [PATCH 083/299] bitcoind: update jsdocs and cleanup --- lib/services/bitcoind.js | 111 +++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index e989a8fc..433c950d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -15,25 +15,41 @@ var _ = bitcore.deps._; var index = require('../'); var log = index.log; -var errors = index.errors; var Service = require('../service'); var Transaction = require('../transaction'); /** - * Provides an interface to native bindings to [Bitcoin Core](https://github.com/bitcoin/bitcoin) - * compiled as a static library. The C++ bindings can be found at `src/libbitcoind.cc` + * Provides a friendly event driven API to bitcoind in Node.js. Manages starting and + * stopping bitcoind as a child process for application support, as well as connecting + * to multiple bitcoind processes for server infrastructure. Results are cached in an + * LRU cache for improved performance and methods added for common queries. + * * @param {Object} options * @param {Node} options.node - A reference to the node */ function Bitcoin(options) { - /* jshint maxstatements: 20 */ - var self = this; if (!(this instanceof Bitcoin)) { return new Bitcoin(options); } Service.call(this, options); + this.options = options; + + this._initCaches(); + + // bitcoind child process + this.spawn = false; + // available bitcoind nodes + this._initClients(); +} +util.inherits(Bitcoin, Service); + +Bitcoin.dependencies = []; + +Bitcoin.DEFAULT_CONFIG = 'whitelist=127.0.0.1\n' + 'txindex=1\n' + 'addressindex=1\n' + 'server=1\n'; + +Bitcoin.prototype._initCaches = function() { // caches valid until there is a new block this.utxosCache = LRU(50000); this.txidsCache = LRU(50000); @@ -49,13 +65,10 @@ function Bitcoin(options) { this.zmqKnownTransactions = LRU(50); this.zmqLastBlock = 0; this.zmqUpdateTipTimeout = false; +}; - this.options = options; - - // bitcoind child process - this.spawn = false; - - // available bitcoind nodes +Bitcoin.prototype._initClients = function() { + var self = this; this.nodes = []; this.nodesIndex = 0; Object.defineProperty(this, 'client', { @@ -67,13 +80,7 @@ function Bitcoin(options) { enumerable: true, configurable: false }); - -} -util.inherits(Bitcoin, Service); - -Bitcoin.dependencies = []; - -Bitcoin.DEFAULT_CONFIG = 'whitelist=127.0.0.1\n' + 'txindex=1\n' + 'addressindex=1\n' + 'server=1\n'; +}; /** * Called by Node to determine the available API methods. @@ -507,7 +514,6 @@ Bitcoin.prototype.start = function(callback) { /** * Helper to determine the state of the database. * @param {Function} callback - * @returns {Boolean} If the database is fully synced */ Bitcoin.prototype.isSynced = function(callback) { this.syncPercentage(function(err, percentage) { @@ -525,7 +531,6 @@ Bitcoin.prototype.isSynced = function(callback) { /** * Helper to determine the progress of the database. * @param {Function} callback - * @returns {Number} An estimated percentage of the syncronization status */ Bitcoin.prototype.syncPercentage = function(callback) { var self = this; @@ -538,6 +543,12 @@ Bitcoin.prototype.syncPercentage = function(callback) { }); }; +/** + * Will get the balance for an address or multiple addresses + * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses + * @param {Object} options + * @param {Function} callback + */ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { var self = this; var addresses = [addressArg]; @@ -561,6 +572,12 @@ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { } }; +/** + * Will get the unspent outputs for an address or multiple addresses + * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses + * @param {Object} options + * @param {Function} callback + */ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callback) { var self = this; var addresses = [addressArg]; @@ -605,6 +622,12 @@ Bitcoin.prototype._getTxidsFromMempool = function(deltas) { return mempoolTxids; }; +/** + * Will get the txids for an address or multiple addresses + * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses + * @param {Object} options + * @param {Function} callback + */ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { var self = this; var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; @@ -776,6 +799,12 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { return txids; }; +/** + * Will detailed transaction history for an address or multiple addresses + * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses + * @param {Object} options + * @param {Function} callback + */ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var self = this; var addresses = [addressArg]; @@ -818,6 +847,12 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { }); }; +/** + * Will get the summary including txids and balance for an address or multiple addresses + * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses + * @param {Object} options + * @param {Function} callback + */ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; var summary = {}; @@ -896,8 +931,9 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { }; /** - * Will retrieve a block as a Node.js Buffer from disk. + * Will retrieve a block as a Bitcore object * @param {String|Number} block - A block hash or block height number + * @param {Function} callback */ Bitcoin.prototype.getBlock = function(blockArg, callback) { // TODO apply performance patch to the RPC method for raw data @@ -935,6 +971,12 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { }; +/** + * Will retrieve an array of block hashes within a range of timestamps + * @param {Number} high - The more recent timestamp in seconds + * @param {Number} low - The older timestamp in seconds + * @param {Function} callback + */ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { var self = this; self.client.getBlockHashes(high, low, function(err, response) { @@ -955,7 +997,7 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { * height: 10 * } * @param {String|Number} block - A block hash or block height - * @returns {Object} + * @param {Function} callback */ Bitcoin.prototype.getBlockHeader = function(block, callback) { var self = this; @@ -990,7 +1032,7 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { /** * Will estimate the fee per kilobyte. * @param {Number} blocks - The number of blocks for the transaction to be confirmed. - * @returns {Number} + * @param {Function} callback */ Bitcoin.prototype.estimateFee = function(blocks, callback) { var self = this; @@ -1003,10 +1045,10 @@ Bitcoin.prototype.estimateFee = function(blocks, callback) { }; /** - * Will add a transaction to the mempool and relay to connected peers, the function - * will throw an error if there were validation problems. - * @param {String} transaction - The hex string of the transaction - * @param {Boolean} allowAbsurdFees - Enable large fees + * Will add a transaction to the mempool and relay to connected peers + * @param {String|Transaction} transaction - The hex string of the transaction + * @param {Boolean=} allowAbsurdFees - Enable large fees + * @param {Function} callback */ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { var self = this; @@ -1031,7 +1073,7 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { }; /** - * Will get a transaction as a Node.js Buffer from disk and the mempool. + * Will get a transaction as a Bitcore Transaction. Results include the mempool. * @param {String} txid - The transaction hash * @param {Boolean} queryMempool - Include the mempool * @param {Function} callback @@ -1059,12 +1101,11 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { }; /** - * Will get a transaction with additional information about the block, in the format: + * Will get a transaction as Bitcore Transaction with additional fields: * { - * blockHash: '2725743288feae6bdaa976590af7cb12d7b535b5a242787de6d2789c73682ed1', - * height: 48, - * timestamp: 1442951110, // in seconds - * buffer: // transaction buffer + * __blockHash: '2725743288feae6bdaa976590af7cb12d7b535b5a242787de6d2789c73682ed1', + * __height: 48, + * __timestamp: 1442951110, // in seconds * } * @param {String} txid - The transaction hash * @param {Boolean} queryMempool - Include the mempool @@ -1100,7 +1141,7 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal /** * Will get the best block hash for the chain. - * @returns {String} + * @param {Function} callback */ Bitcoin.prototype.getBestBlockHash = function(callback) { var self = this; @@ -1127,9 +1168,11 @@ Bitcoin.prototype.getInputForOutput = function(txid, index, options, callback) { * connections: 0, * difficulty: 4.6565423739069247e-10, * testnet: false, + * network: 'testnet' * relayfee: 1000, * errors: '' * } + * @param {Function} callback */ Bitcoin.prototype.getInfo = function(callback) { var self = this; From 9bf6941fdf752aacc43aed1b021c9b9c7532233a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 14:44:24 -0400 Subject: [PATCH 084/299] test: update node regtest --- lib/node.js | 4 +- lib/services/bitcoind.js | 52 ++++++- regtest/node.js | 324 ++++++++++++++++++--------------------- 3 files changed, 203 insertions(+), 177 deletions(-) diff --git a/lib/node.js b/lib/node.js index 893dba1c..9dddf106 100644 --- a/lib/node.js +++ b/lib/node.js @@ -218,7 +218,9 @@ Node.prototype._startService = function(serviceInfo, callback) { Node.prototype._logTitle = function() { console.log('\n\n\n\n\n\n\n\n\n\n\n\n'); - log.info('Using config:', this.configPath); + if (this.configPath) { + log.info('Using config:', this.configPath); + } }; diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 433c950d..b45766f0 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -40,6 +40,11 @@ function Bitcoin(options) { // bitcoind child process this.spawn = false; + // event subscribers + this.subscriptions = {}; + this.subscriptions.transaction = []; + this.subscriptions.block = []; + // available bitcoind nodes this._initClients(); } @@ -107,6 +112,37 @@ Bitcoin.prototype.getAPIMethods = function() { return methods; }; +/** + * Called by the Bus to determine the available events. + */ +Bitcoin.prototype.getPublishEvents = function() { + return [ + { + name: 'bitcoind/transaction', + scope: this, + subscribe: this.subscribe.bind(this, 'transaction'), + unsubscribe: this.unsubscribe.bind(this, 'transaction') + }, + { + name: 'bitcoind/block', + scope: this, + subscribe: this.subscribe.bind(this, 'block'), + unsubscribe: this.unsubscribe.bind(this, 'block') + } + ]; +}; + +Bitcoin.prototype.subscribe = function(name, emitter) { + this.subscriptions[name].push(emitter); +}; + +Bitcoin.prototype.unsubscribe = function(name, emitter) { + var index = this.subscriptions[name].indexOf(emitter); + if (index > -1) { + this.subscriptions[name].splice(index, 1); + } +}; + Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ @@ -268,6 +304,9 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { if (err) { return log.error(err); } + if (Math.round(percentage) >= 100) { + self.emit('synced', self.height); + } log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); }); } @@ -289,6 +328,11 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { updateChain(); } + // Notify block subscribers + for (var i = 0; i < this.subscriptions.block.length; i++) { + this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); + } + }; Bitcoin.prototype._zmqTransactionHandler = function(node, message) { @@ -298,6 +342,12 @@ Bitcoin.prototype._zmqTransactionHandler = function(node, message) { self.zmqKnownTransactions[id] = true; self.emit('tx', message); } + + // Notify transaction subscribers + for (var i = 0; i < this.subscriptions.transaction.length; i++) { + this.subscriptions.transaction[i].emit('bitcoind/transaction', message); + } + }; Bitcoin.prototype._subscribeZmqEvents = function(node) { @@ -1128,7 +1178,7 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, cal var tx = Transaction(); tx.fromString(response.result.hex); tx.__blockHash = response.result.blockhash; - tx.__height = response.result.height; + tx.__height = response.result.height ? response.result.height : -1; tx.__timestamp = response.result.time; var confirmations = self._getConfirmationsDetail(tx); if (confirmations >= self.transactionInfoCacheConfirmations) { diff --git a/regtest/node.js b/regtest/node.js index fa031a38..c99b79f6 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -2,16 +2,12 @@ // To run the tests: $ mocha -R spec regtest/node.js +var path = require('path'); var index = require('..'); var async = require('async'); var log = index.log; log.debug = function() {}; -if (process.env.BITCORENODE_ENV !== 'test') { - log.info('Please set the environment variable BITCORENODE_ENV=test and make sure bindings are compiled for testing'); - process.exit(); -} - var chai = require('chai'); var bitcore = require('bitcore-lib'); var rimraf = require('rimraf'); @@ -23,10 +19,7 @@ var BitcoinRPC = require('bitcoind-rpc'); var index = require('..'); var Transaction = index.Transaction; var BitcoreNode = index.Node; -var AddressService = index.services.Address; var BitcoinService = index.services.Bitcoin; -var encoding = require('../lib/services/address/encoding'); -var DBService = index.services.DB; var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG'; var testKey; var client; @@ -39,7 +32,7 @@ describe('Node Functionality', function() { var regtest; before(function(done) { - this.timeout(30000); + this.timeout(20000); var datadir = __dirname + '/data'; @@ -52,23 +45,17 @@ describe('Node Functionality', function() { } var configuration = { - datadir: datadir, network: 'regtest', services: [ - { - name: 'db', - module: DBService, - config: {} - }, { name: 'bitcoind', module: BitcoinService, - config: {} - }, - { - name: 'address', - module: AddressService, - config: {} + config: { + spawn: { + datadir: datadir, + exec: path.resolve(__dirname, '../bin/bitcoind') + } + } } ] }; @@ -85,24 +72,24 @@ describe('Node Functionality', function() { node.on('ready', function() { client = new BitcoinRPC({ - protocol: 'https', + protocol: 'http', host: '127.0.0.1', - port: 18332, + port: 30331, user: 'bitcoin', pass: 'local321', rejectUnauthorized: false }); var syncedHandler = function() { - if (node.services.db.tip.__height === 150) { - node.removeListener('synced', syncedHandler); + if (node.services.bitcoind.height === 150) { + node.services.bitcoind.removeListener('synced', syncedHandler); done(); } }; - node.on('synced', syncedHandler); + node.services.bitcoind.on('synced', syncedHandler); - client.generate(150, function(err, response) { + client.generate(150, function(err) { if (err) { throw err; } @@ -131,7 +118,7 @@ describe('Node Functionality', function() { var invalidatedBlockHash; - it('will handle a reorganization', function(done) { + it.skip('will handle a reorganization', function(done) { var count; var blockHash; @@ -207,26 +194,31 @@ describe('Node Functionality', function() { }); - it('isMainChain() will return false for stale/orphan block', function(done) { - node.services.bitcoind.isMainChain(invalidatedBlockHash).should.equal(false); - setImmediate(done); - }); - describe('Bus Functionality', function() { it('subscribes and unsubscribes to an event on the bus', function(done) { var bus = node.openBus(); - var block; - bus.subscribe('db/block'); - bus.on('db/block', function(data) { - bus.unsubscribe('db/block'); - data.should.be.equal(block); - done(); + var blockExpected; + var blockReceived; + bus.subscribe('bitcoind/block'); + bus.on('bitcoind/block', function(data) { + bus.unsubscribe('bitcoind/block'); + if (blockExpected) { + data.should.be.equal(blockExpected); + done(); + } else { + blockReceived = data; + } }); client.generate(1, function(err, response) { if (err) { throw err; } - block = response.result[0]; + if (blockReceived) { + blockReceived.should.be.equal(response.result[0]); + done(); + } else { + blockExpected = response.result[0]; + } }); }); }); @@ -234,20 +226,36 @@ describe('Node Functionality', function() { describe('Address Functionality', function() { var address; var unspentOutput; - before(function() { + before(function(done) { address = testKey.toAddress(regtest).toString(); + var startHeight = node.services.bitcoind.height; + node.services.bitcoind.on('tip', function(height) { + if (height === startHeight + 3) { + done(); + } + }); + client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { + if (err) { + throw err; + } + client.generate(3, function(err) { + if (err) { + throw err; + } + }); + }); }); it('should be able to get the balance of the test address', function(done) { - node.services.address.getBalance(address, false, function(err, balance) { + node.getAddressBalance(address, false, function(err, data) { if (err) { throw err; } - balance.should.equal(10 * 1e8); + data.balance.should.equal(10 * 1e8); done(); }); }); it('can get unspent outputs for address', function(done) { - node.services.address.getUnspentOutputs(address, false, function(err, results) { + node.getAddressUnspentOutputs(address, false, function(err, results) { if (err) { throw err; } @@ -262,7 +270,7 @@ describe('Node Functionality', function() { to: 10, queryMempool: false }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -276,7 +284,7 @@ describe('Node Functionality', function() { info.satoshis.should.equal(10 * 1e8); info.confirmations.should.equal(3); info.timestamp.should.be.a('number'); - info.fees.should.be.within(950, 970); + info.fees.should.be.within(950, 4000); info.tx.should.be.an.instanceof(Transaction); done(); }); @@ -285,16 +293,16 @@ describe('Node Functionality', function() { var options = { queryMempool: false }; - node.services.address.getAddressSummary(address, options, function(err, results) { + node.getAddressSummary(address, options, function(err, results) { if (err) { throw err; } results.totalReceived.should.equal(1000000000); results.totalSpent.should.equal(0); results.balance.should.equal(1000000000); - results.unconfirmedBalance.should.equal(0); + should.not.exist(results.unconfirmedBalance); results.appearances.should.equal(1); - results.unconfirmedAppearances.should.equal(0); + should.not.exist(results.unconfirmedAppearances); results.txids.length.should.equal(1); done(); }); @@ -317,6 +325,14 @@ describe('Node Functionality', function() { before(function(done) { /* jshint maxstatements: 50 */ + // Finished once all blocks have been mined + var startHeight = node.services.bitcoind.height; + node.services.bitcoind.on('tip', function(height) { + if (height === startHeight + 5) { + done(); + } + }); + testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x'); address2 = testKey2.toAddress(regtest).toString(); @@ -344,8 +360,6 @@ describe('Node Functionality', function() { unspentOutputSpentTxId = tx.id; - node.services.bitcoind.sendTransaction(tx.serialize()); - function mineBlock(next) { client.generate(1, function(err, response) { if (err) { @@ -356,13 +370,18 @@ describe('Node Functionality', function() { }); } - client.generate(1, function(err, response) { + node.sendTransaction(tx.serialize(), function(err, hash) { if (err) { - throw err; + return done(err); } - should.exist(response); - node.once('synced', function() { - node.services.address.getUnspentOutputs(address, false, function(err, results) { + + client.generate(1, function(err, response) { + if (err) { + throw err; + } + should.exist(response); + + node.getAddressUnspentOutputs(address, false, function(err, results) { /* jshint maxstatements: 50 */ if (err) { throw err; @@ -376,24 +395,36 @@ describe('Node Functionality', function() { tx2.to(address2, results[0].satoshis - 10000); tx2.change(address); tx2.sign(testKey); - node.services.bitcoind.sendTransaction(tx2.serialize()); - mineBlock(next); + node.sendTransaction(tx2.serialize(), function(err) { + if (err) { + return next(err); + } + mineBlock(next); + }); }, function(next) { var tx3 = new Transaction(); tx3.from(results[1]); tx3.to(address3, results[1].satoshis - 10000); tx3.change(address); tx3.sign(testKey); - node.services.bitcoind.sendTransaction(tx3.serialize()); - mineBlock(next); + node.sendTransaction(tx3.serialize(), function(err) { + if (err) { + return next(err); + } + mineBlock(next); + }); }, function(next) { var tx4 = new Transaction(); tx4.from(results[2]); tx4.to(address4, results[2].satoshis - 10000); tx4.change(address); tx4.sign(testKey); - node.services.bitcoind.sendTransaction(tx4.serialize()); - mineBlock(next); + node.sendTransaction(tx4.serialize(), function(err) { + if (err) { + return next(err); + } + mineBlock(next); + }); }, function(next) { var tx5 = new Transaction(); tx5.from(results[3]); @@ -402,19 +433,22 @@ describe('Node Functionality', function() { tx5.to(address6, results[4].satoshis - 10000); tx5.change(address); tx5.sign(testKey); - node.services.bitcoind.sendTransaction(tx5.serialize()); - mineBlock(next); + node.sendTransaction(tx5.serialize(), function(err) { + if (err) { + return next(err); + } + mineBlock(next); + }); } ], function(err) { if (err) { throw err; } - node.once('synced', function() { - done(); - }); }); }); + }); + }); }); @@ -428,20 +462,20 @@ describe('Node Functionality', function() { address6 ]; var options = {}; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } results.totalCount.should.equal(4); var history = results.items; history.length.should.equal(4); - history[0].height.should.equal(157); + history[0].height.should.equal(159); history[0].confirmations.should.equal(1); - history[1].height.should.equal(156); + history[1].height.should.equal(158); should.exist(history[1].addresses[address4]); - history[2].height.should.equal(155); + history[2].height.should.equal(157); should.exist(history[2].addresses[address3]); - history[3].height.should.equal(154); + history[3].height.should.equal(156); should.exist(history[3].addresses[address2]); history[3].satoshis.should.equal(99990000); history[3].confirmations.should.equal(4); @@ -449,7 +483,7 @@ describe('Node Functionality', function() { }); }); - it('five addresses (limited by height)', function(done) { + it.skip('five addresses (limited by height)', function(done) { var addresses = [ address2, address3, @@ -461,7 +495,7 @@ describe('Node Functionality', function() { start: 157, end: 156 }; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } @@ -476,7 +510,7 @@ describe('Node Functionality', function() { }); }); - it('five addresses (limited by height 155 to 154)', function(done) { + it.skip('five addresses (limited by height 155 to 154)', function(done) { var addresses = [ address2, address3, @@ -488,7 +522,7 @@ describe('Node Functionality', function() { start: 155, end: 154 }; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } @@ -501,7 +535,7 @@ describe('Node Functionality', function() { }); }); - it('five addresses (paginated by index)', function(done) { + it.skip('five addresses (paginated by index)', function(done) { var addresses = [ address2, address3, @@ -513,7 +547,7 @@ describe('Node Functionality', function() { from: 0, to: 3 }; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } @@ -533,32 +567,32 @@ describe('Node Functionality', function() { address ]; var options = {}; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } results.totalCount.should.equal(6); var history = results.items; history.length.should.equal(6); - history[0].height.should.equal(157); + history[0].height.should.equal(159); history[0].addresses[address].inputIndexes.should.deep.equal([0, 1]); history[0].addresses[address].outputIndexes.should.deep.equal([2]); history[0].confirmations.should.equal(1); - history[1].height.should.equal(156); - history[2].height.should.equal(155); - history[3].height.should.equal(154); - history[4].height.should.equal(153); + history[1].height.should.equal(158); + history[2].height.should.equal(157); + history[3].height.should.equal(156); + history[4].height.should.equal(155); history[4].satoshis.should.equal(-10000); history[4].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); history[4].addresses[address].inputIndexes.should.deep.equal([0]); - history[5].height.should.equal(150); + history[5].height.should.equal(152); history[5].satoshis.should.equal(10 * 1e8); done(); }); }); it('summary for an address (sending and receiving)', function(done) { - node.services.address.getAddressSummary(address, {}, function(err, results) { + node.getAddressSummary(address, {}, function(err, results) { if (err) { throw err; } @@ -579,7 +613,7 @@ describe('Node Functionality', function() { address ]; var options = {}; - node.services.address.getAddressHistory(addresses, options, function(err, results) { + node.getAddressHistory(addresses, options, function(err, results) { if (err) { throw err; } @@ -588,13 +622,13 @@ describe('Node Functionality', function() { }); }); - describe('Pagination', function() { + describe.skip('Pagination', function() { it('from 0 to 1', function(done) { var options = { from: 0, to: 1 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -609,7 +643,7 @@ describe('Node Functionality', function() { from: 1, to: 2 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -624,7 +658,7 @@ describe('Node Functionality', function() { from: 2, to: 3 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -639,7 +673,7 @@ describe('Node Functionality', function() { from: 3, to: 4 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -654,7 +688,7 @@ describe('Node Functionality', function() { from: 4, to: 5 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -672,7 +706,7 @@ describe('Node Functionality', function() { from: 5, to: 6 }; - node.services.address.getAddressHistory(address, options, function(err, results) { + node.getAddressHistory(address, options, function(err, results) { if (err) { throw err; } @@ -690,7 +724,7 @@ describe('Node Functionality', function() { describe('Mempool Index', function() { var unspentOutput; before(function(done) { - node.services.address.getUnspentOutputs(address, false, function(err, results) { + node.getAddressUnspentOutputs(address, false, function(err, results) { if (err) { throw err; } @@ -701,23 +735,20 @@ describe('Node Functionality', function() { }); it('will update the mempool index after new tx', function(done) { - + var memAddress = bitcore.PrivateKey().toAddress(node.network).toString(); var tx = new Transaction(); tx.from(unspentOutput); - tx.to(address, unspentOutput.satoshis - 1000); + tx.to(memAddress, unspentOutput.satoshis - 1000); tx.fee(1000); tx.sign(testKey); - node.services.bitcoind.sendTransaction(tx.serialize()); - - setImmediate(function() { - var addrObj = encoding.getAddressInfo(address); - node.services.address._getOutputsMempool(address, addrObj.hashBuffer, - addrObj.hashTypeBuffer, function(err, outs) { + node.services.bitcoind.sendTransaction(tx.serialize(), function(err, hash) { + node.getAddressTxids(memAddress, {}, function(err, txids) { if (err) { - throw err; + return done(err); } - outs.length.should.equal(1); + txids.length.should.equal(1); + txids[0].should.equal(hash); done(); }); }); @@ -725,73 +756,6 @@ describe('Node Functionality', function() { }); - describe('#getInputForOutput(db)', function() { - it('will get the input txid and input index', function(done) { - var txid = outputForIsSpentTest1.txid; - var outputIndex = outputForIsSpentTest1.outputIndex; - var options = { - queryMempool: true - }; - node.services.address.getInputForOutput(txid, outputIndex, options, function(err, result) { - result.inputTxId.should.equal(unspentOutputSpentTxId); - result.inputIndex.should.equal(0); - done(); - }); - }); - }); - - describe('#isSpent and #getInputForOutput(mempool)', function() { - var spentOutput; - var spentOutputInputTxId; - it('will return true if an input is spent in a confirmed transaction', function(done) { - var txid = outputForIsSpentTest1.txid; - var outputIndex = outputForIsSpentTest1.outputIndex; - var result = node.services.bitcoind.isSpent(txid, outputIndex); - result.should.equal(true); - done(); - }); - //CCoinsViewMemPool only checks for spent outputs that are not the mempool - it('will correctly return false for an input that is spent in an unconfirmed transaction', function(done) { - node.services.address.getUnspentOutputs(address, false, function(err, results) { - if (err) { - throw err; - } - - var unspentOutput = results[0]; - - var tx = new Transaction(); - tx.from(unspentOutput); - tx.to(address, unspentOutput.satoshis - 1000); - tx.fee(1000); - tx.sign(testKey); - - node.services.bitcoind.sendTransaction(tx.serialize()); - spentOutput = unspentOutput; - spentOutputInputTxId = tx.hash; - - setImmediate(function() { - var result = node.services.bitcoind.isSpent(unspentOutput.txid, unspentOutput.outputIndex); - result.should.equal(false); - done(); - }); - }); - }); - - it('will get the input txid and input index (mempool)', function(done) { - var txid = spentOutput.txid; - var outputIndex = spentOutput.outputIndex; - var options = { - queryMempool: true - }; - node.services.address.getInputForOutput(txid, outputIndex, options, function(err, result) { - result.inputTxId.should.equal(spentOutputInputTxId); - result.inputIndex.should.equal(0); - done(); - }); - }); - - }); - }); describe('Orphaned Transactions', function() { @@ -802,6 +766,14 @@ describe('Node Functionality', function() { var invalidatedBlockHash; async.series([ + function(next) { + client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { + if (err) { + return next(err); + } + client.generate(1, next); + }); + }, function(next) { client.getBlockCount(function(err, response) { if (err) { @@ -826,6 +798,7 @@ describe('Node Functionality', function() { return next(err); } orphanedTransaction = response.result.tx[1]; + should.exist(orphanedTransaction); next(); }); }, @@ -843,12 +816,13 @@ describe('Node Functionality', function() { it('will not show confirmation count for orphaned transaction', function(done) { // This test verifies that in the situation that the transaction is not in the mempool and // is included in an orphaned block transaction index that the confirmation count will be unconfirmed. - node.services.bitcoind.getTransactionWithBlockInfo(orphanedTransaction, false, function(err, data) { + node.getTransactionWithBlockInfo(orphanedTransaction, false, function(err, data) { if (err) { return done(err); } - should.exist(data.height); - data.height.should.equal(-1); + should.exist(data); + should.exist(data.__height); + data.__height.should.equal(-1); done(); }); }); From fd427fa128e3da66f8b096d9124d6287dc2e7309 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 14:49:58 -0400 Subject: [PATCH 085/299] test: increase timeout and remove new lines --- lib/node.js | 1 - regtest/node.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/node.js b/lib/node.js index 9dddf106..f9c49787 100644 --- a/lib/node.js +++ b/lib/node.js @@ -217,7 +217,6 @@ Node.prototype._startService = function(serviceInfo, callback) { }; Node.prototype._logTitle = function() { - console.log('\n\n\n\n\n\n\n\n\n\n\n\n'); if (this.configPath) { log.info('Using config:', this.configPath); } diff --git a/regtest/node.js b/regtest/node.js index c99b79f6..20f91caa 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -759,6 +759,7 @@ describe('Node Functionality', function() { }); describe('Orphaned Transactions', function() { + this.timeout(8000); var orphanedTransaction; before(function(done) { From 1013ad3c56cdf6b39ea6decd812bc71a563f7074 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 14:58:45 -0400 Subject: [PATCH 086/299] build: upgrade chai and mocha --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 0cdfbf4b..8afbee16 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,8 @@ "devDependencies": { "benchmark": "1.0.0", "bitcore-p2p": "^1.1.0", - "chai": "^3.0.0", - "mocha": "~1.16.2", + "chai": "^3.5.0", + "mocha": "^2.4.5", "proxyquire": "^1.3.1", "rimraf": "^2.4.2", "sinon": "^1.15.4" From d11d0300de36cd2f95e2dff250e7cee0312fc65b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 15:29:57 -0400 Subject: [PATCH 087/299] bitcoind: spawn in default configs --- lib/scaffold/default-base-config.js | 6 ++++-- lib/scaffold/default-config.js | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index eae48369..fc5f8c6d 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -20,8 +20,10 @@ function getDefaultBaseConfig(options) { services: ['bitcoind', 'web'], servicesConfig: { bitcoind: { - datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), - exec: path.resolve(__dirname, '../../bin/bitcoind') + spawn: { + datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), + exec: path.resolve(__dirname, '../../bin/bitcoind') + } } } } diff --git a/lib/scaffold/default-config.js b/lib/scaffold/default-config.js index 8a477300..7075a7fc 100644 --- a/lib/scaffold/default-config.js +++ b/lib/scaffold/default-config.js @@ -36,8 +36,10 @@ function getDefaultConfig(options) { services: defaultServices, servicesConfig: { bitcoind: { - datadir: path.resolve(defaultPath, './data'), - exec: path.resolve(__dirname, '../../bin/bitcoind') + spawn: { + datadir: path.resolve(defaultPath, './data'), + exec: path.resolve(__dirname, '../../bin/bitcoind') + } } } }; From b4b560aa450f7837c873e8d42c75197827cf7560 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 15:49:18 -0400 Subject: [PATCH 088/299] bitcoind: get blocks and transactions as buffers --- lib/services/bitcoind.js | 79 +++++++++++++++++++++++++++++++++++++--- regtest/bitcoind.js | 48 ++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b45766f0..d6283cd5 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -63,9 +63,11 @@ Bitcoin.prototype._initCaches = function() { // caches valid indefinitely this.transactionCache = LRU(100000); + this.rawTransactionCache = LRU(50000); this.transactionInfoCache = LRU(100000); this.transactionInfoCacheConfirmations = 6; this.blockCache = LRU(144); + this.rawBlockCache = LRU(72); this.blockHeaderCache = LRU(288); this.zmqKnownTransactions = LRU(50); this.zmqLastBlock = 0; @@ -93,12 +95,14 @@ Bitcoin.prototype._initClients = function() { Bitcoin.prototype.getAPIMethods = function() { var methods = [ ['getBlock', this, this.getBlock, 1], + ['getRawBlock', this, this.getRawBlock, 1], ['getBlockHeader', this, this.getBlockHeader, 1], ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], ['getBestBlockHash', this, this.getBestBlockHash, 0], ['getInfo', this, this.getInfo, 0], ['syncPercentage', this, this.syncPercentage, 0], ['isSynced', this, this.isSynced, 0], + ['getRawTransaction', this, this.getRawTransaction, 1], ['getTransaction', this, this.getTransaction, 2], ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], ['sendTransaction', this, this.sendTransaction, 1], @@ -791,11 +795,9 @@ Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addre */ Bitcoin.prototype._getDetailedTransaction = function(txid, options, next) { var self = this; - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; self.getTransactionWithBlockInfo( txid, - queryMempool, function(err, transaction) { if (err) { return next(err); @@ -980,6 +982,46 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { }; +/** + * Will retrieve a block as a Node.js Buffer + * @param {String|Number} block - A block hash or block height number + * @param {Function} callback + */ +Bitcoin.prototype.getRawBlock = function(blockArg, callback) { + // TODO apply performance patch to the RPC method for raw data + var self = this; + + function queryBlock(blockhash) { + self.client.getBlock(blockhash, false, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + var buffer = new Buffer(response.result, 'hex'); + self.rawBlockCache.set(blockhash, buffer); + callback(null, buffer); + }); + } + + var cachedBlock = self.rawBlockCache.get(blockArg); + if (cachedBlock) { + return setImmediate(function() { + callback(null, cachedBlock); + }); + } else { + if (_.isNumber(blockArg)) { + self.client.getBlockHash(blockArg, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + var blockhash = response.result; + queryBlock(blockhash); + }); + } else { + queryBlock(blockArg); + } + } +}; + /** * Will retrieve a block as a Bitcore object * @param {String|Number} block - A block hash or block height number @@ -995,7 +1037,7 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { return callback(self._wrapRPCError(err)); } var blockObj = bitcore.Block.fromString(response.result); - self.blockCache.set(blockArg, blockObj); + self.blockCache.set(blockhash, blockObj); callback(null, blockObj); }); } @@ -1122,13 +1164,39 @@ Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { }; +/** + * Will get a transaction as a Node.js Buffer. Results include the mempool. + * @param {String} txid - The transaction hash + * @param {Function} callback + */ +Bitcoin.prototype.getRawTransaction = function(txid, callback) { + var self = this; + var tx = self.rawTransactionCache.get(txid); + if (tx) { + return setImmediate(function() { + callback(null, tx); + }); + } else { + self.client.getRawTransaction(txid, function(err, response) { + if (err && response.error.code === -5) { + return callback(null, null); + } else if (err) { + return callback(self._wrapRPCError(err)); + } + var buffer = new Buffer(response.result, 'hex'); + self.rawTransactionCache.set(txid, buffer); + callback(null, buffer); + }); + } +}; + /** * Will get a transaction as a Bitcore Transaction. Results include the mempool. * @param {String} txid - The transaction hash * @param {Boolean} queryMempool - Include the mempool * @param {Function} callback */ -Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { +Bitcoin.prototype.getTransaction = function(txid, callback) { var self = this; var tx = self.transactionCache.get(txid); if (tx) { @@ -1158,10 +1226,9 @@ Bitcoin.prototype.getTransaction = function(txid, queryMempool, callback) { * __timestamp: 1442951110, // in seconds * } * @param {String} txid - The transaction hash - * @param {Boolean} queryMempool - Include the mempool * @param {Function} callback */ -Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) { +Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { var self = this; var tx = self.transactionInfoCache.get(txid); if (tx) { diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 9b7a8f05..b9adf95e 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -149,6 +149,21 @@ describe('Bitcoind Functionality', function() { }); }); + describe('get blocks as buffers', function() { + [0,1,2,3,5,6,7,8,9].forEach(function(i) { + it('generated block ' + i, function(done) { + bitcoind.getRawBlock(blockHashes[i], function(err, block) { + if (err) { + throw err; + } + should.exist(block); + (block instanceof Buffer).should.equal(true); + done(); + }); + }); + }); + }); + describe('get errors as error instances', function() { it('will wrap an rpc into a javascript error', function(done) { bitcoind.client.getBlock(1000000000, function(err, response) { @@ -194,7 +209,7 @@ describe('Bitcoind Functionality', function() { var txhex = transactionData[i]; var tx = new bitcore.Transaction(); tx.fromString(txhex); - bitcoind.getTransaction(tx.hash, true, function(err, response) { + bitcoind.getTransaction(tx.hash, function(err, response) { if (err) { throw err; } @@ -206,7 +221,7 @@ describe('Bitcoind Functionality', function() { it('will return null if the transaction does not exist', function(done) { var txid = '6226c407d0e9705bdd7158e60983e37d0f5d23529086d6672b07d9238d5aa618'; - bitcoind.getTransaction(txid, true, function(err, response) { + bitcoind.getTransaction(txid, function(err, response) { if (err) { throw err; } @@ -214,7 +229,34 @@ describe('Bitcoind Functionality', function() { done(); }); }); + }); + + describe('get transactions as buffers', function() { + [0,1,2,3,4,5,6,7,8,9].forEach(function(i) { + it('for tx ' + i, function(done) { + var txhex = transactionData[i]; + var tx = new bitcore.Transaction(); + tx.fromString(txhex); + bitcoind.getRawTransaction(tx.hash, function(err, response) { + if (err) { + throw err; + } + assert(response.toString('hex') === txhex, 'incorrect tx data result'); + done(); + }); + }); + }); + it('will return null if the transaction does not exist', function(done) { + var txid = '6226c407d0e9705bdd7158e60983e37d0f5d23529086d6672b07d9238d5aa618'; + bitcoind.getRawTransaction(txid, function(err, response) { + if (err) { + throw err; + } + should.not.exist(response); + done(); + }); + }); }); describe('get block header', function() { @@ -390,7 +432,7 @@ describe('Bitcoind Functionality', function() { describe('get transaction with block info', function() { it('should include tx buffer, height and timestamp', function(done) { - bitcoind.getTransactionWithBlockInfo(utxos[0].txid, true, function(err, tx) { + bitcoind.getTransactionWithBlockInfo(utxos[0].txid, function(err, tx) { if (err) { return done(err); } From 3713c6ac1e6adaade7a5c33d1af904ef8e9342d4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 15:52:57 -0400 Subject: [PATCH 089/299] bitcoind: sendTransaction second arg as object --- lib/services/bitcoind.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index d6283cd5..1cc72337 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1139,20 +1139,23 @@ Bitcoin.prototype.estimateFee = function(blocks, callback) { /** * Will add a transaction to the mempool and relay to connected peers * @param {String|Transaction} transaction - The hex string of the transaction - * @param {Boolean=} allowAbsurdFees - Enable large fees + * @param {Object=} options + * @param {Boolean=} options.allowAbsurdFees - Enable large fees * @param {Function} callback */ -Bitcoin.prototype.sendTransaction = function(tx, allowAbsurdFees, callback) { +Bitcoin.prototype.sendTransaction = function(tx, options, callback) { var self = this; + var allowAbsurdFees = false; var txString; if (tx instanceof Transaction) { txString = tx.serialize(); } else { txString = tx; } - if (_.isFunction(allowAbsurdFees) && _.isUndefined(callback)) { - callback = allowAbsurdFees; - allowAbsurdFees = false; + if (_.isFunction(options) && _.isUndefined(callback)) { + callback = options; + } else if (_.isObject(options)) { + allowAbsurdFees = options.allowAbsurdFees; } this.client.sendRawTransaction(txString, allowAbsurdFees, function(err, response) { From 90e354093ceb1154e3809437495f1c8f6802c407 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 15:54:04 -0400 Subject: [PATCH 090/299] bitcoind: increase reindex interval to 10s --- lib/services/bitcoind.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 1cc72337..30a0b968 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -433,7 +433,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { var node = {}; node._reindex = false; - node._reindexWait = 1000; + node._reindexWait = 10000; try { self._loadSpawnConfiguration(node); From f3f2f5961570f456e61d47ecf302bb45ed931be1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 16:00:33 -0400 Subject: [PATCH 091/299] node: add getNetworkName method --- lib/node.js | 8 ++++++++ lib/services/bitcoind.js | 5 +---- regtest/bitcoind.js | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/node.js b/lib/node.js index f9c49787..4d80fc48 100644 --- a/lib/node.js +++ b/lib/node.js @@ -248,6 +248,14 @@ Node.prototype.start = function(callback) { ); }; +Node.prototype.getNetworkName = function() { + var network = this.network.name; + if (this.network.regtestEnabled) { + network = 'regtest'; + } + return network; +}; + /** * Will stop all running services in the reverse order that they * were initially started. diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 30a0b968..be8ce904 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1301,10 +1301,7 @@ Bitcoin.prototype.getInfo = function(callback) { return callback(self._wrapRPCError(err)); } var result = response.result; - result.network = self.node.network.name; - if (self.node.network.regtestEnabled) { - result.network = 'regtest'; - } + result.network = self.node.getNetworkName(); callback(null, result); }); }; diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index b9adf95e..8aa4d71e 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -49,7 +49,10 @@ describe('Bitcoind Functionality', function() { exec: path.resolve(__dirname, '../bin/bitcoind') }, node: { - network: regtestNetwork + network: regtestNetwork, + getNetworkName: function() { + return 'regtest'; + } } }); From 8102761b55a141707368fb2c47847e496c2cf8a4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 16:04:27 -0400 Subject: [PATCH 092/299] bitcoind: normalize address arguments --- lib/services/bitcoind.js | 41 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index be8ce904..f2e5c89a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -597,6 +597,14 @@ Bitcoin.prototype.syncPercentage = function(callback) { }); }; +Bitcoin.prototype._normalizeAddressArg = function(addressArg) { + var addresses = [addressArg]; + if (Array.isArray(addressArg)) { + addresses = addressArg; + } + return addresses; +}; + /** * Will get the balance for an address or multiple addresses * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses @@ -605,10 +613,7 @@ Bitcoin.prototype.syncPercentage = function(callback) { */ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { var self = this; - var addresses = [addressArg]; - if (Array.isArray(addressArg)) { - addresses = addressArg; - } + var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); var balance = self.balanceCache.get(cacheKey); if (balance) { @@ -634,10 +639,7 @@ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { */ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callback) { var self = this; - var addresses = [addressArg]; - if (Array.isArray(addressArg)) { - addresses = addressArg; - } + var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); var utxos = self.utxosCache.get(cacheKey); if (utxos) { @@ -685,10 +687,7 @@ Bitcoin.prototype._getTxidsFromMempool = function(deltas) { Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { var self = this; var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var addresses = [addressArg]; - if (Array.isArray(addressArg)) { - addresses = addressArg; - } + var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); var mempoolTxids = []; var txids = self.txidsCache.get(cacheKey); @@ -859,10 +858,7 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { */ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var self = this; - var addresses = [addressArg]; - if (_.isArray(addressArg)) { - addresses = addressArg; - } + var addresses = self._normalizeAddressArg(addressArg); if (addresses.length > this.maxAddressesQuery) { return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); } @@ -911,18 +907,13 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var summaryTxids = []; var mempoolTxids = []; - - var addresses = [addressArg]; - if (Array.isArray(addressArg)) { - addresses = addressArg; - } - + var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); function querySummary() { async.parallel([ function getTxList(done) { - self.getAddressTxids(addressArg, {queryMempool: false}, function(err, txids) { + self.getAddressTxids(addresses, {queryMempool: false}, function(err, txids) { if (err) { return done(err); } @@ -932,7 +923,7 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { }); }, function getBalance(done) { - self.getAddressBalance(addressArg, options, function(err, data) { + self.getAddressBalance(addresses, options, function(err, data) { if (err) { return done(err); } @@ -946,7 +937,7 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { if (!queryMempool) { return done(); } - self.client.getAddressMempool({'addresses': [addressArg]}, function(err, response) { + self.client.getAddressMempool({'addresses': addresses}, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } From dbcb70f839f2145ca75a770dcbee47471b789923 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 16:16:41 -0400 Subject: [PATCH 093/299] transaction: update getTransaction arguments --- lib/transaction.js | 2 +- regtest/node.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/transaction.js b/lib/transaction.js index 1acb350d..adce3086 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -31,7 +31,7 @@ Transaction.prototype._populateInput = function(db, input, poolTransactions, cal return callback(new Error('Input is expected to have prevTxId as a buffer')); } var txid = input.prevTxId.toString('hex'); - db.getTransaction(txid, true, function(err, prevTx) { + db.getTransaction(txid, function(err, prevTx) { if(!prevTx) { // Check the pool for transaction for(var i = 0; i < poolTransactions.length; i++) { diff --git a/regtest/node.js b/regtest/node.js index 20f91caa..a9e09d9e 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -817,7 +817,7 @@ describe('Node Functionality', function() { it('will not show confirmation count for orphaned transaction', function(done) { // This test verifies that in the situation that the transaction is not in the mempool and // is included in an orphaned block transaction index that the confirmation count will be unconfirmed. - node.getTransactionWithBlockInfo(orphanedTransaction, false, function(err, data) { + node.getTransactionWithBlockInfo(orphanedTransaction, function(err, data) { if (err) { return done(err); } From d7f49cc19280b380432f4efc66d03e3d76930098 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 8 Apr 2016 22:17:45 -0400 Subject: [PATCH 094/299] test: add regtest for multiple bitcoind connections --- .gitignore | 5 +- .travis.yml | 1 + lib/services/bitcoind.js | 29 ++++- regtest/cluster.js | 184 ++++++++++++++++++++++++++++++++ regtest/data/node1/bitcoin.conf | 14 +++ regtest/data/node2/bitcoin.conf | 14 +++ regtest/data/node3/bitcoin.conf | 14 +++ 7 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 regtest/cluster.js create mode 100644 regtest/data/node1/bitcoin.conf create mode 100644 regtest/data/node2/bitcoin.conf create mode 100644 regtest/data/node3/bitcoin.conf diff --git a/.gitignore b/.gitignore index e94d8aba..8132c320 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,7 @@ coverage/* **/*.creator *.log .DS_Store -bin/bitcoin* \ No newline at end of file +bin/bitcoin* +regtest/data/node1/regtest +regtest/data/node2/regtest +regtest/data/node3/regtest diff --git a/.travis.yml b/.travis.yml index 1192bff0..06a1ff97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ node_js: script: - _mocha -R spec regtest/p2p.js - _mocha -R spec regtest/bitcoind.js + - _mocha -R spec regtest/cluster.js - _mocha -R spec regtest/node.js - _mocha -R spec --recursive diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index f2e5c89a..220a8e53 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -111,7 +111,8 @@ Bitcoin.prototype.getAPIMethods = function() { ['getAddressBalance', this, this.getAddressBalance, 2], ['getAddressUnspentOutputs', this, this.getAddressUnspentOutputs, 2], ['getAddressHistory', this, this.getAddressHistory, 2], - ['getAddressSummary', this, this.getAddressSummary, 1] + ['getAddressSummary', this, this.getAddressSummary, 1], + ['generateBlock', this, this.generateBlock, 1] ]; return methods; }; @@ -372,6 +373,18 @@ Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { var self = this; node.zmqSubSocket = zmq.socket('sub'); + node.zmqSubSocket.on('connect', function(fd, endPoint) { + log.info('ZMQ connected to:', endPoint); + }); + + node.zmqSubSocket.on('connect_delay', function(fd, endPoint) { + log.warn('ZMQ connection delay:', endPoint); + }); + + node.zmqSubSocket.on('disconnect', function(fd, endPoint) { + log.warn('ZMQ disconnect:', endPoint); + }); + node.zmqSubSocket.on('monitor_error', function(err) { log.error('Error in monitoring: %s, will restart monitoring in 5 seconds', err); setTimeout(function() { @@ -394,7 +407,6 @@ Bitcoin.prototype._checkReindex = function(node, callback) { log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); if (Math.round(percentSynced) >= 100) { node._reindex = false; - self._subscribeZmqEvents(node); callback(); clearInterval(interval); } @@ -402,7 +414,6 @@ Bitcoin.prototype._checkReindex = function(node, callback) { }, self._reindexWait); } else { - self._subscribeZmqEvents(node); callback(); } }; @@ -481,6 +492,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { if (err) { return callback(err); } + self._subscribeZmqEvents(node); callback(null, node); }); @@ -512,6 +524,7 @@ Bitcoin.prototype._connectProcess = function(config, callback) { } self._initZmqSubSocket(node, config.zmqpubrawtx); + self._subscribeZmqEvents(node); callback(null, node); }); @@ -1297,6 +1310,16 @@ Bitcoin.prototype.getInfo = function(callback) { }); }; +Bitcoin.prototype.generateBlock = function(num, callback) { + var self = this; + this.client.generate(num, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + callback(null, response.result); + }); +}; + /** * Called by Node to stop the service. * @param {Function} callback diff --git a/regtest/cluster.js b/regtest/cluster.js new file mode 100644 index 00000000..30db5377 --- /dev/null +++ b/regtest/cluster.js @@ -0,0 +1,184 @@ +'use strict'; + +var path = require('path'); +var async = require('async'); +var spawn = require('child_process').spawn; + +var BitcoinRPC = require('bitcoind-rpc'); +var rimraf = require('rimraf'); +var bitcore = require('bitcore-lib'); +var chai = require('chai'); +var should = chai.should(); + +var index = require('..'); +var log = index.log; +log.debug = function() {}; +var BitcoreNode = index.Node; +var BitcoinService = index.services.Bitcoin; + +describe('Bitcoin Cluster', function() { + var node; + var daemons = []; + var execPath = path.resolve(__dirname, '../bin/bitcoind'); + var nodesConf = [ + { + datadir: path.resolve(__dirname, './data/node1'), + conf: path.resolve(__dirname, './data/node1/bitcoin.conf'), + rpcuser: 'bitcoin', + rpcpassword: 'local321', + rpcport: 30521, + zmqpubrawtx: 'tcp://127.0.0.1:30611', + zmqpubhashblock: 'tcp://127.0.0.1:30611' + }, + { + datadir: path.resolve(__dirname, './data/node2'), + conf: path.resolve(__dirname, './data/node2/bitcoin.conf'), + rpcuser: 'bitcoin', + rpcpassword: 'local321', + rpcport: 30522, + zmqpubrawtx: 'tcp://127.0.0.1:30622', + zmqpubhashblock: 'tcp://127.0.0.1:30622' + }, + { + datadir: path.resolve(__dirname, './data/node3'), + conf: path.resolve(__dirname, './data/node3/bitcoin.conf'), + rpcuser: 'bitcoin', + rpcpassword: 'local321', + rpcport: 30523, + zmqpubrawtx: 'tcp://127.0.0.1:30633', + zmqpubhashblock: 'tcp://127.0.0.1:30633' + } + ]; + + before(function(done) { + log.info('Starting 3 bitcoind daemons'); + this.timeout(20000); + async.each(nodesConf, function(nodeConf, next) { + var opts = [ + '--regtest', + '--datadir=' + nodeConf.datadir, + '--conf=' + nodeConf.conf + ]; + + rimraf(path.resolve(nodeConf.datadir, './regtest'), function(err) { + if (err) { + return done(err); + } + + var process = spawn(execPath, opts, {stdio: 'inherit'}); + + var client = new BitcoinRPC({ + protocol: 'http', + host: '127.0.0.1', + port: nodeConf.rpcport, + user: nodeConf.rpcuser, + pass: nodeConf.rpcpassword + }); + + daemons.push(process); + + async.retry({times: 10, interval: 5000}, function(ready) { + client.getInfo(ready); + }, next); + + }); + + }, done); + }); + + after(function(done) { + this.timeout(10000); + setTimeout(function() { + async.each(daemons, function(process, next) { + process.once('exit', next); + process.kill('SIGINT'); + }, done); + }, 1000); + }); + + it('step 1: will connect to three bitcoind daemons', function(done) { + this.timeout(20000); + var configuration = { + network: 'regtest', + services: [ + { + name: 'bitcoind', + module: BitcoinService, + config: { + connect: [ + { + rpchost: '127.0.0.1', + rpcport: 30521, + rpcuser: 'bitcoin', + rpcpassword: 'local321', + zmqpubrawtx: 'tcp://127.0.0.1:30611' + }, + { + rpchost: '127.0.0.1', + rpcport: 30522, + rpcuser: 'bitcoin', + rpcpassword: 'local321', + zmqpubrawtx: 'tcp://127.0.0.1:30622' + }, + { + rpchost: '127.0.0.1', + rpcport: 30523, + rpcuser: 'bitcoin', + rpcpassword: 'local321', + zmqpubrawtx: 'tcp://127.0.0.1:30633' + } + ] + } + } + ] + }; + + var regtest = bitcore.Networks.get('regtest'); + should.exist(regtest); + + node = new BitcoreNode(configuration); + + node.on('error', function(err) { + log.error(err); + }); + + node.on('ready', function() { + done(); + }); + + node.start(function(err) { + if (err) { + return done(err); + } + }); + + }); + + it('step 2: receive block events', function(done) { + this.timeout(10000); + node.services.bitcoind.once('tip', function() { + setTimeout(function() { + done(); + }, 1000); + }); + node.generateBlock(1, function(err, hashes) { + if (err) { + return done(err); + } + should.exist(hashes); + }); + }); + + it('step 3: get blocks', function(done) { + async.times(3, function(n, next) { + node.getBlock(1, function(err, block) { + if (err) { + return next(err); + } + should.exist(block); + next(); + }); + }, done); + }); + +}); diff --git a/regtest/data/node1/bitcoin.conf b/regtest/data/node1/bitcoin.conf new file mode 100644 index 00000000..695d6d3e --- /dev/null +++ b/regtest/data/node1/bitcoin.conf @@ -0,0 +1,14 @@ +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +addnode=127.0.0.1:30432 +addnode=127.0.0.1:30433 +port=30431 +rpcport=30521 +zmqpubrawtx=tcp://127.0.0.1:30611 +zmqpubhashblock=tcp://127.0.0.1:30611 +rpcallowip=127.0.0.1 +rpcuser=bitcoin +rpcpassword=local321 diff --git a/regtest/data/node2/bitcoin.conf b/regtest/data/node2/bitcoin.conf new file mode 100644 index 00000000..74d898af --- /dev/null +++ b/regtest/data/node2/bitcoin.conf @@ -0,0 +1,14 @@ +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +addnode=127.0.0.1:30431 +addnode=127.0.0.1:30433 +port=30432 +rpcport=30522 +zmqpubrawtx=tcp://127.0.0.1:30622 +zmqpubhashblock=tcp://127.0.0.1:30622 +rpcallowip=127.0.0.1 +rpcuser=bitcoin +rpcpassword=local321 diff --git a/regtest/data/node3/bitcoin.conf b/regtest/data/node3/bitcoin.conf new file mode 100644 index 00000000..0edb5bee --- /dev/null +++ b/regtest/data/node3/bitcoin.conf @@ -0,0 +1,14 @@ +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +addnode=127.0.0.1:30431 +addnode=127.0.0.1:30432 +port=30433 +rpcport=30523 +zmqpubrawtx=tcp://127.0.0.1:30633 +zmqpubhashblock=tcp://127.0.0.1:30633 +rpcallowip=127.0.0.1 +rpcuser=bitcoin +rpcpassword=local321 From 5bea36edc684ea52065d77e1c797af40e2fc07c6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Apr 2016 10:14:36 -0400 Subject: [PATCH 095/299] bitcoind: try querying all bitcoind nodes --- lib/services/bitcoind.js | 34 +++++++++++++++++++++++----------- regtest/cluster.js | 4 +--- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 220a8e53..77cabe92 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -235,6 +235,10 @@ Bitcoin.prototype._resetCaches = function() { this.summaryCache.reset(); }; +Bitcoin.prototype._tryAll = function(func, callback) { + async.retry({times: this.nodes.length, interval: 1000}, func, callback); +}; + Bitcoin.prototype._wrapRPCError = function(errObj) { var err = new Error(errObj.message); err.code = errObj.code; @@ -1036,14 +1040,16 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { var self = this; function queryBlock(blockhash) { - self.client.getBlock(blockhash, false, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - var blockObj = bitcore.Block.fromString(response.result); - self.blockCache.set(blockhash, blockObj); - callback(null, blockObj); - }); + self._tryAll(function(done) { + self.client.getBlock(blockhash, false, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var blockObj = bitcore.Block.fromString(response.result); + self.blockCache.set(blockhash, blockObj); + done(null, blockObj); + }); + }, callback); } var cachedBlock = self.blockCache.get(blockArg); @@ -1053,11 +1059,17 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { }); } else { if (_.isNumber(blockArg)) { - self.client.getBlockHash(blockArg, function(err, response) { + self._tryAll(function(done) { + self.client.getBlockHash(blockArg, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + done(null, response.result); + }); + }, function(err, blockhash) { if (err) { - return callback(self._wrapRPCError(err)); + return callback(err); } - var blockhash = response.result; queryBlock(blockhash); }); } else { diff --git a/regtest/cluster.js b/regtest/cluster.js index 30db5377..51a6e573 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -157,9 +157,7 @@ describe('Bitcoin Cluster', function() { it('step 2: receive block events', function(done) { this.timeout(10000); node.services.bitcoind.once('tip', function() { - setTimeout(function() { - done(); - }, 1000); + done(); }); node.generateBlock(1, function(err, hashes) { if (err) { From 019626ba157cdb1e315196851919b678caea21bf Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Apr 2016 10:15:12 -0400 Subject: [PATCH 096/299] bitcoind: prevent rapid tip updates for all networks --- lib/services/bitcoind.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 77cabe92..695482b9 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -322,19 +322,15 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { } } - // Prevent a rapid succession of updates with regtest - if (self.node.network.regtestEnabled) { - if (new Date() - self.zmqLastBlock > 1000) { - self.zmqLastBlock = new Date(); - updateChain(); - } else { - clearTimeout(self.zmqUpdateTipTimeout); - self.zmqUpdateTipTimeout = setTimeout(function() { - updateChain(); - }, 1000); - } - } else { + // Prevent a rapid succession of updates + if (new Date() - self.zmqLastBlock > 1000) { + self.zmqLastBlock = new Date(); updateChain(); + } else { + clearTimeout(self.zmqUpdateTipTimeout); + self.zmqUpdateTipTimeout = setTimeout(function() { + updateChain(); + }, 1000); } // Notify block subscribers From d0937fea55c0ac6849c74d6cb3653098237e3d4f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Apr 2016 10:15:15 -0400 Subject: [PATCH 097/299] bitcoind: try to get transaction from all bitcoind nodes --- lib/services/bitcoind.js | 140 +++++++++++++++++++++------------------ regtest/bitcoind.js | 28 +++----- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 695482b9..216c72ae 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -996,14 +996,16 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { var self = this; function queryBlock(blockhash) { - self.client.getBlock(blockhash, false, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - var buffer = new Buffer(response.result, 'hex'); - self.rawBlockCache.set(blockhash, buffer); - callback(null, buffer); - }); + self._tryAll(function(done) { + self.client.getBlock(blockhash, false, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var buffer = new Buffer(response.result, 'hex'); + self.rawBlockCache.set(blockhash, buffer); + done(null, buffer); + }); + }, callback); } var cachedBlock = self.rawBlockCache.get(blockArg); @@ -1013,11 +1015,17 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { }); } else { if (_.isNumber(blockArg)) { - self.client.getBlockHash(blockArg, function(err, response) { + self._tryAll(function(done) { + self.client.getBlockHash(blockArg, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + done(null, response.result); + }); + }, function(err, blockhash) { if (err) { - return callback(self._wrapRPCError(err)); + return callback(err); } - var blockhash = response.result; queryBlock(blockhash); }); } else { @@ -1107,25 +1115,29 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { var self = this; function queryHeader(blockhash) { - self.client.getBlockHeader(blockhash, function(err, response) { - if (err && response.error.code === -5) { - return callback(null, null); - } else if (err) { - return callback(self._wrapRPCError(err)); - } - // TODO format response prevHash instead of previousblockhash, etc. - callback(null, response.result); - }); + self._tryAll(function(done) { + self.client.getBlockHeader(blockhash, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + // TODO format response prevHash instead of previousblockhash, etc. + done(null, response.result); + }); + }, callback); } if (_.isNumber(block)) { - self.client.getBlockHash(block, function(err, response) { - if (err && response.error.code === -8) { - return callback(null, null); - } else if (err) { - return callback(self._wrapRPCError(err)); + self._tryAll(function(done) { + self.client.getBlockHash(block, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + done(null, response.result); + }); + }, function(err, blockhash) { + if (err) { + return callback(err); } - var blockhash = response.result; queryHeader(blockhash); }); } else { @@ -1192,16 +1204,16 @@ Bitcoin.prototype.getRawTransaction = function(txid, callback) { callback(null, tx); }); } else { - self.client.getRawTransaction(txid, function(err, response) { - if (err && response.error.code === -5) { - return callback(null, null); - } else if (err) { - return callback(self._wrapRPCError(err)); - } - var buffer = new Buffer(response.result, 'hex'); - self.rawTransactionCache.set(txid, buffer); - callback(null, buffer); - }); + self._tryAll(function(done) { + self.client.getRawTransaction(txid, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var buffer = new Buffer(response.result, 'hex'); + self.rawTransactionCache.set(txid, buffer); + done(null, buffer); + }); + }, callback); } }; @@ -1219,17 +1231,17 @@ Bitcoin.prototype.getTransaction = function(txid, callback) { callback(null, tx); }); } else { - self.client.getRawTransaction(txid, function(err, response) { - if (err && response.error.code === -5) { - return callback(null, null); - } else if (err) { - return callback(self._wrapRPCError(err)); - } - var tx = Transaction(); - tx.fromString(response.result); - self.transactionCache.set(txid, tx); - callback(null, tx); - }); + self._tryAll(function(done) { + self.client.getRawTransaction(txid, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var tx = Transaction(); + tx.fromString(response.result); + self.transactionCache.set(txid, tx); + done(null, tx); + }); + }, callback); } }; @@ -1251,23 +1263,23 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { callback(null, tx); }); } else { - self.client.getRawTransaction(txid, 1, function(err, response) { - if (err && response.error.code === -5) { - return callback(null, null); - } else if (err) { - return callback(self._wrapRPCError(err)); - } - var tx = Transaction(); - tx.fromString(response.result.hex); - tx.__blockHash = response.result.blockhash; - tx.__height = response.result.height ? response.result.height : -1; - tx.__timestamp = response.result.time; - var confirmations = self._getConfirmationsDetail(tx); - if (confirmations >= self.transactionInfoCacheConfirmations) { - self.transactionInfoCache.set(txid, tx); - } - callback(null, tx); - }); + self._tryAll(function(done) { + self.client.getRawTransaction(txid, 1, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var tx = Transaction(); + tx.fromString(response.result.hex); + tx.__blockHash = response.result.blockhash; + tx.__height = response.result.height ? response.result.height : -1; + tx.__timestamp = response.result.time; + var confirmations = self._getConfirmationsDetail(tx); + if (confirmations >= self.transactionInfoCacheConfirmations) { + self.transactionInfoCache.set(txid, tx); + } + done(null, tx); + }); + }, callback); } }; diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 8aa4d71e..485b33dc 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -222,13 +222,10 @@ describe('Bitcoind Functionality', function() { }); }); - it('will return null if the transaction does not exist', function(done) { + it('will return error if the transaction does not exist', function(done) { var txid = '6226c407d0e9705bdd7158e60983e37d0f5d23529086d6672b07d9238d5aa618'; bitcoind.getTransaction(txid, function(err, response) { - if (err) { - throw err; - } - should.not.exist(response); + should.exist(err); done(); }); }); @@ -250,13 +247,10 @@ describe('Bitcoind Functionality', function() { }); }); - it('will return null if the transaction does not exist', function(done) { + it('will return error if the transaction does not exist', function(done) { var txid = '6226c407d0e9705bdd7158e60983e37d0f5d23529086d6672b07d9238d5aa618'; bitcoind.getRawTransaction(txid, function(err, response) { - if (err) { - throw err; - } - should.not.exist(response); + should.exist(err); done(); }); }); @@ -293,12 +287,9 @@ describe('Bitcoind Functionality', function() { done(); }); }); - it('will get null for block not found', function(done) { + it('will get error for block not found', function(done) { bitcoind.getBlockHeader('notahash', function(err, header) { - if(err) { - return done(err); - } - should.equal(header, null); + should.exist(err); done(); }); }); @@ -321,12 +312,9 @@ describe('Bitcoind Functionality', function() { }); }); }); - it('will get null with number greater than tip', function(done) { + it('will get error with number greater than tip', function(done) { bitcoind.getBlockHeader(100000, function(err, header) { - if (err) { - return done(err); - } - should.equal(header, null); + should.exist(err); done(); }); }); From 52f05f3027a9ea8fd37f8e05d6dc16142b167478 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Apr 2016 10:15:20 -0400 Subject: [PATCH 098/299] bitcoind: emit block events --- lib/services/bitcoind.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 216c72ae..c9dbf4e8 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -70,6 +70,7 @@ Bitcoin.prototype._initCaches = function() { this.rawBlockCache = LRU(72); this.blockHeaderCache = LRU(288); this.zmqKnownTransactions = LRU(50); + this.zmqKnownBlocks = LRU(50); this.zmqLastBlock = 0; this.zmqUpdateTipTimeout = false; }; @@ -322,7 +323,7 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { } } - // Prevent a rapid succession of updates + // Prevent a rapid succession of tip updates if (new Date() - self.zmqLastBlock > 1000) { self.zmqLastBlock = new Date(); updateChain(); @@ -334,8 +335,15 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { } // Notify block subscribers - for (var i = 0; i < this.subscriptions.block.length; i++) { - this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); + var id = message.toString('binary'); + if (!self.zmqKnownBlocks[id]) { + self.zmqKnownBlocks[id] = true; + self.emit('block', message); + + for (var i = 0; i < this.subscriptions.block.length; i++) { + this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); + } + } }; From b757bd3148b42468b1ea2aac25438b892b14cdc9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Apr 2016 12:59:36 -0400 Subject: [PATCH 099/299] docs: update docs for bitcoind with address indexes --- README.md | 19 +-- docs/patch.md | 6 - docs/release.md | 17 +-- docs/services.md | 155 +------------------- docs/services/address.md | 136 ----------------- docs/services/bitcoind.md | 297 +++++++++++++++++++++++++++++++------- docs/services/db.md | 113 --------------- docs/testing.md | 21 +-- 8 files changed, 257 insertions(+), 507 deletions(-) delete mode 100644 docs/patch.md delete mode 100644 docs/services/address.md delete mode 100644 docs/services/db.md diff --git a/README.md b/README.md index 2da202a5..4b87327e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Bitcore Node ============ -A Bitcoin full node for building applications and services with Node.js. A node is extensible and can be configured to run additional services. At the minimum a node has native bindings to Bitcoin Core with the [Bitcoin Service](docs/services/bitcoind.md). Additional services can be enabled to make a node more useful such as exposing new APIs, adding new indexes for addresses with the [Address Service](docs/services/address.md), running a block explorer, wallet service, and other customizations. +A Bitcoin full node for building applications and services with Node.js. A node is extensible and can be configured to run additional services. At the minimum a node has an interface to [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12-bitcore) for more advanced address queries. Additional services can be enabled to make a node more useful such as exposing new APIs, running a block explorer and wallet service. ## Install @@ -10,14 +10,13 @@ npm install -g bitcore-node bitcore-node start ``` -Note: For your convenience, we distribute binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the [Build & Install](docs/build.md) documentation to build the project from source. +Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12-bitcore). ## Prerequisites - Node.js v0.12 or v4.2 -- ~100GB of disk storage +- ~150GB of disk storage - ~4GB of RAM -- Mac OS X >= 10.9, Ubuntu >= 12.04 (libc >= 2.15 and libstdc++ >= 6.0.16) ## Configuration @@ -32,12 +31,6 @@ bitcore-node install https://github.com/yourname/helloworld This will create a directory with configuration files for your node and install the necessary dependencies. For more information about (and developing) services, please see the [Service Documentation](docs/services.md). -To start bitcore-node as a daemon: - -```bash -bitcore-node start --daemon -``` - ## Add-on Services There are several add-on services available to extend the functionality of Bitcore: @@ -49,16 +42,12 @@ There are several add-on services available to extend the functionality of Bitco ## Documentation - [Services](docs/services.md) - - [Bitcoind](docs/services/bitcoind.md) - Native bindings to Bitcoin Core - - [Database](docs/services/db.md) - The foundation API methods for getting information about blocks and transactions. - - [Address](docs/services/address.md) - Adds additional API methods for querying and subscribing to events with bitcoin addresses. + - [Bitcoind](docs/services/bitcoind.md) - Interface to Bitcoin Core - [Web](docs/services/web.md) - Creates an express application over which services can expose their web/API content -- [Build & Install](docs/build.md) - How to build and install from source - [Testing & Development](docs/testing.md) - Developer guide for testing - [Node](docs/node.md) - Details on the node constructor - [Bus](docs/bus.md) - Overview of the event bus constructor - [Errors](docs/errors.md) - Reference for error handling and types -- [Patch](docs/patch.md) - Information about the patch applied to Bitcoin Core - [Release Process](docs/release.md) - Information about verifying a release and the release process. ## Contributing diff --git a/docs/patch.md b/docs/patch.md deleted file mode 100644 index 86c366c6..00000000 --- a/docs/patch.md +++ /dev/null @@ -1,6 +0,0 @@ -# Static Library Patch -To provide native bindings to JavaScript _(or any other language for that matter)_, Bitcoin code, itself, must be linkable. Currently, Bitcoin Core provides a JSON RPC interface to bitcoind as well as a shared library for script validation _(and hopefully more)_ called libbitcoinconsensus. There is a node module, [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus), that exposes these methods. While these interfaces are useful for several use cases, there are additional use cases that are not fulfilled, and being able to implement customized interfaces is necessary. To be able to do this a few simple changes need to be made to Bitcoin Core to compile as a static library. - -The patch is located at `etc/bitcoin.patch` and adds a configure option `--enable-daemonlib` to compile all object files with `-fPIC` (Position Independent Code - needed to create a shared object), exposes leveldb variables and objects, exposes the threadpool to the bindings, and conditionally includes the main function. - -Every effort will be made to ensure that this patch stays up-to-date with the latest release of Bitcoin. At the very least, this project began supporting Bitcoin Core v0.11. diff --git a/docs/release.md b/docs/release.md index 29deb20f..e2ce5313 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,11 +1,11 @@ # Release Process -Binaries for the C++ binding file (which includes libbitcoind statically linked in) are distributed for convenience. The binary binding file `bitcoind.node` is signed and published to S3 for later download and installation. Source files can also be built if binaries are not desired. +Binaries for bitcoind are distributed for convenience and built deterministically with Gitian. ## How to Verify Signatures ``` -cd build/Release -gpg --verify bitcoind.node.sig bitcoind.node +cd bin +gpg --verify bitcoin-0.12.0-linux64.tar.gz.sig bitcoin-0.12.0-linux64.tar.gz ``` To verify signatures, use the following PGP keys: @@ -14,7 +14,7 @@ To verify signatures, use the following PGP keys: - @pnagurny: [https://pgp.mit.edu/pks/lookup?op=get&search=0x0909B33F0AA53013](https://pgp.mit.edu/pks/lookup?op=get&search=0x0909B33F0AA53013) ## How to Release -Ensure you've followed the instructions in the README.md for building the project from source. When building for any platform, be sure to keep in mind the minimum supported C and C++ system libraries and build from source using this library. Example, Ubuntu 12.04 has the earliest system library for Linux that we support, so it would be easiest to build the Linux artifact using this version. You will be using node-gyp to build the C++ bindings. A script will then upload the bindings to S3 for later use. You will also need credentials for BitPay's bitcore-node S3 bucket and be listed as an author for the bitcore-node's npm module. +Ensure you've followed the instructions in the README.md for building the project from source. When building for any platform, be sure to keep in mind the minimum supported C and C++ system libraries and build from source using this library. Example, Ubuntu 12.04 has the earliest system library for Linux that we support, so it would be easiest to build the Linux artifact using this version. A script will then upload the binaries to S3 for later use. You will also need credentials for BitPay's bitcore-node S3 bucket and be listed as an author for the bitcore-node's npm module. - Create a file `.bitcore-node-upload.json` in your home directory - The format of this file should be: @@ -28,7 +28,7 @@ Ensure you've followed the instructions in the README.md for building the projec When publishing to npm, the .gitignore file is used to exclude files from the npm publishing process. Be sure that the bitcore-node directory has only the directories and files that you would like to publish to npm. You might need to run the commands below on each platform that you intend to publish (e.g. Mac and Linux). -To make a release, bump the `version` and `lastBuild` of the `package.json`: +To make a release, bump the `version` of the `package.json`: ```bash git checkout master @@ -40,13 +40,6 @@ npm run upload npm publish ``` -And then update the `version` of the `package.json` for development (e.g. "0.3.2-dev"): - -```bash -git commit -a -m "Bump development version to " -git push upstream master -``` - Create a release tag and push it to the BitPay Github repo: ```bash diff --git a/docs/services.md b/docs/services.md index 34f948f2..e4511388 100644 --- a/docs/services.md +++ b/docs/services.md @@ -10,7 +10,7 @@ The `bitcore-node.json` file describes which services will load for a node: ```json { "services": [ - "bitcoind", "db", "address", "insight-api" + "bitcoind", "web" ] } ``` @@ -37,9 +37,7 @@ If, instead, you would like to run a custom node, you can include services by in var bitcore = require('bitcore-node'); //Services -var Address = bitcore.services.Address; var Bitcoin = bitcore.services.Bitcoin; -var DB = bitcore.services.DB; var Web = bitcore.services.Web; var myNode = new bitcore.Node({ @@ -48,21 +46,11 @@ var myNode = new bitcore.Node({ name: 'livenet' }, "services": [ - { - name: "address", - module: Address, - config: {} - }, { name: 'bitcoind', module: Bitcoin, config: {} }, - { - name: 'db', - module: DB, - config: {} - }, { name: 'web', module: Web, @@ -77,8 +65,8 @@ var myNode = new bitcore.Node({ Now that you've loaded your services you can access them via `myNode.services..`. For example if you wanted to check the balance of an address, you could access the address service like so. ```js -myNode.services.address.getBalance('1HB5XMLmzFVj8ALj6mfBsbifRoD4miY36v', false, function(err, total) { - console.log(total); //Satoshi amount of this address +myNode.services.bitcoind.getBalance('1HB5XMLmzFVj8ALj6mfBsbifRoD4miY36v', false, function(err, total) { + console.log(total.balance); //Satoshi amount of this address }); ``` @@ -96,140 +84,3 @@ The `package.json` for the service module can either export the `Node.Service` d Please take a look at some of the existing services for implementation specifics. -### Adding an index -One quite useful feature exposed to services is the ability to index arbitrary data in the blockchain. To do so we make use of leveldb, a simple key-value store. As a service we can expose a 'blockHandler' function which is called each time a new block is added or removed from the blockchain. This gives us access to every new transaction received, allowing us to index them. Let's take a look at an example where we will index the time that a transaction was confirmed. - -```js -//Index prefix, so that we can determine the difference between our index -//and the indexes provided by other services -MyService.datePrefix = new Buffer('10', 'hex'); - -MyService.minPosition = new Buffer('00000', 'hex'); -MyService.maxPosition = new Buffer('99999', 'hex'); - -//This function is automatically called when a block is added or receieved -MyService.prototype.prototype.blockHandler = function(block, addOutput, callback) { - - //Determine if the block is added or removed, and therefore whether we are adding - //or deleting indexes - var databaseAction = 'put'; - if (!addOutput) { - databaseAction = 'del'; - } - - //An array of all leveldb operations we will be committing - var operations = []; - - //Timestamp of the current block - var blocktime = new Buffer(4); - blocktime.writeUInt32BE(block.header.time); - - for (var i = 0; i < block.transactions.length; i++) { - var transaction = block.transactions[i]; - var txid = new Buffer(transaction.id, 'hex'); - var position = new Buffer(('0000' + i).slice(-5), 'hex'); - - //To be able to query this txid by the block date we create an index, leading with the prefix we - //defined earlier, the the current blocktime, and finally a differentiator, in this case the index - //of this transaction in the block's transaction list - var indexOperation = { - type: databaseAction, - key: Buffer.concat([this.datePrefix, blockTime, position]), - value: txid - }; - - //Now we push this index into our list of operations that should be performed - operations.push(indexOperation); - } - - //Send the list of db operations back so they can be performed - setImmediate(function() { - callback(null, operations); - }); -}; -``` - -### Retrieving data using an index -With our block handler code every transaction in the blockchain will now be indexed. However, if we want to query this data we need to add a method to our service to expose it. - -```js - -MyService.prototype.getTransactionIdsByDate = function(startDateBuffer, endDateBuffer, callback) { - - var error; - var transactions = []; - - //Read data from leveldb which is between our startDate and endDate - var stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ - MyService.datePrefix, - startDateBuffer, - MyService.minPosition - ]), - lte: Buffer.concat([ - MyService.datePrefix, - endDateBuffer, - MyService.maxPosition - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - - stream.on('data', function(data) { - transactions.push(data.value.toString('hex')); - }); - - stream.on('error', function(streamError) { - if (streamError) { - error = streamError; - } - }); - - stream.on('close', function() { - if (error) { - return callback(error); - } - callback(null, transactions); - }); -}; -``` - -If you're new to leveldb and would like to better understand how createReadStream works you can find [more information here](https://github.com/Level/levelup#dbcreatereadstreamoptions). - -### Understanding indexes -You may notice there are several pieces to the index itself. Let's take a look at each piece to make them easier to understand. - -#### Prefixes -Since leveldb is just a simple key-value store we need something to differentiate which keys are part of which index. If we had two services trying to index on the same key, say a txid, they would overwrite each other and their queries would return results from the other index. By introducing a unique prefix per index type that we can prepend our indexes with prevents these collisions. - -```js -//A simple example of indexing the number of inputs and ouputs given a transaction id - -/** Wrong way **/ -var index1key = new Buffer(transaction.id, 'hex'); -var index1value = transaction.inputs.length; - -//Since this key has the same value it would just overwrite index1 when we write to the db -var index2key = new Buffer(transaction.id, 'hex'); -var index2value = transaction.outputs.length; - - -/** Right way **/ -var index1prefix = new Buffer('11', 'hex'); -var index2prefix = new Buffer('12', 'hex'); - -var index1key = Buffer.concat([index1prefix, new Buffer(transaction.id, 'hex')]); -var index1value = transaction.inputs.length; - -//Now that the keys are different, this won't overwrite the index -var index2key = Buffer.concat([index2prefix, new Buffer(transaction.id, 'hex')]); -var index2value = transaction.outputs.length; -``` - -Remember that all indexes are global, so check to make sure no other services you are using make use of the same prefix you plan to use in your service. We recommend documenting which prefixes you use and that you check for collisions with popular services if you plan to release your service for others to use. - -#### Index Key -The index key is the value you want to query by. This value should be deterministic so that it can be removed in the case of a [re-org](https://en.bitcoin.it/wiki/Chain_Reorganization) resulting in a block removal. The value should be unique, as no two indexes can be the same value. If you need two indexes with the same key value, consider adding a deterministic differentiator, such as a position in an array, or instead storing multiple values within the same index data. - -#### Index Data -This is the data which is returned when you search by the index's key. This can be whatever you would like to retrieve. Try to be efficient by not storing data that is already available elsewhere, such as storing a transaction ID instead of an entire transaction. diff --git a/docs/services/address.md b/docs/services/address.md deleted file mode 100644 index 7f46d3ad..00000000 --- a/docs/services/address.md +++ /dev/null @@ -1,136 +0,0 @@ -# Address Service -The address service builds on the [Bitcoin Service](bitcoind.md) and the [Database Service](db.md) to add additional functionality for querying and subscribing to information based on bitcoin addresses. This will typically represent the core functionality for wallet applications. - -## API Documentation -These methods are exposed over the JSON-RPC interface and can be called directly from a node via: - -```js -node.services.address. -``` - -**Get Unspent Outputs** - -One of the most common uses will be to retrieve unspent outputs necessary to create a transaction, here is how to get the unspent outputs for an address: - -```js -var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; -var includeMempool = true; -node.services.address.getUnspentOutputs(address, includeMempool, function(err, unspentOutputs) { - // see below -}); -``` - -The `unspentOutputs` will have the format: - -```js -[ - { - address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW', - txid: '9d956c5d324a1c2b12133f3242deff264a9b9f61be701311373998681b8c1769', - outputIndex: 1, - height: 150, - satoshis: 1000000000, - script: '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', - confirmations: 3 - } -] -``` - -**View Balances** - -```js -var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; -var includeMempool = true; -node.services.address.getBalance(address, includeMempool, function(err, balance) { - // balance will be in satoshis -}); -``` - -**View Address History** - -This method will give history of an address limited by a range of block heights by using the "start" and "end" arguments. The "start" value is the more recent, and greater, block height. The "end" value is the older, and lesser, block height. This feature is most useful for synchronization as previous history can be omitted. Furthermore for large ranges of block heights, results can be paginated by using the "from" and "to" arguments. - -```js -var addresses = ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']; -var options = { - start: 345000, - end: 344000, - queryMempool: true -}; -node.services.address.getAddressHistory(addresses, options, function(err, history) { - // see below -}); -``` - -The history format will be: - -```js -{ - totalCount: 1, // The total number of items within "start" and "end" - items: [ - { - addresses: { - 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW': { - inputIndexes: [], - outputIndexes: [0] - } - }, - satoshis: 1000000000, - height: 150, // the block height of the transaction - confirmations: 3, - timestamp: 1442948127, // in seconds - fees: 191, - tx: // the populated transaction - } - ] -} -``` - -**View Address Summary** - -```js -var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; -var options = { - noTxList: false -}; - -node.services.address.getAddressSummary(address, options, function(err, summary) { - // see below -}); -``` - -The `summary` will have the format (values are in satoshis): - -```js -{ - totalReceived: 1000000000, - totalSpent: 0, - balance: 1000000000, - unconfirmedBalance: 1000000000, - appearances: 1, // number of transactions - unconfirmedAppearances: 0, - txids: [ - '3f7d13efe12e82f873f4d41f7e63bb64708fc4c942eb8c6822fa5bd7606adb00' - ] -} -``` - -## Events -For details on instantiating a bus for a node, see the [Bus Documentation](../bus.md). -- Name: `address/transaction`, Arguments: `[address, address...]` -- Name: `address/balance`, Arguments: `[address, address...]` - -**Examples:** - -```js -bus.subscribe('address/transaction', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); -bus.subscribe('address/balance', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); - -bus.on('address/transaction', function(transaction) { - -}); - -bus.on('address/balance', function(balance) { - -}); -``` diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index 922965c4..d535d52b 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -1,20 +1,120 @@ # Bitcoin Service -The Bitcoin Service adds a native [Node.js](https://nodejs.org) interface to [Bitcoin Core](https://github.com/bitcoin/bitcoin) for querying information about the Bitcoin blockchain. Bindings are linked to Bitcoin Core compiled as a static library. + +The Bitcoin Service is a Node.js interface to [Bitcoin Core](https://github.com/bitcoin/bitcoin) for querying information about the bitcoin block chain. It will manage starting and stopping `bitcoind` or connect to several running `bitcoind` processes. It uses a branch of a [branch of Bitcoin Core](https://github.com/bitpay/bitcoin/tree/0.12-bitcore) with additional indexes for querying information about addresses and blocks. Results are cached for performance and there are several additional API methods added for common queries. + +## Configuration + +The default configuration will include a "spawn" configuration in "bitcoind". This defines the location of the block chain database and the location of the `bitcoind` daemon executable. The below configuration points to a local clone of `bitcoin`, and will start `bitcoind` automatically with your Node.js application. + +```json + "servicesConfig": { + "bitcoind": { + "spawn": { + "datadir": "/home/bitcore/.bitcoin", + "exec": "/home/bitcore/bitcoin/src/bitcoind" + } + } + } +``` + +It's also possible to connect to separately managed `bitcoind` processes with round-robin quering, for example: + +```json + "servicesConfig": { + "bitcoind": { + "connect": [ + { + "rpchost": "127.0.0.1", + "rpcport": 30521, + "rpcuser": "bitcoin", + "rpcpassword": "local321", + "zmqpubrawtx": "tcp://127.0.0.1:30611" + }, + { + "rpchost": "127.0.0.1", + "rpcport": 30522, + "rpcuser": "bitcoin", + "rpcpassword": "local321", + "zmqpubrawtx": "tcp://127.0.0.1:30622" + }, + { + "rpchost": "127.0.0.1", + "rpcport": 30523, + "rpcuser": "bitcoin", + "rpcpassword": "local321", + "zmqpubrawtx": "tcp://127.0.0.1:30633" + } + ] + } + } +``` + +**Note**: For detailed example configuration see [`regtest/cluster.js`](regtest/cluster.js) + ## API Documentation -These methods are currently only available via directly interfacing with a node: +Methods are available by directly interfacing with the service: ```js node.services.bitcoind. ``` +### Chain + +**Getting Latest Blocks** + +```js +// gives the block hashes within a range of timestamps +var high = 1460393372; // Mon Apr 11 2016 12:49:25 GMT-0400 (EDT) +var low = 1460306965; // Mon Apr 10 2016 12:49:25 GMT-0400 (EDT) +node.services.bitcoind.getBlockHashesByTimestamp(high, low, function(err, blockHashes) { + //... +}); + +// get the current tip of the chain +node.services.bitcoind.getBestBlockHash(function(err, blockHash) { + //... +}) +``` + +**Getting Synchronization and Node Status** + +```js +// gives a boolean if the daemon is fully synced (not the initial block download) +node.services.bitcoind.isSynced(function(err, synced) { + //... +}) + +// gives the current estimate of blockchain download as a percentage +node.services.bitcoind.syncPercentage(function(err, percent) { + //... +}); + +// gives information about the chain including total number of blocks +node.services.bitcoind.getInfo(function(err, info) { + //... +}); +``` + +**Generate Blocks** + +```js +// will generate a block for the "regtest" network (development purposes) +var numberOfBlocks = 10; +node.services.bitcoind.generateBlock(numberOfBlocks, function(err, blockHashes) { + //... +}); +``` + +### Blocks and Transactions + **Getting Block Information** -It's possible to query blocks by both block hash and by height. Blocks are given as Node.js buffers and can be parsed via Bitcore: +It's possible to query blocks by both block hash and by height. Blocks are given as Node.js Buffers and can be parsed via Bitcore: ```js var blockHeight = 0; -node.services.bitcoind.getBlock(blockHeight, function(err, blockBuffer) { +node.services.bitcoind.getRawBlock(blockHeight, function(err, blockBuffer) { if (err) { throw err; } @@ -22,36 +122,40 @@ node.services.bitcoind.getBlock(blockHeight, function(err, blockBuffer) { console.log(block); }; -// check if the block is part of the main chain -var mainChain = node.services.bitcoind.isMainChain(block.hash); -console.log(mainChain); +// get a bitcore object of the block (as above) +node.services.bitcoind.getBlock(blockHash, function(err, block) { + //... +}; -// get only the block index (including chain work and previous hash) -var blockIndex = node.services.bitcoind.getBlockIndex(blockHeight); -console.log(blockIndex); +// get only the block header and index (including chain work, height, and previous hash) +node.services.bitcoind.getBlockHeader(blockHeight, function(err, blockHeader) { + //... +}); ``` **Retrieving and Sending Transactions** -Get a transaction asynchronously by reading it from disk, with an argument to optionally not include the mempool: +Get a transaction asynchronously by reading it from disk: ```js var txid = '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623'; -var queryMempool = true; -node.services.bitcoind.getTransaction(txid, queryMempool, function(err, transactionBuffer) { +node.services.bitcoind.getRawTransaction(txid, function(err, transactionBuffer) { if (err) { throw err; } var transaction = bitcore.Transaction().fromBuffer(transactionBuffer); }); +// get a bitcore object of the transaction (as above) +node.services.bitcoind.getTransaction(txid, function(err, transaction) { + //... +}); // also retrieve the block timestamp and height -node.services.bitcoind.getTransactionWithBlockInfo(txid, queryMempool, function(err, info) { - console.log(info.blockHash); - console.log(info.height); - console.log(info.timestamp); // in seconds - var transaction = bitcore.Transaction().fromBuffer(transactionBuffer); +node.services.bitcoind.getTransactionWithBlockInfo(txid, function(err, transaction) { + console.log(transaction.__blockHash); + console.log(transaction.__height); + console.log(transaction.__timestamp); // in seconds }); ``` @@ -59,71 +163,154 @@ Send a transaction to the network: ```js var numberOfBlocks = 3; -var feesPerKilobyte = node.services.bitcoind.estimateFee(numberOfBlocks); // in satoshis +node.services.bitcoind.estimateFee(numberOfBlocks, function(err, feesPerKilobyte) { + //... +}); -try { - node.services.bitcoind.sendTransaction(transaction.serialize()); -} catch(err) { - // handle error -} +node.services.bitcoind.sendTransaction(transaction.serialize(), function(err, hash) { + //... +}); +``` + +### Addresses + +**Get Unspent Outputs** + +One of the most common uses will be to retrieve unspent outputs necessary to create a transaction, here is how to get the unspent outputs for an address: + +```js +var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; +node.services.bitcoind.getAddressUnspentOutputs(address, options, function(err, unspentOutputs) { + // see below +}); +``` + +The `unspentOutputs` will have the format: + +```js +[ + { + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW', + txid: '9d956c5d324a1c2b12133f3242deff264a9b9f61be701311373998681b8c1769', + outputIndex: 1, + height: 150, + satoshis: 1000000000, + script: '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', + confirmations: 3 + } +] ``` -Get all of the transactions in the mempool: +**View Balances** ```js -var mempool = node.services.bitcoind.getMempoolTransactions(); -var transactions = []; -for (var i = 0; i < mempool.length; i++) { - transactions.push(bitcore.Transaction().fromBuffer(transactions[i])); +var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; +node.services.bitcoind.getAddressBalance(address, options, function(err, balance) { + // balance will be in satoshis with "received" and "balance" +}); +``` + +**View Address History** + +This method will give history of an address limited by a range of block heights by using the "start" and "end" arguments. The "start" value is the more recent, and greater, block height. The "end" value is the older, and lesser, block height. This feature is most useful for synchronization as previous history can be omitted. Furthermore for large ranges of block heights, results can be paginated by using the "from" and "to" arguments. + +```js +var addresses = ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']; +var options = { + start: 345000, + end: 344000, + queryMempool: true +}; +node.services.bitcoind.getAddressHistory(addresses, options, function(err, history) { + // see below +}); +``` + +The history format will be: + +```js +{ + totalCount: 1, // The total number of items within "start" and "end" + items: [ + { + addresses: { + 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW': { + inputIndexes: [], + outputIndexes: [0] + } + }, + satoshis: 1000000000, + height: 150, // the block height of the transaction + confirmations: 3, + timestamp: 1442948127, // in seconds + fees: 191, + tx: // the populated transaction + } + ] } ``` -Determine if an output is spent (excluding the mempool): +**View Address Summary** ```js -var spent = node.services.bitcoind.isSpent(txid, outputIndex); -console.log(spent); +var address = 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'; +var options = { + noTxList: false +}; + +node.services.bitcoind.getAddressSummary(address, options, function(err, summary) { + // see below +}); ``` -**Miscellaneous** -- `bitcoind.start(callback)` - Start the JavaScript Bitcoin node, the callback is called when the daemon is ready. -- `bitcoind.getInfo()` - Basic information about the chain including total number of blocks. -- `bitcoind.isSynced()` - Returns a boolean if the daemon is fully synced (not the initial block download) -- `bitcoind.syncPercentage()` - Returns the current estimate of blockchain download as a percentage. -- `bitcoind.stop(callback)` - Stop the JavaScript bitcoin node safely, the callback will be called when bitcoind is closed. This will also be done automatically on `process.exit`. It also takes the bitcoind node off the libuv event loop. If the daemon object is the only thing on the event loop. Node will simply close. +The `summary` will have the format (values are in satoshis): + +```js +{ + totalReceived: 1000000000, + totalSpent: 0, + balance: 1000000000, + unconfirmedBalance: 1000000000, + appearances: 1, // number of transactions + unconfirmedAppearances: 0, + txids: [ + '3f7d13efe12e82f873f4d41f7e63bb64708fc4c942eb8c6822fa5bd7606adb00' + ] +} +``` ## Events -The Bitcoin Service doesn't expose any events via the Bus, however there are a few events that can be directly registered: +The Bitcoin Service exposes two events via the Bus, and there are a few events that can be directly registered: ```js node.services.bitcoind.on('tip', function(blockHash) { - // a new block tip has been added + // a new block tip has been added, if there is a rapid update (with a second) this will not emit every tip update }); -node.services.bitcoind.on('tx', function(txInfo) { +node.services.bitcoind.on('tx', function(transactionBuffer) { // a new transaction has entered the mempool }); -node.services.bitcoind.on('txleave', function(txLeaveInfo) { +node.services.bitcoind.on('block', function(blockHash) { // a new transaction has left the mempool }); ``` -The `txInfo` object will have the format: +For details on instantiating a bus for a node, see the [Bus Documentation](../bus.md). +- Name: `bitcoind/transaction`, Arguments: `[address, address...]` +- Name: `bitcoind/balance`, Arguments: `[address, address...]` + +**Examples:** ```js -{ - buffer: , - mempool: true, // will currently always be true - hash: '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623' -} -``` +bus.subscribe('bitcoind/transaction', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); +bus.subscribe('bitcoind/balance', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); -The `txLeaveInfo` object will have the format: +bus.on('bitcoind/transaction', function(transaction) { + //... +}); -```js -{ - buffer: , - hash: '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623' -} +bus.on('bitcoind/balance', function(balance) { + //... +}); ``` diff --git a/docs/services/db.md b/docs/services/db.md deleted file mode 100644 index 73a6cbb2..00000000 --- a/docs/services/db.md +++ /dev/null @@ -1,113 +0,0 @@ -# Database Service -This service synchronizes a leveldb database with the [Bitcoin Service](bitcoind.md) block chain by connecting and disconnecting blocks to build new indexes that can be queried. Other services can extend the data that is indexed by implementing a `blockHandler` method, similar to the built-in [Address Service](address.md). - -## How to Reindex - -If you need to be able to recreate the database from historical transactions in blocks: -- Shutdown your node -- Remove the `bitcore-node.db` directory in the data directory (e.g. `~/.bitcore/bitcore-node.db`) -- Start your node again - -The database will then ask bitcoind for all the blocks again and recreate the database. This is sometimes required during upgrading as the format of the keys and values has changed. For "livenet" this can take half a day or more, for "testnet" this can take around an hour. - -## Adding Indexes -For a service to include additional block data, it can implement a `blockHandler` method that will be run to when there are new blocks added or removed. - -```js -CustomService.prototype.blockHandler = function(block, add, callback) { - var transactions = block.transactions; - var operations = []; - operations.push({ - type: add ? 'put' : 'del', - key: 'key', - value: 'value' - }); - callback(null, operations); -}; -``` - -Take a look at the Address Service implementation for more details about how to encode the key, value for the best efficiency and ways to format the keys for streaming reads. - -## API Documentation -These methods are exposed over the JSON-RPC interface and can be called directly from a node via: - -```js -node.services.db. -``` - -**Query Blocks by Date** - -One of the additional indexes created by the Database Service is querying for blocks by ranges of dates: - -```js -var newest = 1441914000; // Notice time is in seconds not milliseconds -var oldest = 1441911000; - -node.services.db.getBlockHashesByTimestamp(newest, oldest, function(err, hashes) { - // hashes will be an array of block hashes -}); -``` - -**Working with Blocks and Transactions as Bitcore Instances** - -```js - -var txid = 'c349b124b820fe6e32136c30e99f6c4f115fce4d750838edf0c46d3cb4d7281e'; -var includeMempool = true; -node.services.db.getTransaction(txid, includeMempool, function(err, transaction) { - console.log(transaction.toObject()); -}); - -var txid = 'c349b124b820fe6e32136c30e99f6c4f115fce4d750838edf0c46d3cb4d7281e'; -var includeMempool = true; -node.services.db.getTransactionWithBlockInfo(txid, includeMempool, function(err, transaction) { - console.log(transaction.toObject()); - console.log(transaction.__blockHash); - console.log(transaction.__height); - console.log(transaction.__timestamp); -}); - -var blockHash = '00000000d17332a156a807b25bc5a2e041d2c730628ceb77e75841056082a2c2'; -node.services.db.getBlock(blockHash, function(err, block) { - console.log(block.toObject()); -}); - -// contruct a transaction -var transaction = bitcore.Transaction(); - -node.services.db.sendTransaction(transaction, function(err) { - if (err) { - throw err; - } - // otherwise the transaction has been sent -}); -``` - -## Events -For details on instantiating a bus for a node, see the [Bus Documentation](../bus.md). -- Name: `db/transaction` -- Name: `db/block` - -**Examples:** - -```js -bus.subscribe('db/transaction'); -bus.subscribe('db/block'); - -bus.on('db/block', function(blockHash) { - // blockHash will be a hex string of the block hash -}); - -bus.on('db/transaction', function(txInfo) { - // see below -}); -``` - -The `txInfo` object will have the format: - -```js -{ - rejected: true, // If the transaction was rejected into the mempool - tx: // a Bitcore Transaction instance -} -``` diff --git a/docs/testing.md b/docs/testing.md index 30658bb6..a1a6844e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5,34 +5,19 @@ To run all of the JavaScript tests: npm run test ``` -To run tests against the bindings, as defined in `bindings.gyp` the regtest feature of Bitcoin Core is used, and to enable this feature we currently need to build with the wallet enabled _(not a part of the regular build)_. To do this, export an environment variable and recompile: - -```bash -export BITCORENODE_ENV=test -npm run build -``` - If you do not already have mocha installed: ```bash npm install mocha -g ``` -To run the integration tests: +To run the regression tests: ```bash -mocha -R spec integration/regtest.js +mocha -R spec regtest/bitcoind.js ``` -If any changes have been made to the bindings in the "src" directory, manually compile the Node.js bindings, as defined in `bindings.gyp`, you can run (-d for debug): - -```bash -$ node-gyp -d rebuild -``` - -Note: `node-gyp` can be installed with `npm install node-gyp -g` - -To be able to debug you'll need to have `gdb` and `node` compiled for debugging with gdb using `--gdb` (sometimes called node_g), and you can then run: +To be able to debug bitcoind you'll need to have `gdb` and `node` compiled for debugging with gdb using `--gdb` (sometimes called node_g), and you can then run: ```bash $ gdb --args node examples/node.js From 37f31fdb1945475911810f5987c17dd097c87050 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:02:37 -0400 Subject: [PATCH 100/299] bitcoind: added getspentinfo method --- lib/services/bitcoind.js | 30 ++++++++++++++++++++++-------- lib/transaction.js | 24 ++++++++++++++++++++++++ package.json | 2 +- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index c9dbf4e8..84d253c6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -60,12 +60,11 @@ Bitcoin.prototype._initCaches = function() { this.txidsCache = LRU(50000); this.balanceCache = LRU(50000); this.summaryCache = LRU(50000); + this.transactionInfoCache = LRU(100000); // caches valid indefinitely this.transactionCache = LRU(100000); this.rawTransactionCache = LRU(50000); - this.transactionInfoCache = LRU(100000); - this.transactionInfoCacheConfirmations = 6; this.blockCache = LRU(144); this.rawBlockCache = LRU(72); this.blockHeaderCache = LRU(288); @@ -100,6 +99,7 @@ Bitcoin.prototype.getAPIMethods = function() { ['getBlockHeader', this, this.getBlockHeader, 1], ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], ['getBestBlockHash', this, this.getBestBlockHash, 0], + ['getSpentInfo', this, this.getSpentInfo, 1], ['getInfo', this, this.getInfo, 0], ['syncPercentage', this, this.syncPercentage, 0], ['isSynced', this, this.isSynced, 0], @@ -230,6 +230,7 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { }; Bitcoin.prototype._resetCaches = function() { + this.transactionInfoCache.reset(); this.utxosCache.reset(); this.txidsCache.reset(); this.balanceCache.reset(); @@ -1281,10 +1282,12 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { tx.__blockHash = response.result.blockhash; tx.__height = response.result.height ? response.result.height : -1; tx.__timestamp = response.result.time; - var confirmations = self._getConfirmationsDetail(tx); - if (confirmations >= self.transactionInfoCacheConfirmations) { - self.transactionInfoCache.set(txid, tx); + + for (var i = 0; i < response.result.vout.length; i++) { + tx.outputs[i].__spentTxId = response.result.vout[i].spentTxId; + tx.outputs[i].__spentIndex = response.result.vout[i].spentIndex; } + self.transactionInfoCache.set(txid, tx); done(null, tx); }); }, callback); @@ -1305,9 +1308,20 @@ Bitcoin.prototype.getBestBlockHash = function(callback) { }); }; -Bitcoin.prototype.getInputForOutput = function(txid, index, options, callback) { - // TODO - setImmediate(callback); +/** + * Will give the txid and inputIndex that spent an output + * @param {Function} callback + */ +Bitcoin.prototype.getSpentInfo = function(options, callback) { + var self = this; + this.client.getSpentInfo(options, function(err, response) { + if (err && err.code === -5) { + return callback(null, {}); + } else if (err) { + return callback(self._wrapRPCError(err)); + } + callback(null, response.result); + }); }; /** diff --git a/lib/transaction.js b/lib/transaction.js index adce3086..e4cccd8b 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -9,6 +9,30 @@ var errors = index.errors; var MAX_TRANSACTION_LIMIT = 5; +Transaction.prototype.populateSpentInfo = function(db, options, callback) { + var self = this; + var txid = self.hash; + + async.eachLimit( + Object.keys(self.outputs), + db.maxTransactionlimit || MAX_TRANSACTION_LIMIT, + function(outputIndex, next) { + db.getSpentInfo({ + txid: txid, + index: parseInt(outputIndex) + }, function(err, info) { + if (err) { + return next(err); + } + self.outputs[outputIndex].__spentTxId = info.txid; + self.outputs[outputIndex].__spentIndex = info.index; + next(); + }); + }, + callback + ); +}; + Transaction.prototype.populateInputs = function(db, poolTransactions, callback) { var self = this; diff --git a/package.json b/package.json index 8afbee16..a7bc9fdd 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "braydonf/bitcoind-rpc#8d27a545f4e7de5a8faca5de6bdbb1a6c1e41f5c", + "bitcoind-rpc": "braydonf/bitcoind-rpc#4850733b9806bc5e8e1508fa90f3c45782e6ee80", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", From cdfe572344fe5228c4b7a9a919f2cd6b8ea86d2b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:30:17 -0400 Subject: [PATCH 101/299] bitcoind: include height in spentinfo --- lib/services/bitcoind.js | 1 + lib/transaction.js | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 84d253c6..f935e826 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1286,6 +1286,7 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { for (var i = 0; i < response.result.vout.length; i++) { tx.outputs[i].__spentTxId = response.result.vout[i].spentTxId; tx.outputs[i].__spentIndex = response.result.vout[i].spentIndex; + tx.outputs[i].__spentHeight = response.result.vout[i].spentHeight; } self.transactionInfoCache.set(txid, tx); done(null, tx); diff --git a/lib/transaction.js b/lib/transaction.js index e4cccd8b..ef1d7d2a 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -26,6 +26,7 @@ Transaction.prototype.populateSpentInfo = function(db, options, callback) { } self.outputs[outputIndex].__spentTxId = info.txid; self.outputs[outputIndex].__spentIndex = info.index; + self.outputs[outputIndex].__spentHeight = info.height; next(); }); }, From c36b0777d48c6e575e4393047b022b898d429b1b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:36:59 -0400 Subject: [PATCH 102/299] bitcoind: add checkstate for spentindex --- lib/services/bitcoind.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index f935e826..be30eb02 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -201,6 +201,13 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { 'necessary with reindex=1' ); + $.checkState( + spawnConfig.spentindex && spawnConfig.spentindex === 1, + '"spentindex" option is required in order to use spent info query features of bitcore-node. ' + + 'Please add "spentindex=1" to your configuration and reindex an existing database if ' + + 'necessary with reindex=1' + ); + $.checkState( spawnConfig.server && spawnConfig.server === 1, '"server" option is required to communicate to bitcoind from bitcore. ' + From 042576474ff03350c12ce93b5929878c4918843d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 16:42:00 -0400 Subject: [PATCH 103/299] build: bump bitcoin build tag to v0.12.0-bitcore-beta2 --- regtest/data/bitcoin.conf | 1 + regtest/data/node1/bitcoin.conf | 1 + regtest/data/node2/bitcoin.conf | 1 + regtest/data/node3/bitcoin.conf | 1 + scripts/install | 2 +- 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/regtest/data/bitcoin.conf b/regtest/data/bitcoin.conf index 84aafd8c..95cf0afb 100644 --- a/regtest/data/bitcoin.conf +++ b/regtest/data/bitcoin.conf @@ -3,6 +3,7 @@ whitelist=127.0.0.1 txindex=1 addressindex=1 timestampindex=1 +spentindex=1 zmqpubrawtx=tcp://127.0.0.1:30332 zmqpubhashblock=tcp://127.0.0.1:30332 rpcallowip=127.0.0.1 diff --git a/regtest/data/node1/bitcoin.conf b/regtest/data/node1/bitcoin.conf index 695d6d3e..19bbf70a 100644 --- a/regtest/data/node1/bitcoin.conf +++ b/regtest/data/node1/bitcoin.conf @@ -3,6 +3,7 @@ whitelist=127.0.0.1 txindex=1 addressindex=1 timestampindex=1 +spentindex=1 addnode=127.0.0.1:30432 addnode=127.0.0.1:30433 port=30431 diff --git a/regtest/data/node2/bitcoin.conf b/regtest/data/node2/bitcoin.conf index 74d898af..9e08fe9a 100644 --- a/regtest/data/node2/bitcoin.conf +++ b/regtest/data/node2/bitcoin.conf @@ -3,6 +3,7 @@ whitelist=127.0.0.1 txindex=1 addressindex=1 timestampindex=1 +spentindex=1 addnode=127.0.0.1:30431 addnode=127.0.0.1:30433 port=30432 diff --git a/regtest/data/node3/bitcoin.conf b/regtest/data/node3/bitcoin.conf index 0edb5bee..954b0892 100644 --- a/regtest/data/node3/bitcoin.conf +++ b/regtest/data/node3/bitcoin.conf @@ -3,6 +3,7 @@ whitelist=127.0.0.1 txindex=1 addressindex=1 timestampindex=1 +spentindex=1 addnode=127.0.0.1:30431 addnode=127.0.0.1:30432 port=30433 diff --git a/scripts/install b/scripts/install index 3ded019c..c0e18cb4 100755 --- a/scripts/install +++ b/scripts/install @@ -5,7 +5,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12.0-bitcore-beta1" +tag="v0.12.0-bitcore-beta2" cd "${root_dir}/bin" From 1d358a69947ab09603e1065e7c873f0d6c248df5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 17:22:37 -0400 Subject: [PATCH 104/299] test: update pagination test --- regtest/node.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/regtest/node.js b/regtest/node.js index a9e09d9e..ebb58cab 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -622,7 +622,7 @@ describe('Node Functionality', function() { }); }); - describe.skip('Pagination', function() { + describe('Pagination', function() { it('from 0 to 1', function(done) { var options = { from: 0, @@ -634,7 +634,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(157); + history[0].height.should.equal(159); done(); }); }); @@ -649,7 +649,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(156); + history[0].height.should.equal(158); done(); }); }); @@ -664,7 +664,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(155); + history[0].height.should.equal(157); done(); }); }); @@ -679,7 +679,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(154); + history[0].height.should.equal(156); done(); }); }); @@ -694,7 +694,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(153); + history[0].height.should.equal(155); history[0].satoshis.should.equal(-10000); history[0].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); history[0].addresses[address].inputIndexes.should.deep.equal([0]); @@ -712,7 +712,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(150); + history[0].height.should.equal(152); history[0].satoshis.should.equal(10 * 1e8); done(); }); From c2eda9b3c29d3dd69cbb9c4db118a63d26da1428 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 18:05:26 -0400 Subject: [PATCH 105/299] bitcoin: address history by height range --- lib/services/bitcoind.js | 40 ++++++++++++++++++++++++++++++++++++---- regtest/node.js | 30 +++++++++++++++--------------- scripts/install | 2 +- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index be30eb02..fdc2a612 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -707,6 +707,21 @@ Bitcoin.prototype._getTxidsFromMempool = function(deltas) { return mempoolTxids; }; +Bitcoin.prototype._getHeightRangeQuery = function(options, clone) { + if (options.start >= 0 && options.end >=0) { + if (options.end > options.start) { + throw new TypeError('"end" is expected to be less than or equal to "start"'); + } + if (clone) { + // reverse start and end as the order in bitcore is most recent to less recent + clone.start = options.end; + clone.end = options.start; + } + return true; + } + return false; +}; + /** * Will get the txids for an address or multiple addresses * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses @@ -716,24 +731,41 @@ Bitcoin.prototype._getTxidsFromMempool = function(deltas) { Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { var self = this; var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; + var rangeQuery = false; + try { + rangeQuery = self._getHeightRangeQuery(options); + } catch(err) { + return callback(err); + } + if (rangeQuery) { + queryMempool = false; + } var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); var mempoolTxids = []; var txids = self.txidsCache.get(cacheKey); function finish() { - if (txids) { + if (txids && !rangeQuery) { var allTxids = mempoolTxids.reverse().concat(txids); return setImmediate(function() { callback(null, allTxids); }); } else { - self.client.getAddressTxids({addresses: addresses}, function(err, response) { + var txidOpts = { + addresses: addresses + }; + if (rangeQuery) { + self._getHeightRangeQuery(options, txidOpts); + } + self.client.getAddressTxids(txidOpts, function(err, response) { if (err) { return callback(self._wrapRPCError(err)); } response.result.reverse(); - self.txidsCache.set(cacheKey, response.result); + if (!rangeQuery) { + self.txidsCache.set(cacheKey, response.result); + } var allTxids = mempoolTxids.reverse().concat(response.result); return callback(null, allTxids); }); @@ -895,7 +927,7 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var addressStrings = this._getAddressStrings(addresses); - self.getAddressTxids(addresses, {}, function(err, txids) { + self.getAddressTxids(addresses, options, function(err, txids) { if (err) { return callback(err); } diff --git a/regtest/node.js b/regtest/node.js index ebb58cab..9e9e281f 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -483,7 +483,7 @@ describe('Node Functionality', function() { }); }); - it.skip('five addresses (limited by height)', function(done) { + it('five addresses (limited by height)', function(done) { var addresses = [ address2, address3, @@ -492,8 +492,8 @@ describe('Node Functionality', function() { address6 ]; var options = { - start: 157, - end: 156 + start: 158, + end: 157 }; node.getAddressHistory(addresses, options, function(err, results) { if (err) { @@ -502,15 +502,15 @@ describe('Node Functionality', function() { results.totalCount.should.equal(2); var history = results.items; history.length.should.equal(2); - history[0].height.should.equal(157); - history[0].confirmations.should.equal(1); - history[1].height.should.equal(156); - should.exist(history[1].addresses[address4]); + history[0].height.should.equal(158); + history[0].confirmations.should.equal(2); + history[1].height.should.equal(157); + should.exist(history[1].addresses[address3]); done(); }); }); - it.skip('five addresses (limited by height 155 to 154)', function(done) { + it('five addresses (limited by height 155 to 154)', function(done) { var addresses = [ address2, address3, @@ -519,8 +519,8 @@ describe('Node Functionality', function() { address6 ]; var options = { - start: 155, - end: 154 + start: 157, + end: 156 }; node.getAddressHistory(addresses, options, function(err, results) { if (err) { @@ -529,13 +529,13 @@ describe('Node Functionality', function() { results.totalCount.should.equal(2); var history = results.items; history.length.should.equal(2); - history[0].height.should.equal(155); - history[1].height.should.equal(154); + history[0].height.should.equal(157); + history[1].height.should.equal(156); done(); }); }); - it.skip('five addresses (paginated by index)', function(done) { + it('five addresses (paginated by index)', function(done) { var addresses = [ address2, address3, @@ -554,9 +554,9 @@ describe('Node Functionality', function() { results.totalCount.should.equal(4); var history = results.items; history.length.should.equal(3); - history[0].height.should.equal(157); + history[0].height.should.equal(159); history[0].confirmations.should.equal(1); - history[1].height.should.equal(156); + history[1].height.should.equal(158); should.exist(history[1].addresses[address4]); done(); }); diff --git a/scripts/install b/scripts/install index c0e18cb4..f1f7bd9d 100755 --- a/scripts/install +++ b/scripts/install @@ -5,7 +5,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12.0-bitcore-beta2" +tag="v0.12.0-bitcore-beta3" cd "${root_dir}/bin" From 848dc297777bbccb1d9519259f83c1db9c7aa309 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 13 Apr 2016 09:17:28 -0400 Subject: [PATCH 106/299] docs: update get balance method --- docs/services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/services.md b/docs/services.md index e4511388..85271347 100644 --- a/docs/services.md +++ b/docs/services.md @@ -65,7 +65,7 @@ var myNode = new bitcore.Node({ Now that you've loaded your services you can access them via `myNode.services..`. For example if you wanted to check the balance of an address, you could access the address service like so. ```js -myNode.services.bitcoind.getBalance('1HB5XMLmzFVj8ALj6mfBsbifRoD4miY36v', false, function(err, total) { +myNode.services.bitcoind.getAddressBalance('1HB5XMLmzFVj8ALj6mfBsbifRoD4miY36v', false, function(err, total) { console.log(total.balance); //Satoshi amount of this address }); ``` From 890b38744d314c5d82d1779c6b2fb8595c9070f4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 13 Apr 2016 11:13:44 -0400 Subject: [PATCH 107/299] test: update unit tests, refactoring and cleanup --- .travis.yml | 8 +- lib/errors.js | 17 +- lib/scaffold/default-base-config.js | 1 + lib/services/bitcoind.js | 197 +- lib/transaction.js | 28 +- package.json | 1 + scripts/regtest | 8 + test/bin/get-tarball-name.js | 18 - test/data/bitcoin.conf | 14 +- test/data/default.bitcoin.conf | 11 + test/index.unit.js | 19 - .../default-base-config.integration.js | 21 +- test/scaffold/default-config.integration.js | 44 +- test/scaffold/start.integration.js | 16 +- test/scaffold/start.unit.js | 104 - test/services/address/encoding.unit.js | 103 - test/services/address/history.unit.js | 544 ---- test/services/address/index.unit.js | 2676 ----------------- test/services/bitcoind.unit.js | 2409 +++++++++++++-- test/services/db.unit.js | 1038 ------- test/transaction.unit.js | 90 +- 21 files changed, 2385 insertions(+), 4982 deletions(-) create mode 100755 scripts/regtest delete mode 100644 test/bin/get-tarball-name.js create mode 100644 test/data/default.bitcoin.conf delete mode 100644 test/index.unit.js delete mode 100644 test/services/address/encoding.unit.js delete mode 100644 test/services/address/history.unit.js delete mode 100644 test/services/address/index.unit.js delete mode 100644 test/services/db.unit.js diff --git a/.travis.yml b/.travis.yml index 06a1ff97..8e1fcec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,9 +14,5 @@ node_js: - "v0.12.7" - "v4" script: - - _mocha -R spec regtest/p2p.js - - _mocha -R spec regtest/bitcoind.js - - _mocha -R spec regtest/cluster.js - - _mocha -R spec regtest/node.js - - _mocha -R spec --recursive - + - npm run regtest + - npm run test diff --git a/lib/errors.js b/lib/errors.js index 0a1b6a42..c534d0a2 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -3,23 +3,10 @@ var createError = require('errno').create; var BitcoreNodeError = createError('BitcoreNodeError'); -var NoOutputs = createError('NoOutputs', BitcoreNodeError); -var NoOutput = createError('NoOutput', BitcoreNodeError); -var Wallet = createError('WalletError', BitcoreNodeError); -Wallet.InsufficientFunds = createError('InsufficientFunds', Wallet); - -var Consensus = createError('Consensus', BitcoreNodeError); -Consensus.BlockExists = createError('BlockExists', Consensus); - -var Transaction = createError('Transaction', BitcoreNodeError); -Transaction.NotFound = createError('NotFound', Transaction); +var RPCError = createError('RPCError', BitcoreNodeError); module.exports = { Error: BitcoreNodeError, - NoOutputs: NoOutputs, - NoOutput: NoOutput, - Wallet: Wallet, - Consensus: Consensus, - Transaction: Transaction + RPCError: RPCError }; diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index fc5f8c6d..1f584b55 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -7,6 +7,7 @@ var path = require('path'); * or default locations. * @param {Object} options * @param {String} options.network - "testnet" or "livenet" + * @param {String} options.datadir - Absolute path to bitcoin database directory */ function getDefaultBaseConfig(options) { if (!options) { diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fdc2a612..fe2d89d4 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -14,7 +14,9 @@ var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var index = require('../'); +var errors = index.errors; var log = index.log; +var utils = require('../utils'); var Service = require('../service'); var Transaction = require('../transaction'); @@ -45,6 +47,12 @@ function Bitcoin(options) { this.subscriptions.transaction = []; this.subscriptions.block = []; + // limits + this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; + + // try all interval + this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; + // available bitcoind nodes this._initClients(); } @@ -52,7 +60,21 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; -Bitcoin.DEFAULT_CONFIG = 'whitelist=127.0.0.1\n' + 'txindex=1\n' + 'addressindex=1\n' + 'server=1\n'; +Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; +Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; +Bitcoin.DEFAULT_CONFIG_SETTINGS = { + server: 1, + whitelist: '127.0.0.1', + txindex: 1, + addressindex: 1, + timestampindex: 1, + spentindex: 1, + zmqpubrawtx: 'tcp://127.0.0.1:28332', + zmqpubhashblock: 'tcp://127.0.0.1:28332', + rpcallowip: '127.0.0.1', + rpcuser: 'bitcoin', + rpcpassword: 'local321' +}; Bitcoin.prototype._initCaches = function() { // caches valid until there is a new block @@ -70,8 +92,8 @@ Bitcoin.prototype._initCaches = function() { this.blockHeaderCache = LRU(288); this.zmqKnownTransactions = LRU(50); this.zmqKnownBlocks = LRU(50); - this.zmqLastBlock = 0; - this.zmqUpdateTipTimeout = false; + this.lastTip = 0; + this.lastTipTimeout = false; }; Bitcoin.prototype._initClients = function() { @@ -149,6 +171,15 @@ Bitcoin.prototype.unsubscribe = function(name, emitter) { } }; +Bitcoin.prototype._getDefaultConfig = function() { + var config = ''; + var defaults = Bitcoin.DEFAULT_CONFIG_SETTINGS; + for(var key in defaults) { + config += key + '=' + defaults[key] + '\n'; + } + return config; +}; + Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ @@ -169,6 +200,11 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { mkdirp.sync(spawnOptions.datadir); } + if (!fs.existsSync(configPath)) { + var defaultConfig = this._getDefaultConfig(); + fs.writeFileSync(configPath, defaultConfig); + } + var file = fs.readFileSync(configPath); var unparsed = file.toString().split('\n'); for(var i = 0; i < unparsed.length; i++) { @@ -187,6 +223,11 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { var spawnConfig = this.spawn.config; + this._checkConfigIndexes(spawnConfig, node); + +}; + +Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { $.checkState( spawnConfig.txindex && spawnConfig.txindex === 1, '"txindex" option is required in order to use transaction query features of bitcore-node. ' + @@ -233,7 +274,6 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { 'of bitcore-node services will start.'); node._reindex = true; } - }; Bitcoin.prototype._resetCaches = function() { @@ -245,11 +285,11 @@ Bitcoin.prototype._resetCaches = function() { }; Bitcoin.prototype._tryAll = function(func, callback) { - async.retry({times: this.nodes.length, interval: 1000}, func, callback); + async.retry({times: this.nodes.length, interval: this.tryAllInterval || 1000}, func, callback); }; Bitcoin.prototype._wrapRPCError = function(errObj) { - var err = new Error(errObj.message); + var err = new errors.RPCError(errObj.message); err.code = errObj.code; return err; }; @@ -274,11 +314,11 @@ Bitcoin.prototype._initChain = function(callback) { return callback(self._wrapRPCError(err)); } var blockhash = response.result; - self.getBlock(blockhash, function(err, block) { + self.getRawBlock(blockhash, function(err, blockBuffer) { if (err) { return callback(err); } - self.genesisBuffer = block.toBuffer(); + self.genesisBuffer = blockBuffer; self.emit('ready'); log.info('Bitcoin Daemon Ready'); callback(); @@ -303,55 +343,72 @@ Bitcoin.prototype._getNetworkOption = function() { Bitcoin.prototype._zmqBlockHandler = function(node, message) { var self = this; - function updateChain() { - var hex = message.toString('hex'); - if (hex !== self.tiphash) { - self._resetCaches(); - self.tiphash = message.toString('hex'); - node.client.getBlock(self.tiphash, function(err, response) { - if (err) { - return log.error(self._wrapRPCError(err)); - } - self.height = response.result.height; - $.checkState(self.height >= 0); - self.emit('tip', self.height); - }); + // Update the current chain tip + self._rapidProtectedUpdateTip(node, message); - if(!self.node.stopping) { - self.syncPercentage(function(err, percentage) { - if (err) { - return log.error(err); - } - if (Math.round(percentage) >= 100) { - self.emit('synced', self.height); - } - log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); - }); - } + // Notify block subscribers + var id = message.toString('binary'); + if (!self.zmqKnownBlocks.get(id)) { + self.zmqKnownBlocks.set(id, true); + self.emit('block', message); + + for (var i = 0; i < this.subscriptions.block.length; i++) { + this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); } } +}; + +Bitcoin.prototype._rapidProtectedUpdateTip = function(node, message) { + var self = this; + // Prevent a rapid succession of tip updates - if (new Date() - self.zmqLastBlock > 1000) { - self.zmqLastBlock = new Date(); - updateChain(); + if (new Date() - self.lastTip > 1000) { + self.lastTip = new Date(); + self._updateTip(node, message); } else { - clearTimeout(self.zmqUpdateTipTimeout); - self.zmqUpdateTipTimeout = setTimeout(function() { - updateChain(); + clearTimeout(self.lastTipTimeout); + self.lastTipTimeout = setTimeout(function() { + self._updateTip(node, message); }, 1000); } +}; - // Notify block subscribers - var id = message.toString('binary'); - if (!self.zmqKnownBlocks[id]) { - self.zmqKnownBlocks[id] = true; - self.emit('block', message); +Bitcoin.prototype._updateTip = function(node, message) { + var self = this; - for (var i = 0; i < this.subscriptions.block.length; i++) { - this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); - } + var hex = message.toString('hex'); + if (hex !== self.tiphash) { + self.tiphash = message.toString('hex'); + + // reset block valid caches + self._resetCaches(); + node.client.getBlock(self.tiphash, function(err, response) { + if (err) { + var error = self._wrapRPCError(err); + log.error(error); + self.emit('error', error); + } else { + self.height = response.result.height; + $.checkState(self.height >= 0); + self.emit('tip', self.height); + } + }); + + if(!self.node.stopping) { + self.syncPercentage(function(err, percentage) { + if (err) { + log.error(err); + self.emit('error', err); + } else { + if (Math.round(percentage) >= 100) { + self.emit('synced', self.height); + } + log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + } + }); + } } }; @@ -359,16 +416,15 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { Bitcoin.prototype._zmqTransactionHandler = function(node, message) { var self = this; var id = message.toString('binary'); - if (!self.zmqKnownTransactions[id]) { - self.zmqKnownTransactions[id] = true; + if (!self.zmqKnownTransactions.get(id)) { + self.zmqKnownTransactions.set(id, true); self.emit('tx', message); - } - // Notify transaction subscribers - for (var i = 0; i < this.subscriptions.transaction.length; i++) { - this.subscriptions.transaction[i].emit('bitcoind/transaction', message); + // Notify transaction subscribers + for (var i = 0; i < this.subscriptions.transaction.length; i++) { + this.subscriptions.transaction[i].emit('bitcoind/transaction', message.toString('hex')); + } } - }; Bitcoin.prototype._subscribeZmqEvents = function(node) { @@ -414,21 +470,24 @@ Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { Bitcoin.prototype._checkReindex = function(node, callback) { var self = this; + var interval; + function finish(err) { + clearInterval(interval); + callback(err); + } if (node._reindex) { - var interval = setInterval(function() { + interval = setInterval(function() { node.client.syncPercentage(function(err, percentSynced) { if (err) { - return log.error(self._wrapRPCError(err)); + return finish(self._wrapRPCError(err)); } log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); if (Math.round(percentSynced) >= 100) { node._reindex = false; - callback(); - clearInterval(interval); + finish(); } }); }, self._reindexWait); - } else { callback(); } @@ -787,11 +846,15 @@ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { }; Bitcoin.prototype._getConfirmationsDetail = function(transaction) { + $.checkState(this.height > 0, 'current height is unknown'); var confirmations = 0; if (transaction.__height >= 0) { confirmations = this.height - transaction.__height + 1; } - return confirmations; + if (confirmations < 0) { + log.warn('Negative confirmations calculated for transaction:', transaction.hash); + } + return Math.max(0, confirmations); }; Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addressStrings) { @@ -904,6 +967,7 @@ Bitcoin.prototype._getAddressStrings = function(addresses) { Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { var txids; if (from >= 0 && to >= 0) { + $.checkState(from < to, '"from" is expected to be less than "to"'); txids = fullTxids.slice(from, to); } else { txids = fullTxids; @@ -933,7 +997,11 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { } var totalCount = txids.length; - txids = self._paginateTxids(txids, options.from, options.to); + try { + txids = self._paginateTxids(txids, options.from, options.to); + } catch(e) { + return callback(e); + } async.mapSeries( txids, @@ -1304,6 +1372,7 @@ Bitcoin.prototype.getTransaction = function(txid, callback) { * @param {Function} callback */ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { + // TODO give response back as standard js object with bitcore tx var self = this; var tx = self.transactionInfoCache.get(txid); if (tx) { @@ -1408,9 +1477,11 @@ Bitcoin.prototype.generateBlock = function(num, callback) { */ Bitcoin.prototype.stop = function(callback) { if (this.spawn && this.spawn.process) { - this.spawn.process.once('exit', function(err, status) { - if (err) { - return callback(err); + this.spawn.process.once('exit', function(code) { + if (code !== 0) { + var error = new Error('bitcoind spawned process exited with status code: ' + code); + error.code = code; + return callback(error); } else { return callback(); } diff --git a/lib/transaction.js b/lib/transaction.js index ef1d7d2a..66042a16 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -4,9 +4,6 @@ var async = require('async'); var bitcore = require('bitcore-lib'); var Transaction = bitcore.Transaction; -var index = require('./'); -var errors = index.errors; - var MAX_TRANSACTION_LIMIT = 5; Transaction.prototype.populateSpentInfo = function(db, options, callback) { @@ -53,11 +50,13 @@ Transaction.prototype.populateInputs = function(db, poolTransactions, callback) Transaction.prototype._populateInput = function(db, input, poolTransactions, callback) { if (!input.prevTxId || !Buffer.isBuffer(input.prevTxId)) { - return callback(new Error('Input is expected to have prevTxId as a buffer')); + return callback(new TypeError('Input is expected to have prevTxId as a buffer')); } var txid = input.prevTxId.toString('hex'); db.getTransaction(txid, function(err, prevTx) { - if(!prevTx) { + if(err) { + return callback(err); + } else if (!prevTx) { // Check the pool for transaction for(var i = 0; i < poolTransactions.length; i++) { if(txid === poolTransactions[i].hash) { @@ -65,25 +64,10 @@ Transaction.prototype._populateInput = function(db, input, poolTransactions, cal return callback(); } } - return callback(new Error('Previous tx ' + input.prevTxId.toString('hex') + ' not found')); - } else if(err) { - callback(err); - } else { - input.output = prevTx.outputs[input.outputIndex]; - callback(); - } - }); -}; - -Transaction.prototype._checkSpent = function(db, input, poolTransactions, callback) { - // TODO check and see if another transaction in the pool spent the output - db.isSpentDB(input, function(spent) { - if(spent) { - return callback(new Error('Input already spent')); - } else { - callback(); } + input.output = prevTx.outputs[input.outputIndex]; + callback(); }); }; diff --git a/package.json b/package.json index a7bc9fdd..4f4848bb 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "scripts": { "install": "./scripts/install", "test": "NODE_ENV=test mocha -R spec --recursive", + "regtest": "./scripts/regtest", "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" }, "tags": [ diff --git a/scripts/regtest b/scripts/regtest new file mode 100755 index 00000000..96078911 --- /dev/null +++ b/scripts/regtest @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +_mocha -R spec regtest/p2p.js +_mocha -R spec regtest/bitcoind.js +_mocha -R spec regtest/cluster.js +_mocha -R spec regtest/node.js diff --git a/test/bin/get-tarball-name.js b/test/bin/get-tarball-name.js deleted file mode 100644 index 6f8bdbde..00000000 --- a/test/bin/get-tarball-name.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var path = require('path'); -var getTarballName = require('../../bin/get-tarball-name'); -var execSync = require('child_process').execSync; - -describe('#getTarballName', function() { - it('will return the expected tarball name', function() { - var name = getTarballName(); - var version = require(path.resolve(__dirname + '../../../package.json')).version; - var platform = process.platform; - var arch = execSync(path.resolve(__dirname) + '/../../bin/variables.sh arch'); - var abi = process.versions.modules; - var expected = 'libbitcoind-' + version + '-node' + abi + '-' + platform + '-' + arch + '.tgz'; - name.should.equal(expected); - }); -}); diff --git a/test/data/bitcoin.conf b/test/data/bitcoin.conf index 475d225d..353387fa 100644 --- a/test/data/bitcoin.conf +++ b/test/data/bitcoin.conf @@ -1,17 +1,23 @@ #testnet=1 #irc=0 -#upnp=0 +upnp=0 server=1 whitelist=127.0.0.1 txindex=1 +addressindex=1 +timestampindex=1 +spentindex=1 +dbcache=8192 +checkblocks=144 +maxuploadtarget=1024 +zmqpubrawtx=tcp://127.0.0.1:28332 +zmqpubhashblock=tcp://127.0.0.1:28332 -# listen on different ports port=20000 +rpcport=50001 rpcallowip=127.0.0.1 rpcuser=bitcoin rpcpassword=local321 - - diff --git a/test/data/default.bitcoin.conf b/test/data/default.bitcoin.conf new file mode 100644 index 00000000..0f1bfde7 --- /dev/null +++ b/test/data/default.bitcoin.conf @@ -0,0 +1,11 @@ +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +spentindex=1 +zmqpubrawtx=tcp://127.0.0.1:28332 +zmqpubhashblock=tcp://127.0.0.1:28332 +rpcallowip=127.0.0.1 +rpcuser=bitcoin +rpcpassword=local321 diff --git a/test/index.unit.js b/test/index.unit.js deleted file mode 100644 index c2190441..00000000 --- a/test/index.unit.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var index = require('..'); - -describe('Index', function() { - describe('#nodeVersionCheck', function() { - it('will throw informative error message with incompatible Node.js version 4.1.2', function() { - (function() { - index.nodeVersionCheck('4.1.2', '>=0.12.0 <1'); - }).should.throw('Node.js version'); - }); - it('will throw informative error message with incompatible Node.js version 0.10.40', function() { - (function() { - index.nodeVersionCheck('4.1.2', '>=0.12.0 <1'); - }).should.throw('Node.js version'); - }); - }); -}); diff --git a/test/scaffold/default-base-config.integration.js b/test/scaffold/default-base-config.integration.js index 5b09b3cd..b622c742 100644 --- a/test/scaffold/default-base-config.integration.js +++ b/test/scaffold/default-base-config.integration.js @@ -1,6 +1,7 @@ 'use strict'; var should = require('chai').should(); +var path = require('path'); var defaultBaseConfig = require('../../lib/scaffold/default-base-config'); describe('#defaultBaseConfig', function() { @@ -9,29 +10,19 @@ describe('#defaultBaseConfig', function() { var home = process.env.HOME; var info = defaultBaseConfig(); info.path.should.equal(cwd); - info.config.datadir.should.equal(home + '/.bitcoin'); info.config.network.should.equal('livenet'); info.config.port.should.equal(3001); - info.config.services.should.deep.equal(['bitcoind', 'db', 'address', 'web']); + info.config.services.should.deep.equal(['bitcoind', 'web']); + var bitcoind = info.config.servicesConfig.bitcoind; + bitcoind.spawn.datadir.should.equal(home + '/.bitcoin'); + bitcoind.spawn.exec.should.equal(path.resolve(__dirname, '../../bin/bitcoind')); }); it('be able to specify a network', function() { - var cwd = process.cwd(); - var home = process.env.HOME; var info = defaultBaseConfig({network: 'testnet'}); - info.path.should.equal(cwd); - info.config.datadir.should.equal(home + '/.bitcoin'); info.config.network.should.equal('testnet'); - info.config.port.should.equal(3001); - info.config.services.should.deep.equal(['bitcoind', 'db', 'address', 'web']); }); it('be able to specify a datadir', function() { - var cwd = process.cwd(); - var home = process.env.HOME; var info = defaultBaseConfig({datadir: './data2', network: 'testnet'}); - info.path.should.equal(cwd); - info.config.datadir.should.equal('./data2'); - info.config.network.should.equal('testnet'); - info.config.port.should.equal(3001); - info.config.services.should.deep.equal(['bitcoind', 'db', 'address', 'web']); + info.config.servicesConfig.bitcoind.spawn.datadir.should.equal('./data2'); }); }); diff --git a/test/scaffold/default-config.integration.js b/test/scaffold/default-config.integration.js index cbe00816..83e674e5 100644 --- a/test/scaffold/default-config.integration.js +++ b/test/scaffold/default-config.integration.js @@ -1,21 +1,29 @@ 'use strict'; +var path = require('path'); var should = require('chai').should(); var sinon = require('sinon'); var proxyquire = require('proxyquire'); describe('#defaultConfig', function() { + var expectedExecPath = path.resolve(__dirname, '../../bin/bitcoind'); + it('will return expected configuration', function() { var config = JSON.stringify({ - datadir: process.env.HOME + '/.bitcore/data', network: 'livenet', port: 3001, services: [ 'bitcoind', - 'db', - 'address', 'web' - ] + ], + servicesConfig: { + bitcoind: { + spawn: { + datadir: process.env.HOME + '/.bitcore/data', + exec: expectedExecPath + } + } + } }, null, 2); var defaultConfig = proxyquire('../../lib/scaffold/default-config', { fs: { @@ -32,28 +40,35 @@ describe('#defaultConfig', function() { sync: sinon.stub() } }); - var cwd = process.cwd(); var home = process.env.HOME; var info = defaultConfig(); info.path.should.equal(home + '/.bitcore'); - info.config.datadir.should.equal(home + '/.bitcore/data'); info.config.network.should.equal('livenet'); info.config.port.should.equal(3001); - info.config.services.should.deep.equal(['bitcoind', 'db', 'address', 'web']); + info.config.services.should.deep.equal(['bitcoind', 'web']); + var bitcoind = info.config.servicesConfig.bitcoind; + should.exist(bitcoind); + bitcoind.spawn.datadir.should.equal(home + '/.bitcore/data'); + bitcoind.spawn.exec.should.equal(expectedExecPath); }); it('will include additional services', function() { var config = JSON.stringify({ - datadir: process.env.HOME + '/.bitcore/data', network: 'livenet', port: 3001, services: [ 'bitcoind', - 'db', - 'address', 'web', 'insight-api', 'insight-ui' - ] + ], + servicesConfig: { + bitcoind: { + spawn: { + datadir: process.env.HOME + '/.bitcore/data', + exec: expectedExecPath + } + } + } }, null, 2); var defaultConfig = proxyquire('../../lib/scaffold/default-config', { fs: { @@ -75,16 +90,17 @@ describe('#defaultConfig', function() { additionalServices: ['insight-api', 'insight-ui'] }); info.path.should.equal(home + '/.bitcore'); - info.config.datadir.should.equal(home + '/.bitcore/data'); info.config.network.should.equal('livenet'); info.config.port.should.equal(3001); info.config.services.should.deep.equal([ 'bitcoind', - 'db', - 'address', 'web', 'insight-api', 'insight-ui' ]); + var bitcoind = info.config.servicesConfig.bitcoind; + should.exist(bitcoind); + bitcoind.spawn.datadir.should.equal(home + '/.bitcore/data'); + bitcoind.spawn.exec.should.equal(expectedExecPath); }); }); diff --git a/test/scaffold/start.integration.js b/test/scaffold/start.integration.js index 2374d163..b2f4f61c 100644 --- a/test/scaffold/start.integration.js +++ b/test/scaffold/start.integration.js @@ -3,7 +3,7 @@ var should = require('chai').should(); var sinon = require('sinon'); var proxyquire = require('proxyquire'); -var AddressService = require('../../lib/services/address'); +var BitcoinService = require('../../lib/services/bitcoind'); describe('#start', function() { @@ -13,8 +13,8 @@ describe('#start', function() { var node; var TestNode = function(options) { options.services[0].should.deep.equal({ - name: 'address', - module: AddressService, + name: 'bitcoind', + module: BitcoinService, config: {} }); }; @@ -32,7 +32,7 @@ describe('#start', function() { path: __dirname, config: { services: [ - 'address' + 'bitcoind' ], datadir: './data' } @@ -67,8 +67,8 @@ describe('#start', function() { var node; var TestNode = function(options) { options.services[0].should.deep.equal({ - name: 'address', - module: AddressService, + name: 'bitcoind', + module: BitcoinService, config: { param: 'test' } @@ -88,10 +88,10 @@ describe('#start', function() { path: __dirname, config: { services: [ - 'address' + 'bitcoind' ], servicesConfig: { - 'address': { + 'bitcoind': { param: 'test' } }, diff --git a/test/scaffold/start.unit.js b/test/scaffold/start.unit.js index c955cbf9..85b2c507 100644 --- a/test/scaffold/start.unit.js +++ b/test/scaffold/start.unit.js @@ -212,110 +212,6 @@ describe('#start', function() { }); }); }); - describe('#spawnChildProcess', function() { - - it('should build the appropriate arguments to spawn a child process', function() { - var child = { - unref: function() {} - }; - var _process = { - exit: function() {}, - env: { - __bitcore_node: false - }, - argv: [ - 'node', - 'bitcore-node' - ], - cwd: function(){return ''}, - pid: 999, - execPath: '/tmp' - }; - var fd = {}; - var spawn = sinon.stub().returns(child); - var openSync = sinon.stub().returns(fd); - var spawnChildProcess = proxyquire('../../lib/scaffold/start', { - fs: { - openSync: openSync - }, - child_process: { - spawn: spawn - } - }).spawnChildProcess; - - spawnChildProcess('/tmp', _process); - - spawn.callCount.should.equal(1); - spawn.args[0][0].should.equal(_process.execPath); - var expected = [].concat(_process.argv); - expected.shift(); - spawn.args[0][1].should.deep.equal(expected); - var cp_opt = { - stdio: ['ignore', fd, fd], - env: _process.env, - cwd: '', - detached: true - }; - spawn.args[0][2].should.deep.equal(cp_opt); - openSync.callCount.should.equal(1); - openSync.args[0][0].should.equal('/tmp/bitcore-node.log'); - openSync.args[0][1].should.equal('a+'); - }); - it('should not spawn a new child process if there is already a daemon running', function() { - var _process = { - exit: function() {}, - env: { - __bitcore_node: true - }, - argv: [ - 'node', - 'bitcore-node' - ], - cwd: 'cwd', - pid: 999, - execPath: '/tmp' - }; - var spawnChildProcess = proxyquire('../../lib/scaffold/start', {}).spawnChildProcess; - spawnChildProcess('/tmp', _process).should.equal(999); - }); - }); - describe('daemon', function() { - var sandbox; - var spawn; - var setup; - var registerSync; - var registerExit; - var start = require('../../lib/scaffold/start'); - var options = { - config: { - datadir: '/tmp', - daemon: true - } - } - beforeEach(function() { - sandbox = sinon.sandbox.create(); - spawn = sandbox.stub(start, 'spawnChildProcess', function() {}); - setup = sandbox.stub(start, 'setupServices', function() {}); - registerSync = sandbox.stub(start, 'registerSyncHandlers', function() {}); - registerExit = sandbox.stub(start, 'registerExitHandlers', function() {}); - }); - afterEach(function() { - sandbox.restore(); - }); - it('call spawnChildProcess if there is a config option to do so', function() { - start(options); - registerSync.callCount.should.equal(1); - registerExit.callCount.should.equal(1); - spawn.callCount.should.equal(1); - }); - it('not call spawnChildProcess if there is not an option to do so', function() { - options.config.daemon = false; - start(options); - registerSync.callCount.should.equal(1); - registerExit.callCount.should.equal(1); - spawn.callCount.should.equal(0); - }); - }); describe('#registerExitHandlers', function() { var stub; var registerExitHandlers = require('../../lib/scaffold/start').registerExitHandlers; diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js deleted file mode 100644 index e5ba7376..00000000 --- a/test/services/address/encoding.unit.js +++ /dev/null @@ -1,103 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var should = chai.should(); -var sinon = require('sinon'); -var bitcorenode = require('../../../'); -var bitcore = require('bitcore-lib'); -var Address = bitcore.Address; -var Script = bitcore.Script; -var AddressService = bitcorenode.services.Address; -var Networks = bitcore.Networks; -var encoding = require('../../../lib/services/address/encoding'); - -var mockdb = { -}; - -var mocknode = { - network: Networks.testnet, - datadir: 'testdir', - db: mockdb, - services: { - bitcoind: { - on: sinon.stub() - } - } -}; - -describe('Address Service Encoding', function() { - - describe('#encodeSpentIndexSyncKey', function() { - it('will encode to 36 bytes (string)', function() { - var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); - key.length.should.equal(36); - }); - it('will be able to decode encoded value', function() { - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var txidBuffer = new Buffer(txid, 'hex'); - var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); - var keyBuffer = new Buffer(key, 'binary'); - keyBuffer.slice(0, 32).toString('hex').should.equal(txid); - var outputIndex = keyBuffer.readUInt32BE(32); - outputIndex.should.equal(12); - }); - }); - - describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { - var encoded; - var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - encoded = encoding.encodeInputKeyMap(outputTxIdBuffer, 13); - }); - it('decode key', function() { - var key = encoding.decodeInputKeyMap(encoded); - key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); - key.outputIndex.should.equal(13); - }); - }); - - describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { - var encoded; - var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - encoded = encoding.encodeInputValueMap(inputTxIdBuffer, 7); - }); - it('decode key', function() { - var key = encoding.decodeInputValueMap(encoded); - key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); - key.inputIndex.should.equal(7); - }); - }); - - - describe('#extractAddressInfoFromScript', function() { - it('pay-to-publickey', function() { - var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); - var script = Script.buildPublicKeyOut(pubkey); - var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); - }); - it('pay-to-publickeyhash', function() { - var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); - var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('pay-to-scripthash', function() { - var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); - var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); - info.addressType.should.equal(Address.PayToScriptHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('non-address script type', function() { - var buf = new Buffer(40); - buf.fill(0); - var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); - var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); - info.should.equal(false); - }); - }); - -}); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js deleted file mode 100644 index 2b6df06c..00000000 --- a/test/services/address/history.unit.js +++ /dev/null @@ -1,544 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var sinon = require('sinon'); -var bitcore = require('bitcore-lib'); -var Transaction = require('../../../lib/transaction'); -var AddressHistory = require('../../../lib/services/address/history'); - -describe('Address Service History', function() { - - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - - describe('@constructor', function() { - it('will construct a new instance', function() { - var node = {}; - var options = {}; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history.should.be.instanceof(AddressHistory); - history.node.should.equal(node); - history.options.should.equal(options); - history.addresses.should.equal(addresses); - history.detailedArray.should.deep.equal([]); - }); - it('will set addresses an array if only sent a string', function() { - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: address - }); - history.addresses.should.deep.equal([address]); - }); - }); - - describe('#get', function() { - it('will give an error if length of addresses is too long', function(done) { - var node = {}; - var options = {}; - var addresses = []; - for (var i = 0; i < 101; i++) { - addresses.push(address); - } - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history.maxAddressesQuery = 100; - history.get(function(err) { - should.exist(err); - err.message.match(/Maximum/); - done(); - }); - }); - it('give error from getAddressSummary with one address', function(done) { - var node = { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, new Error('test')) - } - } - }; - var options = {}; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history.get(function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - it('give error from getAddressSummary with multiple addresses', function(done) { - var node = { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, new Error('test2')) - } - } - }; - var options = {}; - var addresses = [address, address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history.get(function(err) { - should.exist(err); - err.message.should.equal('test2'); - done(); - }); - }); - it('will query get address summary directly with one address', function(done) { - var txids = []; - var summary = { - txids: txids - }; - var node = { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, null, summary) - } - } - }; - var options = {}; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history._mergeAndSortTxids = sinon.stub(); - history._paginateWithDetails = sinon.stub().callsArg(1); - history.get(function() { - history.node.services.address.getAddressSummary.callCount.should.equal(1); - history.node.services.address.getAddressSummary.args[0][0].should.equal(address); - history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ - noBalance: true - }); - history._paginateWithDetails.callCount.should.equal(1); - history._paginateWithDetails.args[0][0].should.equal(txids); - history._mergeAndSortTxids.callCount.should.equal(0); - done(); - }); - }); - it('will merge multiple summaries with multiple addresses', function(done) { - var txids = []; - var summary = { - txids: txids - }; - var node = { - services: { - address: { - getAddressSummary: sinon.stub().callsArgWith(2, null, summary) - } - } - }; - var options = {}; - var addresses = [address, address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - history._mergeAndSortTxids = sinon.stub().returns(txids); - history._paginateWithDetails = sinon.stub().callsArg(1); - history.get(function() { - history.node.services.address.getAddressSummary.callCount.should.equal(2); - history.node.services.address.getAddressSummary.args[0][0].should.equal(address); - history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ - fullTxList: true, - noBalance: true - }); - history._paginateWithDetails.callCount.should.equal(1); - history._paginateWithDetails.args[0][0].should.equal(txids); - history._mergeAndSortTxids.callCount.should.equal(1); - done(); - }); - }); - }); - - describe('#_paginateWithDetails', function() { - it('slice txids based on "from" and "to" (3 to 30)', function() { - var node = {}; - var options = { - from: 3, - to: 30 - }; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - sinon.stub(history, 'getDetailedInfo', function(txid, next) { - this.detailedArray.push(txid); - next(); - }); - history._paginateWithDetails(txids, function(err, result) { - result.totalCount.should.equal(11); - result.items.should.deep.equal([7, 6, 5, 4, 3, 2, 1, 0]); - }); - }); - it('slice txids based on "from" and "to" (0 to 3)', function() { - var node = {}; - var options = { - from: 0, - to: 3 - }; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - sinon.stub(history, 'getDetailedInfo', function(txid, next) { - this.detailedArray.push(txid); - next(); - }); - history._paginateWithDetails(txids, function(err, result) { - result.totalCount.should.equal(11); - result.items.should.deep.equal([10, 9, 8]); - }); - }); - it('will given an error if the full details is too long', function() { - var node = {}; - var options = { - from: 0, - to: 3 - }; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - sinon.stub(history, 'getDetailedInfo', function(txid, next) { - this.detailedArray.push(txid); - next(); - }); - history.maxHistoryQueryLength = 1; - history._paginateWithDetails(txids, function(err) { - should.exist(err); - err.message.match(/Maximum/); - }); - }); - it('will give full result without pagination options', function() { - var node = {}; - var options = {}; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - sinon.stub(history, 'getDetailedInfo', function(txid, next) { - this.detailedArray.push(txid); - next(); - }); - history._paginateWithDetails(txids, function(err, result) { - result.totalCount.should.equal(11); - result.items.should.deep.equal([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]); - }); - }); - }); - - describe('#_mergeAndSortTxids', function() { - it('will merge and sort multiple summaries', function() { - var summaries = [ - { - totalReceived: 10000000, - totalSpent: 0, - balance: 10000000, - appearances: 2, - unconfirmedBalance: 20000000, - unconfirmedAppearances: 2, - appearanceIds: { - '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579': 154, - 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce': 120 - }, - unconfirmedAppearanceIds: { - 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, - 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964 - } - }, - { - totalReceived: 59990000, - totalSpent: 0, - balance: 49990000, - appearances: 3, - unconfirmedBalance: 1000000, - unconfirmedAppearances: 3, - appearanceIds: { - 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2': 156, - 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47': 152, - 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f': 151 - }, - unconfirmedAppearanceIds: { - 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, - 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, - 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 - } - } - ]; - var node = {}; - var options = {}; - var addresses = [address]; - var history = new AddressHistory({ - node: node, - options: options, - addresses: addresses - }); - var txids = history._mergeAndSortTxids(summaries); - txids.should.deep.equal([ - 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', - 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', - 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47', - '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579', - 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', - 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', - 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', - 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a', - 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693', - 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681' - ]); - }); - }); - - describe('#getDetailedInfo', function() { - it('will add additional information to existing this.transactions', function(done) { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - var tx = { - populateInputs: sinon.stub().callsArg(2), - __height: 20, - __timestamp: 1453134151, - isCoinbase: sinon.stub().returns(false), - getFee: sinon.stub().returns(1000) - }; - var history = new AddressHistory({ - node: { - services: { - db: { - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, tx), - tip: { - __height: 300 - } - } - } - }, - options: {}, - addresses: [] - }); - history.getAddressDetailsForTransaction = sinon.stub().returns({ - addresses: {}, - satoshis: 1000, - }); - history.getDetailedInfo(txid, function(err) { - if (err) { - throw err; - } - history.node.services.db.getTransactionWithBlockInfo.callCount.should.equal(1); - done(); - }); - }); - it('will handle error from getTransactionFromBlock', function(done) { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - var history = new AddressHistory({ - node: { - services: { - db: { - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, new Error('test')), - } - } - }, - options: {}, - addresses: [] - }); - history.getDetailedInfo(txid, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will handle error from populateInputs', function(done) { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - var history = new AddressHistory({ - node: { - services: { - db: { - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, { - populateInputs: sinon.stub().callsArgWith(2, new Error('test')) - }), - } - } - }, - options: {}, - addresses: [] - }); - history.getDetailedInfo(txid, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will set this.transactions with correct information', function(done) { - // block #314159 - // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e - // outputIndex 1 - var txAddress = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; - var txString = '0100000001a08ee59fcd5d86fa170abb6d925d62d5c5c476359681b70877c04f270c4ef246000000008a47304402203fb9b476bb0c37c9b9ed5784ebd67ae589492be11d4ae1612be29887e3e4ce750220741ef83781d1b3a5df8c66fa1957ad0398c733005310d7d9b1d8c2310ef4f74c0141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02b0a75fac000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac20badc02000000001976a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac00000000'; - var previousTxString = '010000000155532fad2869bb951b0bd646a546887f6ee668c4c0ee13bf3f1c4bce6d6e3ed9000000008c4930460221008540795f4ef79b1d2549c400c61155ca5abbf3089c84ad280e1ba6db2a31abce022100d7d162175483d51174d40bba722e721542c924202a0c2970b07e680b51f3a0670141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02f0af3caf000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac80969800000000001976a91421277e65777760d1f3c7c982ba14ed8f934f005888ac00000000'; - var transaction = new Transaction(); - var previousTransaction = new Transaction(); - previousTransaction.fromString(previousTxString); - var previousTransactionTxid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - transaction.fromString(txString); - var txid = transaction.hash; - transaction.__blockHash = '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45'; - transaction.__height = 314159; - transaction.__timestamp = 1407292005; - var history = new AddressHistory({ - node: { - services: { - db: { - tip: { - __height: 314159 - }, - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, transaction), - getTransaction: function(prevTxid, queryMempool, callback) { - prevTxid.should.equal(previousTransactionTxid); - setImmediate(function() { - callback(null, previousTransaction); - }); - } - } - } - }, - options: {}, - addresses: [txAddress] - }); - var transactionInfo = { - addresses: {}, - txid: txid, - timestamp: 1407292005, - satoshis: 48020000, - address: txAddress - }; - transactionInfo.addresses[txAddress] = {}; - transactionInfo.addresses[txAddress].outputIndexes = [1]; - transactionInfo.addresses[txAddress].inputIndexes = []; - history.getDetailedInfo(txid, function(err) { - if (err) { - throw err; - } - var info = history.detailedArray[0]; - info.addresses[txAddress].should.deep.equal({ - outputIndexes: [1], - inputIndexes: [] - }); - info.satoshis.should.equal(48020000); - info.height.should.equal(314159); - info.confirmations.should.equal(1); - info.timestamp.should.equal(1407292005); - info.fees.should.equal(20000); - info.tx.should.equal(transaction); - done(); - }); - }); - }); - - describe('#getAddressDetailsForTransaction', function() { - it('will calculate details for the transaction', function(done) { - /* jshint sub:true */ - var tx = bitcore.Transaction({ - 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', - 'version': 1, - 'inputs': [ - { - 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', - 'outputIndex': 0, - 'sequenceNumber': 4294967295, - 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', - 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', - 'output': { - 'satoshis': 1000000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - } - } - ], - 'outputs': [ - { - 'satoshis': 100000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - }, - { - 'satoshis': 200000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - }, - { - 'satoshis': 50000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - }, - { - 'satoshis': 300000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - }, - { - 'satoshis': 349990000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - } - ], - 'nLockTime': 0 - }); - var history = new AddressHistory({ - node: { - network: bitcore.Networks.testnet - }, - options: {}, - addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] - }); - var details = history.getAddressDetailsForTransaction(tx); - should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']); - details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]); - details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([ - 0, 1, 2, 3, 4 - ]); - details.satoshis.should.equal(-10000); - done(); - }); - }); - - describe('#getConfirmationsDetail', function() { - it('the correct confirmations when included in the tip', function(done) { - var history = new AddressHistory({ - node: { - services: { - db: { - tip: { - __height: 100 - } - } - } - }, - options: {}, - addresses: [] - }); - var transaction = { - __height: 100 - }; - history.getConfirmationsDetail(transaction).should.equal(1); - done(); - }); - }); -}); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js deleted file mode 100644 index 48bda199..00000000 --- a/test/services/address/index.unit.js +++ /dev/null @@ -1,2676 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var sinon = require('sinon'); -var stream = require('stream'); -var levelup = require('levelup'); -var proxyquire = require('proxyquire'); -var bitcorenode = require('../../../'); -var AddressService = bitcorenode.services.Address; -var blockData = require('../../data/livenet-345003.json'); -var bitcore = require('bitcore-lib'); -var _ = bitcore.deps._; -var memdown = require('memdown'); -var leveldown = require('leveldown'); -var Networks = bitcore.Networks; -var EventEmitter = require('events').EventEmitter; -var errors = bitcorenode.errors; -var Transaction = require('../../../lib/transaction'); -var txData = require('../../data/transaction.json'); -var index = require('../../../lib'); -var log = index.log; -var constants = require('../../../lib/services/address/constants'); -var encoding = require('../../../lib/services/address/encoding'); - -var mockdb = { -}; - -var mocknode = { - network: Networks.testnet, - datadir: 'testdir', - db: mockdb, - services: { - bitcoind: { - on: sinon.stub() - } - } -}; - -describe('Address Service', function() { - var txBuf = new Buffer(txData[0], 'hex'); - - describe('@constructor', function() { - it('config to use memdown for mempool index', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.levelupStore.should.equal(memdown); - }); - it('config to use leveldown for mempool index', function() { - var am = new AddressService({ - node: mocknode - }); - am.levelupStore.should.equal(leveldown); - }); - }); - - describe('#start', function() { - it('will flush existing mempool', function(done) { - var leveldownmock = { - destroy: sinon.stub().callsArgWith(1, null) - }; - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(true) - }, - 'leveldown': leveldownmock, - 'levelup': sinon.stub().callsArgWith(2, null), - 'mkdirp': sinon.stub().callsArgWith(1, null) - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function() { - leveldownmock.destroy.callCount.should.equal(1); - leveldownmock.destroy.args[0][0].should.equal('testdir/testnet3/bitcore-addressmempool.db'); - done(); - }); - }); - it('will mkdirp if directory does not exist', function(done) { - var leveldownmock = { - destroy: sinon.stub().callsArgWith(1, null) - }; - var mkdirpmock = sinon.stub().callsArgWith(1, null); - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(false) - }, - 'leveldown': leveldownmock, - 'levelup': sinon.stub().callsArgWith(2, null), - 'mkdirp': mkdirpmock - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function() { - mkdirpmock.callCount.should.equal(1); - mkdirpmock.args[0][0].should.equal('testdir/testnet3/bitcore-addressmempool.db'); - done(); - }); - }); - it('start levelup db for mempool', function(done) { - var levelupStub = sinon.stub().callsArg(2); - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(true) - }, - 'leveldown': { - destroy: sinon.stub().callsArgWith(1, null) - }, - 'levelup': levelupStub, - 'mkdirp': sinon.stub().callsArgWith(1, null) - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function() { - levelupStub.callCount.should.equal(1); - var dbPath1 = levelupStub.args[0][0]; - dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); - var options = levelupStub.args[0][1]; - options.db.should.equal(memdown); - options.keyEncoding.should.equal('binary'); - options.valueEncoding.should.equal('binary'); - options.fillCache.should.equal(false); - done(); - }); - }); - it('handle error from mkdirp', function(done) { - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(false) - }, - 'leveldown': { - destroy: sinon.stub().callsArgWith(1, null) - }, - 'levelup': sinon.stub().callsArgWith(2, null), - 'mkdirp': sinon.stub().callsArgWith(1, new Error('testerror')) - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function(err) { - err.message.should.equal('testerror'); - done(); - }); - }); - it('handle error from levelup', function(done) { - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(false) - }, - 'leveldown': { - destroy: sinon.stub().callsArgWith(1, null) - }, - 'levelup': sinon.stub().callsArgWith(2, new Error('leveltesterror')), - 'mkdirp': sinon.stub().callsArgWith(1, null) - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function(err) { - err.message.should.equal('leveltesterror'); - done(); - }); - }); - it('handle error from leveldown.destroy', function(done) { - var TestAddressService = proxyquire('../../../lib/services/address', { - 'fs': { - existsSync: sinon.stub().returns(true) - }, - 'leveldown': { - destroy: sinon.stub().callsArgWith(1, new Error('destroy')) - }, - 'levelup': sinon.stub().callsArgWith(2, null), - 'mkdirp': sinon.stub().callsArgWith(1, null) - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.start(function(err) { - err.message.should.equal('destroy'); - done(); - }); - }); - }); - - describe('#stop', function() { - it('will close mempool levelup', function(done) { - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - db: mockdb, - services: { - bitcoind: { - on: sinon.stub(), - removeListener: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.mempoolIndex = {}; - am.mempoolIndex.close = sinon.stub().callsArg(0); - am.stop(function() { - am.mempoolIndex.close.callCount.should.equal(1); - am.node.services.bitcoind.removeListener.callCount.should.equal(2); - done(); - }); - }); - }); - - describe('#_setMempoolIndexPath', function() { - it('should set the database path', function() { - var testnode = { - network: Networks.livenet, - datadir: process.env.HOME + '/.bitcoin', - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am._setMempoolIndexPath(); - am.mempoolIndexPath.should.equal(process.env.HOME + '/.bitcoin/bitcore-addressmempool.db'); - }); - it('should load the db for testnet', function() { - var testnode = { - network: Networks.testnet, - datadir: process.env.HOME + '/.bitcoin', - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am._setMempoolIndexPath(); - am.mempoolIndexPath.should.equal(process.env.HOME + '/.bitcoin/testnet3/bitcore-addressmempool.db'); - }); - it('error with unknown network', function() { - var testnode = { - network: 'unknown', - datadir: process.env.HOME + '/.bitcoin', - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - (function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - }).should.throw('Unknown network'); - }); - it('should load the db with regtest', function() { - // Switch to use regtest - Networks.enableRegtest(); - var regtest = Networks.get('regtest'); - var testnode = { - network: regtest, - datadir: process.env.HOME + '/.bitcoin', - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.mempoolIndexPath.should.equal(process.env.HOME + '/.bitcoin/regtest/bitcore-addressmempool.db'); - Networks.disableRegtest(); - }); - }); - - describe('#getAPIMethods', function() { - it('should return the correct methods', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var methods = am.getAPIMethods(); - methods.length.should.equal(7); - }); - }); - - describe('#getPublishEvents', function() { - it('will return an array of publish event objects', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.subscribe = sinon.spy(); - am.unsubscribe = sinon.spy(); - var events = am.getPublishEvents(); - - var callCount = 0; - function testName(event, name) { - event.name.should.equal(name); - event.scope.should.equal(am); - var emitter = new EventEmitter(); - var addresses = []; - event.subscribe(emitter, addresses); - am.subscribe.callCount.should.equal(callCount + 1); - am.subscribe.args[callCount][0].should.equal(name); - am.subscribe.args[callCount][1].should.equal(emitter); - am.subscribe.args[callCount][2].should.equal(addresses); - am.subscribe.thisValues[callCount].should.equal(am); - event.unsubscribe(emitter, addresses); - am.unsubscribe.callCount.should.equal(callCount + 1); - am.unsubscribe.args[callCount][0].should.equal(name); - am.unsubscribe.args[callCount][1].should.equal(emitter); - am.unsubscribe.args[callCount][2].should.equal(addresses); - am.unsubscribe.thisValues[callCount].should.equal(am); - callCount++; - } - events.forEach(function(event) { - testName(event, event.name); - }); - - }); - }); - - describe('#transactionOutputHandler', function() { - it('create a message for an address', function() { - var txBuf = new Buffer('01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000', 'hex'); - var tx = bitcore.Transaction().fromBuffer(txBuf); - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.node.network = Networks.livenet; - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var addrObj = bitcore.Address(address); - var hashHex = addrObj.hashBuffer.toString('hex'); - var hashType = addrObj.type; - var messages = {}; - am.transactionOutputHandler(messages, tx, 0, true); - should.exist(messages[hashHex]); - var message = messages[hashHex]; - message.tx.should.equal(tx); - message.outputIndexes.should.deep.equal([0]); - message.addressInfo.hashBuffer.toString('hex').should.equal(hashHex); - message.addressInfo.addressType.should.equal(hashType); - message.addressInfo.hashHex.should.equal(hashHex); - message.rejected.should.equal(true); - }); - }); - - describe('#transactionHandler', function() { - it('will pass outputs to transactionOutputHandler and call transactionEventHandler and balanceEventHandler', function(done) { - var txBuf = new Buffer('01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000', 'hex'); - var am1 = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var message = {}; - am1.transactionOutputHandler = function(messages) { - messages[address] = message; - }; - am1.transactionEventHandler = sinon.stub(); - am1.balanceEventHandler = sinon.stub(); - am1.transactionHandler({ - buffer: txBuf - }, function(err) { - if (err) { - throw err; - } - am1.transactionEventHandler.callCount.should.equal(1); - am1.balanceEventHandler.callCount.should.equal(1); - done(); - }); - - }); - }); - - describe('#blockHandler', function() { - var am; - var testBlock = bitcore.Block.fromString(blockData); - - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.node.network = Networks.livenet; - }); - - it('should create the correct operations when updating/adding outputs', function(done) { - var block = { - __height: 345003, - header: { - timestamp: 1424836934 - }, - transactions: testBlock.transactions.slice(0, 8) - }; - - am.blockHandler(block, true, function(err, operations) { - should.not.exist(err); - operations.length.should.equal(151); - operations[0].type.should.equal('put'); - operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b0100000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); - operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); - operations[3].type.should.equal('put'); - operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e0100000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); - operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); - operations[4].type.should.equal('put'); - operations[4].key.toString('hex').should.equal('053d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); - operations[4].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); - operations[121].type.should.equal('put'); - operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c75280100000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); - operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); - done(); - }); - }); - it('should create the correct operations when removing outputs', function(done) { - var block = { - __height: 345003, - header: { - timestamp: 1424836934 - }, - transactions: testBlock.transactions.slice(0, 8) - }; - am.blockHandler(block, false, function(err, operations) { - should.not.exist(err); - operations.length.should.equal(151); - operations[0].type.should.equal('del'); - operations[0].key.toString('hex').should.equal('0202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b0100000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); - operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); - operations[3].type.should.equal('del'); - operations[3].key.toString('hex').should.equal('03fdbd324b28ea69e49c998816407dc055fb81d06e0100000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); - operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); - operations[121].type.should.equal('del'); - operations[121].key.toString('hex').should.equal('029780ccd5356e2acc0ee439ee04e0fe69426c75280100000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); - operations[121].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); - done(); - }); - }); - it('should continue if output script is null', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode, - }); - - var block = { - __height: 345003, - header: { - timestamp: 1424836934 - }, - transactions: [ - { - id: '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', - inputs: [], - outputs: [ - { - script: null, - satoshis: 1000, - } - ], - isCoinbase: sinon.stub().returns(false) - } - ] - }; - - am.blockHandler(block, false, function(err, operations) { - should.not.exist(err); - operations.length.should.equal(0); - done(); - }); - }); - it('will call event handlers', function() { - var testBlock = bitcore.Block.fromString(blockData); - var db = {}; - var testnode = { - datadir: 'testdir', - db: db, - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.transactionEventHandler = sinon.spy(); - am.balanceEventHandler = sinon.spy(); - - var block = { - __height: 345003, - header: { - timestamp: 1424836934 - }, - transactions: testBlock.transactions.slice(0, 8) - }; - - am.blockHandler( - block, - true, - function(err) { - if (err) { - throw err; - } - am.transactionEventHandler.callCount.should.equal(11); - am.balanceEventHandler.callCount.should.equal(11); - } - ); - }); - }); - - describe('#transactionEventHandler', function() { - it('will emit a transaction if there is a subscriber', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - am.subscriptions['address/transaction'] = {}; - am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] = [emitter]; - var block = { - __height: 0, - timestamp: new Date() - }; - var tx = {}; - emitter.on('address/transaction', function(obj) { - obj.address.toString().should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - obj.tx.should.equal(tx); - obj.timestamp.should.equal(block.timestamp); - obj.height.should.equal(block.__height); - obj.outputIndexes.should.deep.equal([1]); - done(); - }); - am.transactionEventHandler({ - addressInfo: { - hashHex: address.hashBuffer.toString('hex'), - hashBuffer: address.hashBuffer, - addressType: address.type - }, - height: block.__height, - timestamp: block.timestamp, - outputIndexes: [1], - tx: tx - }); - }); - }); - - describe('#balanceEventHandler', function() { - it('will emit a balance if there is a subscriber', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] = [emitter]; - var block = {}; - var balance = 1000; - am.getBalance = sinon.stub().callsArgWith(2, null, balance); - emitter.on('address/balance', function(a, bal, b) { - a.toString().should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - bal.should.equal(balance); - b.should.equal(block); - done(); - }); - am.balanceEventHandler(block, { - hashHex: address.hashBuffer.toString('hex'), - hashBuffer: address.hashBuffer, - addressType: address.type - }); - }); - }); - - describe('#subscribe', function() { - it('will add emitters to the subscribers array (transaction)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - var name = 'address/transaction'; - am.subscribe(name, emitter, [address]); - am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter]); - - var address2 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); - am.subscribe(name, emitter, [address2]); - am.subscriptions['address/transaction'][address2.hashBuffer.toString('hex')] - .should.deep.equal([emitter]); - - var emitter2 = new EventEmitter(); - am.subscribe(name, emitter2, [address]); - am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter, emitter2]); - }); - it('will add an emitter to the subscribers array (balance)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var name = 'address/balance'; - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - am.subscribe(name, emitter, [address]); - am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter]); - - var address2 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); - am.subscribe(name, emitter, [address2]); - am.subscriptions['address/balance'][address2.hashBuffer.toString('hex')] - .should.deep.equal([emitter]); - - var emitter2 = new EventEmitter(); - am.subscribe(name, emitter2, [address]); - am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter, emitter2]); - }); - }); - - describe('#unsubscribe', function() { - it('will remove emitter from subscribers array (transaction)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var emitter2 = new EventEmitter(); - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] = [emitter, emitter2]; - var name = 'address/transaction'; - am.unsubscribe(name, emitter, [address]); - am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter2]); - }); - it('will remove emitter from subscribers array (balance)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var emitter2 = new EventEmitter(); - var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - var name = 'address/balance'; - am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] = [emitter, emitter2]; - am.unsubscribe(name, emitter, [address]); - am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] - .should.deep.equal([emitter2]); - }); - it('should unsubscribe from all addresses if no addresses are specified', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var emitter = new EventEmitter(); - var emitter2 = new EventEmitter(); - var address1 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); - var hashHex1 = address1.hashBuffer.toString('hex'); - var address2 = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); - var hashHex2 = address2.hashBuffer.toString('hex'); - am.subscriptions['address/balance'][hashHex1] = [emitter, emitter2]; - am.subscriptions['address/balance'][hashHex2] = [emitter2, emitter]; - am.unsubscribe('address/balance', emitter); - am.subscriptions['address/balance'][hashHex1].should.deep.equal([emitter2]); - am.subscriptions['address/balance'][hashHex2].should.deep.equal([emitter2]); - }); - }); - - describe('#getBalance', function() { - it('should sum up the unspent outputs', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var outputs = [ - {satoshis: 1000}, {satoshis: 2000}, {satoshis: 3000} - ]; - am.getUnspentOutputs = sinon.stub().callsArgWith(2, null, outputs); - am.getBalance('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N', false, function(err, balance) { - should.not.exist(err); - balance.should.equal(6000); - done(); - }); - }); - - it('will handle error from unspent outputs', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.getUnspentOutputs = sinon.stub().callsArgWith(2, new Error('error')); - am.getBalance('someaddress', false, function(err) { - should.exist(err); - err.message.should.equal('error'); - done(); - }); - }); - - }); - - describe('#createInputsStream', function() { - it('transform stream from buffer into object', function(done) { - var testnode = { - network: Networks.livenet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 157 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - addressService.createInputsDBStream = sinon.stub().returns(streamStub); - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createInputsStream(address, {}); - testStream.once('data', function(data) { - data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); - data.hashType.should.equal('pubkeyhash'); - data.txid.should.equal('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df72'); - data.inputIndex.should.equal(1); - data.height.should.equal(157); - data.confirmations.should.equal(1); - done(); - }); - streamStub.emit('data', { - key: new Buffer('030b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d786cfeae288d74aaf9f51f215f9882e7bd7bc18af7a550683c4d7c6962f6372900000004', 'hex'), - value: new Buffer('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df7200000001', 'hex') - }); - streamStub.emit('end'); - }); - }); - - describe('#createInputsDBStream', function() { - it('will stream all keys', function() { - var streamStub = sinon.stub().returns({}); - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - store: { - createReadStream: streamStub - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = {}; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createInputsDBStream(address, options); - should.exist(testStream); - streamStub.callCount.should.equal(1); - var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; - // The expected "lt" value should be one value above the start value, due - // to the keys having additional data following it and can't be "equal". - var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; - streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); - streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); - }); - it('will stream keys based on a range of block heights', function() { - var streamStub = sinon.stub().returns({}); - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - store: { - createReadStream: streamStub - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = { - start: 1, - end: 0 - }; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createInputsDBStream(address, options); - should.exist(testStream); - streamStub.callCount.should.equal(1); - var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; - // The expected "lt" value should be one value above the start value, due - // to the keys having additional data following it and can't be "equal". - var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; - streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); - streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); - }); - }); - - describe('#getInputs', function() { - var am; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - var db = { - tip: { - __height: 1 - } - }; - var testnode = { - network: Networks.livenet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - }); - - it('will add mempool inputs on close', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var db = { - store: { - createReadStream: sinon.stub().returns(testStream) - }, - tip: { - __height: 10 - } - }; - var testnode = { - network: Networks.livenet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var args = { - start: 15, - end: 12, - queryMempool: true - }; - am._getInputsMempool = sinon.stub().callsArgWith(3, null, { - address: address, - hashType: 'pubkeyhash', - height: -1, - confirmations: 0 - }); - am.getInputs(address, args, function(err, inputs) { - should.not.exist(err); - inputs.length.should.equal(1); - inputs[0].address.should.equal(address); - inputs[0].height.should.equal(-1); - done(); - }); - testStream.push(null); - }); - it('will get inputs for an address and timestamp', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var args = { - start: 15, - end: 12, - queryMempool: true - }; - var createReadStreamCallCount = 0; - am.node.services.db.store = { - createReadStream: function(ops) { - var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, - hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gt.toString('hex').should.equal(gt.toString('hex')); - var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, - hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lt.toString('hex').should.equal(lt.toString('hex')); - createReadStreamCallCount++; - return testStream; - } - }; - am.node.services.bitcoind = { - getMempoolInputs: sinon.stub().returns([]) - }; - am._getInputsMempool = sinon.stub().callsArgWith(3, null, []); - am.getInputs(address, args, function(err, inputs) { - should.not.exist(err); - inputs.length.should.equal(1); - inputs[0].address.should.equal(address); - inputs[0].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); - inputs[0].inputIndex.should.equal(0); - inputs[0].height.should.equal(15); - done(); - }); - createReadStreamCallCount.should.equal(1); - var data = { - key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), - value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') - }; - testStream.emit('data', data); - testStream.push(null); - }); - it('should get inputs for address', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var args = { - queryMempool: true - }; - var createReadStreamCallCount = 0; - am.node.services.db.store = { - createReadStream: function(ops) { - var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000000', 'hex')]); - ops.gt.toString('hex').should.equal(gt.toString('hex')); - var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ffffffffff', 'hex')]); - ops.lt.toString('hex').should.equal(lt.toString('hex')); - createReadStreamCallCount++; - return testStream; - } - }; - am.node.services.bitcoind = { - getMempoolInputs: sinon.stub().returns([]) - }; - am.getInputs(address, args, function(err, inputs) { - should.not.exist(err); - inputs.length.should.equal(1); - inputs[0].address.should.equal(address); - inputs[0].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); - inputs[0].inputIndex.should.equal(0); - inputs[0].height.should.equal(15); - done(); - }); - createReadStreamCallCount.should.equal(1); - var data = { - key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), - value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') - }; - testStream.emit('data', data); - testStream.push(null); - }); - it('should give an error if the readstream has an error', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - am.node.services.db.store = { - createReadStream: sinon.stub().returns(testStream) - }; - - am.getInputs(address, {}, function(err, outputs) { - should.exist(err); - err.message.should.equal('readstreamerror'); - done(); - }); - - testStream.emit('error', new Error('readstreamerror')); - setImmediate(function() { - testStream.push(null); - }); - }); - - }); - - describe('#_getInputsMempool', function() { - var am; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - var db = { - tip: { - __height: 1 - } - }; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - }); - it('it will handle error', function(done) { - var testStream = new EventEmitter(); - am.mempoolIndex = {}; - am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - - am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { - should.exist(err); - err.message.should.equal('readstreamerror'); - done(); - }); - - testStream.emit('error', new Error('readstreamerror')); - setImmediate(function() { - testStream.emit('close'); - }); - }); - it('it will parse data', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - am.mempoolIndex = {}; - am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - - var nowTime = new Date().getTime(); - - am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, inputs) { - should.not.exist(err); - inputs.length.should.equal(1); - var input = inputs[0]; - input.address.should.equal(address); - input.txid.should.equal(txid); - input.hashType.should.equal('pubkeyhash'); - input.hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - input.inputIndex.should.equal(5); - input.height.should.equal(-1); - input.confirmations.should.equal(0); - input.timestamp.should.equal(nowTime); - done(); - }); - - var txid = '5d32f0fff6871c377e00c16f48ebb5e89c723d0b9dd25f68fdda70c3392bee61'; - var inputIndex = 5; - var inputIndexBuffer = new Buffer(4); - var timestampBuffer = new Buffer(new Array(8)); - timestampBuffer.writeDoubleBE(nowTime); - inputIndexBuffer.writeUInt32BE(inputIndex); - var valueData = Buffer.concat([ - new Buffer(txid, 'hex'), - inputIndexBuffer, - timestampBuffer - ]); - // Note: key is not used currently - testStream.emit('data', { - value: valueData - }); - testStream.emit('close'); - }); - }); - - describe('#_getSpentMempool', function() { - it('will decode data from the database', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.mempoolIndex = {}; - var mempoolValue = Buffer.concat([ - new Buffer('85630d684f1f414264f88a31bddfc79dd0c00659330dcdc393b321c121f4078b', 'hex'), - new Buffer('00000003', 'hex') - ]); - am.mempoolIndex.get = sinon.stub().callsArgWith(1, null, mempoolValue); - var prevTxIdBuffer = new Buffer('e7888264d286be2da26b0a4dbd2fc5c9ece82b3e40e6791b137e4155b6da8981', 'hex'); - var outputIndex = 1; - var outputIndexBuffer = new Buffer('00000001', 'hex'); - var expectedKey = Buffer.concat([ - new Buffer('03', 'hex'), - prevTxIdBuffer, - outputIndexBuffer - ]).toString('hex'); - am._getSpentMempool(prevTxIdBuffer, outputIndex, function(err, value) { - if (err) { - throw err; - } - am.mempoolIndex.get.args[0][0].toString('hex').should.equal(expectedKey); - value.inputTxId.should.equal('85630d684f1f414264f88a31bddfc79dd0c00659330dcdc393b321c121f4078b'); - value.inputIndex.should.equal(3); - }); - }); - it('handle error from levelup', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.mempoolIndex = {}; - am.mempoolIndex.get = sinon.stub().callsArgWith(1, new Error('test')); - var prevTxIdBuffer = new Buffer('e7888264d286be2da26b0a4dbd2fc5c9ece82b3e40e6791b137e4155b6da8981', 'hex'); - var outputIndex = 1; - am._getSpentMempool(prevTxIdBuffer, outputIndex, function(err) { - err.message.should.equal('test'); - }); - }); - }); - - describe('#createOutputsStream', function() { - it('transform stream from buffer into object', function(done) { - var testnode = { - network: Networks.livenet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - tip: { - __height: 157 - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - addressService.createOutputsDBStream = sinon.stub().returns(streamStub); - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createOutputsStream(address, {}); - testStream.once('data', function(data) { - data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); - data.hashType.should.equal('pubkeyhash'); - data.txid.should.equal('4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f9'); - data.outputIndex.should.equal(2); - data.height.should.equal(157); - data.satoshis.should.equal(10000); - data.script.toString('hex').should.equal('76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'); - data.confirmations.should.equal(1); - done(); - }); - streamStub.emit('data', { - key: new Buffer('020b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f900000002', 'hex'), - value: new Buffer('40c388000000000076a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', 'hex') - }); - streamStub.emit('end'); - }); - }); - - describe('#createOutputsDBStream', function() { - it('will stream all keys', function() { - var streamStub = sinon.stub().returns({}); - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - store: { - createReadStream: streamStub - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = {}; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createOutputsDBStream(address, options); - should.exist(testStream); - streamStub.callCount.should.equal(1); - var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; - // The expected "lt" value should be one value above the start value, due - // to the keys having additional data following it and can't be "equal". - var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; - streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); - streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); - }); - it('will stream keys based on a range of block heights', function() { - var streamStub = sinon.stub().returns({}); - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - }, - db: { - store: { - createReadStream: streamStub - } - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = { - start: 1, - end: 0 - }; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var testStream = addressService.createOutputsDBStream(address, options); - should.exist(testStream); - streamStub.callCount.should.equal(1); - var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; - // The expected "lt" value should be one value above the start value, due - // to the keys having additional data following it and can't be "equal". - var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; - streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); - streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); - }); - }); - - describe('#getOutputs', function() { - var am; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - var db = { - tip: { - __height: 1 - } - }; - var testnode = { - network: Networks.livenet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - var options = { - queryMempool: true - }; - - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - }); - - it('will get outputs for an address and timestamp', function(done) { - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var args = { - start: 15, - end: 12, - queryMempool: true - }; - var createReadStreamCallCount = 0; - am.node.services.db.store = { - createReadStream: function(ops) { - var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gt.toString('hex').should.equal(gt.toString('hex')); - var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lt.toString('hex').should.equal(lt.toString('hex')); - createReadStreamCallCount++; - return testStream; - } - }; - am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); - am.getOutputs(address, args, function(err, outputs) { - should.not.exist(err); - outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - outputs[0].outputIndex.should.equal(1); - outputs[0].satoshis.should.equal(4527773864); - outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); - outputs[0].height.should.equal(15); - done(); - }); - createReadStreamCallCount.should.equal(1); - var data = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68701000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), - value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') - }; - testStream.emit('data', data); - testStream.push(null); - }); - - it('should get outputs for an address', function(done) { - var readStream1 = new stream.Readable(); - readStream1._read = function() { /* do nothing */ }; - am.node.services.db.store = { - createReadStream: sinon.stub().returns(readStream1) - }; - - am._getOutputsMempool = sinon.stub().callsArgWith(3, null, [ - { - address: address, - height: -1, - hashType: 'pubkeyhash', - confirmations: 0, - txid: 'aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371', - satoshis: 307627737, - script: '76a914f6db95c81dea3d10f0ff8d890927751bf7b203c188ac', - outputIndex: 0 - } - ]); - - am.getOutputs(address, options, function(err, outputs) { - should.not.exist(err); - outputs.length.should.equal(3); - outputs[0].address.should.equal(address); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); - outputs[0].outputIndex.should.equal(1); - outputs[0].satoshis.should.equal(4527773864); - outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); - outputs[0].height.should.equal(345000); - outputs[1].address.should.equal(address); - outputs[1].hashType.should.equal('pubkeyhash'); - outputs[1].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); - outputs[1].outputIndex.should.equal(2); - outputs[1].satoshis.should.equal(10000); - outputs[1].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); - outputs[1].height.should.equal(345004); - outputs[2].address.should.equal(address); - outputs[2].hashType.should.equal('pubkeyhash'); - outputs[2].txid.should.equal('aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371'); - outputs[2].script.should.equal('76a914f6db95c81dea3d10f0ff8d890927751bf7b203c188ac'); - outputs[2].height.should.equal(-1); - outputs[2].confirmations.should.equal(0); - done(); - }); - - var data1 = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b6870100000543a8125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), - value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') - }; - - var data2 = { - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b6870100000543ac3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000002', 'hex'), - value: new Buffer('40c388000000000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') - }; - - readStream1.emit('data', data1); - readStream1.emit('data', data2); - readStream1.push(null); - }); - - it('should give an error if the readstream has an error', function(done) { - var readStream2 = new stream.Readable(); - readStream2._read = function() { /* do nothing */ }; - am.node.services.db.store = { - createReadStream: sinon.stub().returns(readStream2) - }; - - am.getOutputs(address, options, function(err, outputs) { - should.exist(err); - err.message.should.equal('readstreamerror'); - done(); - }); - - readStream2.emit('error', new Error('readstreamerror')); - setImmediate(function() { - readStream2.push(null); - }); - }); - - it('should print outputs for a p2sh address', function(done) { - // This address has the redeemScript 0x038a213afdfc551fc658e9a2a58a86e98d69b687, - // which is the same as the pkhash for the address 1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W. - // See https://github.com/bitpay/bitcore-node/issues/377 - var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; - var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var args = { - start: 15, - end: 12, - queryMempool: true - }; - var createReadStreamCallCount = 0; - am.node.services.db.store = { - createReadStream: function(ops) { - var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gt.toString('hex').should.equal(gt.toString('hex')); - var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lt.toString('hex').should.equal(lt.toString('hex')); - createReadStreamCallCount++; - return testStream; - } - }; - am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); - am.getOutputs(address, args, function(err, outputs) { - should.not.exist(err); - outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); - outputs[0].hashType.should.equal('scripthash'); - outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - outputs[0].outputIndex.should.equal(1); - outputs[0].satoshis.should.equal(4527773864); - outputs[0].script.should.equal('a914038a213afdfc551fc658e9a2a58a86e98d69b68787'); - outputs[0].height.should.equal(15); - done(); - }); - createReadStreamCallCount.should.equal(1); - var data = { - // note '68702', '02' meaning p2sh redeemScript, not p2pkh - // value is also the p2sh script, not p2pkh - key: new Buffer('02038a213afdfc551fc658e9a2a58a86e98d69b68702000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), - value: new Buffer('41f0de058a800000a914038a213afdfc551fc658e9a2a58a86e98d69b68787', 'hex') - }; - testStream.emit('data', data); - testStream.push(null); - }); - - it('should not print outputs for a p2pkh address, if the output was sent to a p2sh redeemScript', function(done) { - // This address has the redeemScript 0x038a213afdfc551fc658e9a2a58a86e98d69b687, - // which is the same as the pkhash for the address 1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W. - // See https://github.com/bitpay/bitcore-node/issues/377 - var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; - var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; - var testStream = new stream.Readable(); - testStream._read = function() { /* do nothing */ }; - var args = { - start: 15, - end: 12, - queryMempool: true - }; - var createReadStreamCallCount = 0; - - // Verifying that the db query is looking for a redeemScript, *not* a p2pkh - am.node.services.db.store = { - createReadStream: function(ops) { - var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gt.toString('hex').should.equal(gt.toString('hex')); - var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lt.toString('hex').should.equal(lt.toString('hex')); - createReadStreamCallCount++; - return testStream; - } - }; - am._getOutputsMempool = sinon.stub().callsArgWith(3, null, []); - am.getOutputs(address, args, function(err, outputs) { - should.not.exist(err); - outputs.length.should.equal(0); - done(); - }); - createReadStreamCallCount.should.equal(1); - testStream.push(null); - }); - }); - - describe('#_getOutputsMempool', function() { - var am; - var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; - var db = { - tip: { - __height: 1 - } - }; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - }); - it('it will handle error', function(done) { - var testStream = new EventEmitter(); - am.mempoolIndex = {}; - am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getOutputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { - should.exist(err); - err.message.should.equal('readstreamerror'); - done(); - }); - testStream.emit('error', new Error('readstreamerror')); - setImmediate(function() { - testStream.emit('close'); - }); - }); - it('it will parse data', function(done) { - var testStream = new EventEmitter(); - am.mempoolIndex = {}; - am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - - am._getOutputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { - if (err) { - throw err; - } - outputs.length.should.equal(1); - var output = outputs[0]; - output.address.should.equal(address); - output.hashType.should.equal('pubkeyhash'); - output.txid.should.equal(txid); - output.outputIndex.should.equal(outputIndex); - output.height.should.equal(-1); - output.satoshis.should.equal(3); - output.script.should.equal('ac'); - output.timestamp.should.equal(1452696715750); - output.confirmations.should.equal(0); - done(); - }); - - var txid = '5d32f0fff6871c377e00c16f48ebb5e89c723d0b9dd25f68fdda70c3392bee61'; - var txidBuffer = new Buffer(txid, 'hex'); - var outputIndex = 3; - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var keyData = Buffer.concat([ - constants.MEMPREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - txidBuffer, - outputIndexBuffer - ]); - - var valueData = Buffer.concat([ - new Buffer('4008000000000000', 'hex'), - new Buffer('427523b78c1e6000', 'hex'), - new Buffer('ac', 'hex') - ]); - - // Note: key is not used currently - testStream.emit('data', { - key: keyData, - value: valueData - }); - setImmediate(function() { - testStream.emit('close'); - }); - }); - }); - - describe('#getUnspentOutputs', function() { - it('should concatenate utxos for multiple addresses, even those with none found', function(done) { - var addresses = { - 'addr1': ['utxo1', 'utxo2'], - 'addr2': new errors.NoOutputs(), - 'addr3': ['utxo3'] - }; - - var db = {}; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.getUnspentOutputsForAddress = function(address, queryMempool, callback) { - var result = addresses[address]; - if(result instanceof Error) { - return callback(result); - } else { - return callback(null, result); - } - }; - - am.getUnspentOutputs(['addr1', 'addr2', 'addr3'], true, function(err, utxos) { - should.not.exist(err); - utxos.should.deep.equal(['utxo1', 'utxo2', 'utxo3']); - done(); - }); - }); - it('should give an error if an error occurred', function(done) { - var addresses = { - 'addr1': ['utxo1', 'utxo2'], - 'addr2': new Error('weird error'), - 'addr3': ['utxo3'] - }; - - var db = {}; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - db: db, - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.getUnspentOutputsForAddress = function(address, queryMempool, callback) { - var result = addresses[address]; - if(result instanceof Error) { - return callback(result); - } else { - return callback(null, result); - } - }; - - am.getUnspentOutputs(['addr1', 'addr2', 'addr3'], true, function(err, utxos) { - should.exist(err); - err.message.should.equal('weird error'); - done(); - }); - }); - - it('should also work for a single address', function(done) { - var addresses = { - 'addr1': ['utxo1', 'utxo2'], - 'addr2': new Error('weird error'), - 'addr3': ['utxo3'] - }; - - var db = {}; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - db: db, - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.getUnspentOutputsForAddress = function(address, queryMempool, callback) { - var result = addresses[address]; - if(result instanceof Error) { - return callback(result); - } else { - return callback(null, result); - } - }; - - am.getUnspentOutputs('addr1', true, function(err, utxos) { - should.not.exist(err); - utxos.should.deep.equal(['utxo1', 'utxo2']); - done(); - }); - }); - }); - - describe('#getUnspentOutputsForAddress', function() { - it('should filter out spent outputs', function(done) { - var outputs = [ - { - satoshis: 1000, - spent: false, - }, - { - satoshis: 2000, - spent: true - }, - { - satoshis: 3000, - spent: false - } - ]; - var i = 0; - - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.getOutputs = sinon.stub().callsArgWith(2, null, outputs); - am.isUnspent = function(output, options, callback) { - callback(!outputs[i].spent); - i++; - }; - - am.getUnspentOutputsForAddress('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', false, function(err, outputs) { - should.not.exist(err); - outputs.length.should.equal(2); - outputs[0].satoshis.should.equal(1000); - outputs[1].satoshis.should.equal(3000); - done(); - }); - }); - it('should handle an error from getOutputs', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.getOutputs = sinon.stub().callsArgWith(2, new Error('error')); - am.getUnspentOutputsForAddress('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', false, function(err, outputs) { - should.exist(err); - err.message.should.equal('error'); - done(); - }); - }); - it('should handle when there are no outputs', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.getOutputs = sinon.stub().callsArgWith(2, null, []); - am.getUnspentOutputsForAddress('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', false, function(err, outputs) { - should.exist(err); - err.should.be.instanceof(errors.NoOutputs); - outputs.length.should.equal(0); - done(); - }); - }); - }); - - describe('#isUnspent', function() { - var am; - - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - }); - - it('should give true when isSpent() gives false', function(done) { - am.isSpent = sinon.stub().callsArgWith(2, false); - am.isUnspent('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', {}, function(unspent) { - unspent.should.equal(true); - done(); - }); - }); - - it('should give false when isSpent() gives true', function(done) { - am.isSpent = sinon.stub().callsArgWith(2, true); - am.isUnspent('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', {},function(unspent) { - unspent.should.equal(false); - done(); - }); - }); - - it('should give false when isSpent() returns an error', function(done) { - am.isSpent = sinon.stub().callsArgWith(2, new Error('error')); - am.isUnspent('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', {}, function(unspent) { - unspent.should.equal(false); - done(); - }); - }); - }); - - describe('#isSpent', function() { - var db = {}; - var testnode = { - network: Networks.testnet, - datadir: 'testdir', - db: db, - services: { - bitcoind: { - on: sinon.stub() - } - } - }; - it('should give true if bitcoind.isSpent gives true (with output info)', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var isSpent = sinon.stub().returns(true); - am.node.services.bitcoind = { - isSpent: isSpent, - on: sinon.stub() - }; - var output = { - txid: '4228d3f41051f914f71a1dcbbe4098e29a07cc2672fdadab0763d88ffd6ffa57', - outputIndex: 3 - }; - am.isSpent(output, {}, function(spent) { - isSpent.callCount.should.equal(1); - isSpent.args[0][0].should.equal(output.txid); - isSpent.args[0][1].should.equal(output.outputIndex); - spent.should.equal(true); - done(); - }); - }); - it('should give true if bitcoind.isSpent gives true (with input)', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var isSpent = sinon.stub().returns(true); - am.node.services.bitcoind = { - isSpent: isSpent, - on: sinon.stub() - }; - var txid = '4228d3f41051f914f71a1dcbbe4098e29a07cc2672fdadab0763d88ffd6ffa57'; - var output = { - prevTxId: new Buffer(txid, 'hex'), - outputIndex: 4 - }; - am.isSpent(output, {}, function(spent) { - isSpent.callCount.should.equal(1); - isSpent.args[0][0].should.equal(txid); - isSpent.args[0][1].should.equal(output.outputIndex); - spent.should.equal(true); - done(); - }); - }); - it('should give true if bitcoind.isSpent is false and mempoolSpentIndex is true', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.node.services.bitcoind = { - isSpent: sinon.stub().returns(false), - on: sinon.stub() - }; - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var outputIndex = 0; - var output = { - prevTxId: new Buffer(txid, 'hex'), - outputIndex: outputIndex - }; - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var spentKey = Buffer.concat([ - new Buffer(txid, 'hex'), - outputIndexBuffer - ]).toString('binary'); - am.mempoolSpentIndex[spentKey] = true; - am.isSpent(output, {queryMempool: true}, function(spent) { - spent.should.equal(true); - done(); - }); - }); - it('should give false if spent in mempool with queryMempool set to false', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.node.services.bitcoind = { - isSpent: sinon.stub().returns(false), - on: sinon.stub() - }; - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var outputIndex = 0; - var output = { - prevTxId: new Buffer(txid, 'hex'), - outputIndex: outputIndex - }; - var spentKey = [txid, outputIndex].join('-'); - am.mempoolSpentIndex[spentKey] = new Buffer(5); - am.isSpent(output, {queryMempool: false}, function(spent) { - spent.should.equal(false); - done(); - }); - }); - it('default to querying the mempool', function(done) { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - am.node.services.bitcoind = { - isSpent: sinon.stub().returns(false), - on: sinon.stub() - }; - var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - var outputIndex = 0; - var output = { - prevTxId: txidBuffer, - outputIndex: outputIndex - }; - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var spentKey = Buffer.concat([ - txidBuffer, - outputIndexBuffer - ]).toString('binary'); - am.mempoolSpentIndex[spentKey] = true; - am.isSpent(output, {}, function(spent) { - spent.should.equal(true); - done(); - }); - }); - }); - - describe('#getAddressHistory', function() { - it('will call get on address history instance', function(done) { - function TestAddressHistory(args) { - args.node.should.equal(mocknode); - args.addresses.should.deep.equal([]); - args.options.should.deep.equal({}); - } - TestAddressHistory.prototype.get = sinon.stub().callsArg(0); - var TestAddressService = proxyquire('../../../lib/services/address', { - './history': TestAddressHistory - }); - var am = new TestAddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.getAddressHistory([], {}, function(err, history) { - TestAddressHistory.prototype.get.callCount.should.equal(1); - done(); - }); - }); - }); - describe('#updateMempoolIndex/#removeMempoolIndex', function() { - var am; - var tx = Transaction().fromBuffer(txBuf); - var clock; - - beforeEach(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - clock = sinon.useFakeTimers(); - }); - - afterEach(function() { - clock.restore(); - }); - - it('will update the input and output indexes', function() { - am.mempoolIndex = {}; - am.mempoolIndex.batch = function(operations, callback) { - callback.should.be.a('function'); - Object.keys(am.mempoolSpentIndex).length.should.equal(14); - Object.keys(am.mempoolAddressIndex).length.should.equal(5); - _.values(am.mempoolAddressIndex).should.deep.equal([1,1,12,1,1]); - for (var i = 0; i < operations.length; i++) { - operations[i].type.should.equal('put'); - } - var nowTime = new Date().getTime(); - var nowTimeBuffer = new Buffer(8); - nowTimeBuffer.writeDoubleBE(nowTime); - var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002' + - nowTimeBuffer.toString('hex'); - operations[7].value.toString('hex').should.equal(expectedValue); - var matches = 0; - - - for (var j = 0; j < operations.length; j++) { - var match = Buffer.concat([ - constants.MEMPREFIXES.SPENTS, - bitcore.Address('1JT7KDYwT9JY9o2vyqcKNSJgTWeKfV3ui8').hashBuffer - ]).toString('hex'); - - if (operations[j].key.slice(0, 21).toString('hex') === match) { - matches++; - } - } - matches.should.equal(12); - }; - am.updateMempoolIndex(tx, true); - }); - - it('will remove the input and output indexes', function() { - am.mempoolIndex = {}; - am.mempoolIndex.batch = function(operations, callback) { - callback.should.be.a('function'); - Object.keys(am.mempoolSpentIndex).length.should.equal(0); - for (var i = 0; i < operations.length; i++) { - operations[i].type.should.equal('del'); - } - Object.keys(am.mempoolAddressIndex).length.should.equal(0); - }; - am.updateMempoolIndex(tx, false); - }); - - }); - - describe('#getAddressSummary', function() { - var clock; - beforeEach(function() { - clock = sinon.useFakeTimers(); - sinon.stub(log, 'warn'); - }); - afterEach(function() { - clock.restore(); - log.warn.restore(); - }); - it('will handle error from _getAddressConfirmedSummary', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, new Error('test')); - addressService.getAddressSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - it('will handle error from _getAddressMempoolSummary', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - addressService._getAddressConfirmedSummary = sinon.stub().callsArg(2); - addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(2, new Error('test2')); - addressService.getAddressSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test2'); - done(); - }); - }); - it('will pass cache and summary between functions correctly', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = {}; - var summary = {}; - addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); - addressService.getAddressSummary(address, options, function(err, sum) { - addressService._getAddressConfirmedSummary.callCount.should.equal(1); - addressService._getAddressMempoolSummary.callCount.should.equal(1); - addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); - addressService._setAndSortTxidsFromAppearanceIds.callCount.should.equal(1); - addressService._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(cache); - addressService._transformAddressSummaryFromResult.callCount.should.equal(1); - addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); - sum.should.equal(summary); - done(); - }); - }); - it('will log if there is a slow query', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var addressService = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; - var options = {}; - var cache = {}; - var summary = {}; - addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); - addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); - addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); - addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); - addressService.getAddressSummary(address, options, function() { - log.warn.callCount.should.equal(1); - done(); - }); - clock.tick(6000); - }); - }); - - describe('#_getAddressConfirmedSummary', function() { - it('will pass arguments correctly', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = {}; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); - as._getAddressConfirmedSummary(address, options, function(err) { - if (err) { - return done(err); - } - var expectedResult = { - appearanceIds: {}, - totalReceived: 0, - balance: 0, - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - }; - as._getAddressConfirmedInputsSummary.args[0][0].should.equal(address); - as._getAddressConfirmedInputsSummary.args[0][1].should.deep.equal(expectedResult); - as._getAddressConfirmedInputsSummary.args[0][2].should.deep.equal(options); - as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); - as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); - as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); - done(); - }); - }); - it('will pass error correctly (inputs)', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = {}; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, new Error('test')); - as._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - it('will pass error correctly (outputs)', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = {}; - as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); - as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, new Error('test')); - as._getAddressConfirmedSummary(address, options, function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - }); - - describe('#_getAddressConfirmedInputsSummary', function() { - it('will stream inputs and collect txids', function(done) { - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = { - appearanceIds: {} - }; - var options = {}; - var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - as.createInputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedInputsSummary(address, result, options, function(err, result) { - if (err) { - return done(err); - } - result.appearanceIds[txid].should.equal(10); - done(); - }); - - streamStub.emit('data', { - txid: txid, - height: 10 - }); - streamStub.push(null); - }); - it('handle stream error', function(done) { - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var cache = {}; - var options = {}; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - as.createInputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - - streamStub.emit('error', new Error('test')); - streamStub.push(null); - }); - }); - - describe('#_getAddressConfirmedOutputsSummary', function() { - it('will stream inputs and collect txids', function(done) { - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub(), - isSpent: sinon.stub().returns(false) - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - }; - - var options = { - queryMempool: true - }; - var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - - as.createOutputsStream = sinon.stub().returns(streamStub); - - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); - as.mempoolSpentIndex[spentIndexSyncKey] = true; - - as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { - if (err) { - return done(err); - } - result.appearanceIds[txid].should.equal(10); - result.balance.should.equal(1000); - result.totalReceived.should.equal(1000); - result.unconfirmedBalance.should.equal(-1000); - done(); - }); - - streamStub.emit('data', { - txid: txid, - height: 10, - outputIndex: 2, - satoshis: 1000 - }); - streamStub.push(null); - }); - it('handle stream error', function(done) { - var streamStub = new stream.Readable(); - streamStub._read = function() { /* do nothing */ }; - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, - balance: 0, - totalReceived: 0, - unconfirmedBalance: 0 - }; - - var options = {}; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - as.createOutputsStream = sinon.stub().returns(streamStub); - as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - - streamStub.emit('error', new Error('test')); - streamStub.push(null); - }); - }); - - describe('#_setAndSortTxidsFromAppearanceIds', function() { - it('will sort correctly', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var result = { - appearanceIds: { - '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, - '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, - '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 - }, - unconfirmedAppearanceIds: { - 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, - 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964, - 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, - 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, - 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 - } - }; - as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { - if (err) { - return done(err); - } - should.exist(result.txids); - result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); - result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); - result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); - result.unconfirmedTxids[0].should.equal('f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345'); - result.unconfirmedTxids[1].should.equal('f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9'); - result.unconfirmedTxids[2].should.equal('edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a'); - result.unconfirmedTxids[3].should.equal('ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693'); - result.unconfirmedTxids[4].should.equal('ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681'); - done(); - }); - }); - }); - - - describe('#_updateAddressIndex', function() { - it('should add using 2 keys', function() { - var as = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - - _.values(as.mempoolAddressIndex).should.deep.equal([]); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index2', true); - as._updateAddressIndex('index2', true); - as.mempoolAddressIndex.should.deep.equal({ - "index1": 4, - "index2": 2 - }); - }); - - it('should add/remove using 2 keys', function() { - var as = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - _.values(as.mempoolAddressIndex).should.deep.equal([]); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', true); - as._updateAddressIndex('index1', false); - - as._updateAddressIndex('index2', true); - as._updateAddressIndex('index2', true); - as._updateAddressIndex('index2', false); - as._updateAddressIndex('index2', false); - as.mempoolAddressIndex.should.deep.equal({ - "index1": 3 - }); - as._updateAddressIndex('index2', false); - as.mempoolAddressIndex.should.deep.equal({ - "index1": 3 - }); - }); - }); - - - describe('#_getAddressMempoolSummary', function() { - it('skip if options not enabled', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var resultBase = { - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = {}; - as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { - if (err) { - return done(err); - } - Object.keys(result.unconfirmedAppearanceIds).length.should.equal(0); - result.unconfirmedBalance.should.equal(0); - done(); - }); - }); - it('include all txids and balance from inputs and outputs', function(done) { - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var resultBase = { - unconfirmedAppearanceIds: {}, - unconfirmedBalance: 0 - }; - var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); - var options = { - queryMempool: true - }; - var mempoolInputs = [ - { - address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', - hashType: 'scripthash', - txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', - inputIndex: 0, - timestamp: 1452874536321, - height: -1, - confirmations: 0 - } - ]; - var mempoolOutputs = [ - { - address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', - hashType: 'scripthash', - txid: '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d', - outputIndex: 0, - height: -1, - timestamp: 1452874521466, - satoshis: 131368318, - script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', - confirmations: 0 - }, - { - address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', - hashType: 'scripthash', - txid: '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', - outputIndex: 0, - height: -1, - timestamp: 1452874521466, - satoshis: 131368318, - script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', - confirmations: 0 - } - ]; - var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( - new Buffer(mempoolOutputs[1].txid, 'hex'), - 0 - ); - as.mempoolSpentIndex[spentIndexSyncKey] = true; - - var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - var addressIndex = encoding.encodeMempoolAddressIndexKey(address.hashBuffer, hashTypeBuffer); - as.mempoolAddressIndex[addressIndex] = 1; - - as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); - as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); - as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { - if (err) { - return done(err); - } - var txid1 = '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8'; - var txid2 = '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d'; - var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; - result.unconfirmedAppearanceIds[txid1].should.equal(1452874536321); - result.unconfirmedAppearanceIds[txid2].should.equal(1452874521466); - result.unconfirmedAppearanceIds[txid3].should.equal(1452874521466); - result.unconfirmedBalance.should.equal(131368318); - done(); - }); - }); - }); - - describe('#_transformAddressSummaryFromResult', function() { - var result = { - totalReceived: 1000000, - balance: 500000, - txids: [ - '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', - 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92' - ], - appearanceIds: { - 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92': 100000, - '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8': 200000 - }, - unconfirmedAppearanceIds: { - '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, - '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 - }, - unconfirmedTxids: [ - '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', - '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d' - ], - unconfirmedBalance: 500000 - }; - var testnode = { - network: Networks.testnet, - services: { - bitcoind: { - on: sinon.stub() - } - }, - datadir: 'testdir' - }; - it('will transform result into summary', function() { - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = {}; - var summary = as._transformAddressSummaryFromResult(result, options); - summary.totalReceived.should.equal(1000000); - summary.totalSpent.should.equal(500000); - summary.balance.should.equal(500000); - summary.appearances.should.equal(2); - summary.unconfirmedAppearances.should.equal(2); - summary.unconfirmedBalance.should.equal(500000); - summary.txids.length.should.equal(4); - }); - it('will omit txlist', function() { - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = { - noTxList: true - }; - var summary = as._transformAddressSummaryFromResult(result, options); - should.not.exist(summary.txids); - }); - it('will include full appearance ids', function() { - var as = new AddressService({ - mempoolMemoryIndex: true, - node: testnode - }); - var options = { - fullTxList: true - }; - var summary = as._transformAddressSummaryFromResult(result, options); - should.exist(summary.appearanceIds); - should.exist(summary.unconfirmedAppearanceIds); - }); - }); - -}); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 47156b50..f80b5f60 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1,26 +1,41 @@ 'use strict'; +var path = require('path'); +var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); +var crypto = require('crypto'); +var bitcore = require('bitcore-lib'); var sinon = require('sinon'); var proxyquire = require('proxyquire'); var fs = require('fs'); var sinon = require('sinon'); -var readFileSync = sinon.stub().returns(fs.readFileSync(__dirname + '/../data/bitcoin.conf')); + +var index = require('../../lib'); +var log = index.log; +var errors = index.errors; + +var Transaction = require('../../lib/transaction'); +var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/bitcoin.conf'))); var BitcoinService = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync } }); +var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.bitcoin.conf'), 'utf8'); describe('Bitcoin Service', function() { + var txhex = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000'; + var baseConfig = { node: { + network: bitcore.Networks.testnet + }, + spawn: { datadir: 'testdir', - network: { - name: 'regtest' - } + exec: 'testpath' } }; + describe('@constructor', function() { it('will create an instance', function() { var bitcoind = new BitcoinService(baseConfig); @@ -30,34 +45,172 @@ describe('Bitcoin Service', function() { var bitcoind = BitcoinService(baseConfig); should.exist(bitcoind); }); + it('will init caches', function() { + var bitcoind = new BitcoinService(baseConfig); + should.exist(bitcoind.utxosCache); + should.exist(bitcoind.txidsCache); + should.exist(bitcoind.balanceCache); + should.exist(bitcoind.summaryCache); + should.exist(bitcoind.transactionInfoCache); + + should.exist(bitcoind.transactionCache); + should.exist(bitcoind.rawTransactionCache); + should.exist(bitcoind.blockCache); + should.exist(bitcoind.rawBlockCache); + should.exist(bitcoind.blockHeaderCache); + should.exist(bitcoind.zmqKnownTransactions); + should.exist(bitcoind.zmqKnownBlocks); + should.exist(bitcoind.lastTip); + should.exist(bitcoind.lastTipTimeout); + }); + it('will init clients', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.should.deep.equal([]); + bitcoind.nodesIndex.should.equal(0); + bitcoind.nodes.push({client: sinon.stub()}); + should.exist(bitcoind.client); + }); + it('will set subscriptions', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.subscriptions.should.deep.equal({ + transaction: [], + block: [] + }); + }); }); + describe('@dependencies', function() { it('will have no dependencies', function() { BitcoinService.dependencies.should.deep.equal([]); }); }); - describe('#_loadConfiguration', function() { + + describe('#getAPIMethods', function() { + it('will return spec', function() { + var bitcoind = new BitcoinService(baseConfig); + var methods = bitcoind.getAPIMethods(); + should.exist(methods); + methods.length.should.equal(20); + }); + }); + + describe('#getPublishEvents', function() { + it('will return spec', function() { + var bitcoind = new BitcoinService(baseConfig); + var events = bitcoind.getPublishEvents(); + should.exist(events); + events.length.should.equal(2); + events[0].name.should.equal('bitcoind/transaction'); + events[0].scope.should.equal(bitcoind); + events[0].subscribe.should.be.a('function'); + events[0].unsubscribe.should.be.a('function'); + events[1].name.should.equal('bitcoind/block'); + events[1].scope.should.equal(bitcoind); + events[1].subscribe.should.be.a('function'); + events[1].unsubscribe.should.be.a('function'); + }); + it('will call subscribe/unsubscribe with correct args', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.subscribe = sinon.stub(); + bitcoind.unsubscribe = sinon.stub(); + var events = bitcoind.getPublishEvents(); + + events[0].subscribe('test'); + bitcoind.subscribe.args[0][0].should.equal('transaction'); + bitcoind.subscribe.args[0][1].should.equal('test'); + + events[0].unsubscribe('test'); + bitcoind.unsubscribe.args[0][0].should.equal('transaction'); + bitcoind.unsubscribe.args[0][1].should.equal('test'); + + events[1].subscribe('test'); + bitcoind.subscribe.args[1][0].should.equal('block'); + bitcoind.subscribe.args[1][1].should.equal('test'); + + events[1].unsubscribe('test'); + bitcoind.unsubscribe.args[1][0].should.equal('block'); + bitcoind.unsubscribe.args[1][1].should.equal('test'); + }); + }); + + describe('#subscribe', function() { + it('will push to subscriptions', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter = {}; + bitcoind.subscribe('block', emitter); + bitcoind.subscriptions.block[0].should.equal(emitter); + + var emitter2 = {}; + bitcoind.subscribe('transaction', emitter2); + bitcoind.subscriptions.transaction[0].should.equal(emitter2); + }); + }); + + describe('#unsubscribe', function() { + it('will remove item from subscriptions', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = {}; + var emitter2 = {}; + var emitter3 = {}; + var emitter4 = {}; + var emitter5 = {}; + bitcoind.subscribe('block', emitter1); + bitcoind.subscribe('block', emitter2); + bitcoind.subscribe('block', emitter3); + bitcoind.subscribe('block', emitter4); + bitcoind.subscribe('block', emitter5); + bitcoind.subscriptions.block.length.should.equal(5); + + bitcoind.unsubscribe('block', emitter3); + bitcoind.subscriptions.block.length.should.equal(4); + bitcoind.subscriptions.block[0].should.equal(emitter1); + bitcoind.subscriptions.block[1].should.equal(emitter2); + bitcoind.subscriptions.block[2].should.equal(emitter4); + bitcoind.subscriptions.block[3].should.equal(emitter5); + }); + }); + + describe('#_getDefaultConfig', function() { + it('will generate config file from defaults', function() { + var bitcoind = new BitcoinService(baseConfig); + var config = bitcoind._getDefaultConfig(); + config.should.equal(defaultBitcoinConf); + }); + }); + + describe('#_loadSpawnConfiguration', function() { it('will parse a bitcoin.conf file', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync, - existsSync: sinon.stub().returns(true) + existsSync: sinon.stub().returns(true), + writeFileSync: sinon.stub() }, mkdirp: { sync: sinon.stub() } }); var bitcoind = new TestBitcoin(baseConfig); - bitcoind._loadConfiguration({datadir: process.env.HOME + '/.bitcoin'}); - should.exist(bitcoind.configuration); - bitcoind.configuration.should.deep.equal({ - server: 1, - whitelist: '127.0.0.1', - txindex: 1, + bitcoind._loadSpawnConfiguration({datadir: process.env.HOME + '/.bitcoin'}); + should.exist(bitcoind.spawn.config); + bitcoind.spawn.config.should.deep.equal({ + addressindex: 1, + checkblocks: 144, + dbcache: 8192, + maxuploadtarget: 1024, port: 20000, + rpcport: 50001, rpcallowip: '127.0.0.1', rpcuser: 'bitcoin', - rpcpassword: 'local321' + rpcpassword: 'local321', + server: 1, + spentindex: 1, + timestampindex: 1, + txindex: 1, + upnp: 0, + whitelist: '127.0.0.1', + zmqpubhashblock: 'tcp://127.0.0.1:28332', + zmqpubrawtx: 'tcp://127.0.0.1:28332' }); }); it('should throw an exception if txindex isn\'t enabled in the configuration', function() { @@ -72,12 +225,12 @@ describe('Bitcoin Service', function() { }); var bitcoind = new TestBitcoin(baseConfig); (function() { - bitcoind._loadConfiguration({datadir: './test'}); - }).should.throw('Txindex option'); + bitcoind._loadSpawnConfiguration({datadir: './test'}); + }).should.throw(bitcore.errors.InvalidState); }); - it('should set https options if node https options are set', function() { + it('should NOT set https options if node https options are set', function() { var writeFileSync = function(path, config) { - config.should.equal('whitelist=127.0.0.1\ntxindex=1\nrpcssl=1\nrpcsslprivatekeyfile=key.pem\nrpcsslcertificatechainfile=cert.pem\n'); + config.should.equal(defaultBitcoinConf); }; var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { @@ -91,7 +244,6 @@ describe('Bitcoin Service', function() { }); var config = { node: { - datadir: 'testdir', network: { name: 'regtest' }, @@ -100,347 +252,1996 @@ describe('Bitcoin Service', function() { key: 'key.pem', cert: 'cert.pem' } + }, + spawn: { + datadir: 'testdir', + exec: 'testexec' } }; var bitcoind = new TestBitcoin(config); - bitcoind._loadConfiguration({datadir: process.env.HOME + '/.bitcoin'}); - }); - describe('reindex', function() { - var log = require('../../lib/').log; - var stub; - beforeEach(function() { - stub = sinon.stub(log, 'warn'); - }); - after(function() { - stub.restore(); - }); - it('should warn the user if reindex is set to 1 in the bitcoin.conf file', function() { - var readFileSync = function() { - return "txindex=1\nreindex=1"; - }; - var testbitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync, - existsSync: sinon.stub().returns(true) - }, - mkdirp: { - sync: sinon.stub() - }, - }); - var bitcoind = new testbitcoin(baseConfig); - bitcoind._loadConfiguration(); - stub.callCount.should.equal(1); - }); + bitcoind._loadSpawnConfiguration({datadir: process.env.HOME + '/.bitcoin'}); }); }); - describe('#_registerEventHandlers', function() { - it('will emit tx with transactions from bindings', function(done) { - var transaction = {}; - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - onTipUpdate: sinon.stub(), - startTxMon: sinon.stub().callsArgWith(0, [transaction]), - startTxMonLeave: sinon.stub().callsArgWith(0, [transaction]) - }; + + describe('#_checkConfigIndexes', function() { + var stub; + beforeEach(function() { + stub = sinon.stub(log, 'warn'); + }); + after(function() { + stub.restore(); + }); + it('should warn the user if reindex is set to 1 in the bitcoin.conf file', function() { + var bitcoind = new BitcoinService(baseConfig); + var config = { + txindex: 1, + addressindex: 1, + spentindex: 1, + server: 1, + zmqpubrawtx: 1, + zmqpubhashblock: 1, + reindex: 1 + }; + var node = {}; + bitcoind._checkConfigIndexes(config, node); + log.warn.callCount.should.equal(1); + node._reindex.should.equal(true); + }); + }); + + describe('#_resetCaches', function() { + it('will reset LRU caches', function() { + var bitcoind = new BitcoinService(baseConfig); + var keys = []; + for (var i = 0; i < 10; i++) { + keys.push(crypto.randomBytes(32)); + bitcoind.transactionInfoCache.set(keys[i], {}); + bitcoind.utxosCache.set(keys[i], {}); + bitcoind.txidsCache.set(keys[i], {}); + bitcoind.balanceCache.set(keys[i], {}); + bitcoind.summaryCache.set(keys[i], {}); + } + bitcoind._resetCaches(); + should.equal(bitcoind.transactionInfoCache.get(keys[0]), undefined); + should.equal(bitcoind.utxosCache.get(keys[0]), undefined); + should.equal(bitcoind.txidsCache.get(keys[0]), undefined); + should.equal(bitcoind.balanceCache.get(keys[0]), undefined); + should.equal(bitcoind.summaryCache.get(keys[0]), undefined); + }); + }); + + describe('#_tryAll', function() { + it('will retry the number of bitcoind nodes', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({}); + bitcoind.nodes.push({}); + bitcoind.nodes.push({}); + var count = 0; + var func = function(callback) { + count++; + if (count <= 2) { + callback(new Error('test')); + } else { + callback(); } + }; + bitcoind._tryAll(function(next) { + func(next); + }, function() { + count.should.equal(3); + done(); }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind.on('tx', function(tx) { - tx.should.equal(transaction); + }); + it('will get error if all fail', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({}); + bitcoind.nodes.push({}); + bitcoind.nodes.push({}); + var count = 0; + var func = function(callback) { + count++; + callback(new Error('test')); + }; + bitcoind._tryAll(function(next) { + func(next); + }, function(err) { + should.exist(err); + err.message.should.equal('test'); + count.should.equal(3); done(); }); - bitcoind._registerEventHandlers(); }); - it('will emit tip from bindings', function(done) { - var height = 1; - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - syncPercentage: function() { - return height * 10; - }, - onTipUpdate: function(callback) { - if (height >= 10) { - return callback(undefined); - } - setImmediate(function() { - callback(height++); + }); + + describe('#_wrapRPCError', function() { + it('will convert bitcoind-rpc error object into JavaScript error', function() { + var bitcoind = new BitcoinService(baseConfig); + var error = bitcoind._wrapRPCError({message: 'Test error', code: -1}); + error.should.be.an.instanceof(errors.RPCError); + error.code.should.equal(-1); + error.message.should.equal('Test error'); + }); + }); + + describe('#_initChain', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('will set height and genesis buffer', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var genesisBuffer = new Buffer([]); + bitcoind.getRawBlock = sinon.stub().callsArgWith(1, null, genesisBuffer); + bitcoind.nodes.push({ + client: { + getBestBlockHash: function(callback) { + callback(null, { + result: 'bestblockhash' + }); + }, + getBlock: function(hash, callback) { + if (hash === 'bestblockhash') { + callback(null, { + result: { + height: 5000 + } }); - }, - startTxMon: sinon.stub(), - startTxMonLeave: sinon.stub() - }; + } + }, + getBlockHash: function(num, callback) { + callback(null, { + result: 'genesishash' + }); + } } }); - var bitcoind = new TestBitcoin(baseConfig); - var tipCallCount = 0; - bitcoind.on('tip', function(height) { - should.exist(height); - tipCallCount++; - if (height === 9) { - tipCallCount.should.equal(9); - done(); - } + bitcoind._initChain(function() { + log.info.callCount.should.equal(1); + bitcoind.getRawBlock.callCount.should.equal(1); + bitcoind.getRawBlock.args[0][0].should.equal('genesishash'); + bitcoind.height.should.equal(5000); + bitcoind.genesisBuffer.should.equal(genesisBuffer); + done(); }); - bitcoind._registerEventHandlers(); }); }); - describe('#_onReady', function(done) { - var genesisBuffer = new Buffer('0100000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000bac8b0fa927c0ac8234287e33c5f74d38d354820e24756ad709d7038fc5f31f020e7494dffff001d03e4b6720101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0420e7494d017f062f503253482fffffffff0100f2052a010000002321021aeaf2f8638a129a3156fbe7e5ef635226b0bafd495ff03afe2c843d7e3a4b51ac00000000', 'hex'); - it('will emit ready and set the height and genesisBuffer', function(done) { - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - onTipUpdate: sinon.stub(), - startTxMon: sinon.stub(), - getInfo: sinon.stub().returns({ - blocks: 101 - }), - getBlock: sinon.stub().callsArgWith(1, null, genesisBuffer) - }; - } + + describe('#_getNetworkOption', function() { + afterEach(function() { + bitcore.Networks.disableRegtest(); + baseConfig.node.network = bitcore.Networks.testnet; + }); + it('return --testnet for testnet', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.node.network = bitcore.Networks.testnet; + bitcoind._getNetworkOption().should.equal('--testnet'); + }); + it('return --regtest for testnet', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.node.network = bitcore.Networks.testnet; + bitcore.Networks.enableRegtest(); + bitcoind._getNetworkOption().should.equal('--regtest'); + }); + it('return undefined for livenet', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.node.network = bitcore.Networks.livenet; + bitcore.Networks.enableRegtest(); + should.equal(bitcoind._getNetworkOption(), undefined); + }); + }); + + describe('#_zmqBlockHandler', function() { + it('will emit block', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + bitcoind._rapidProtectedUpdateTip = sinon.stub(); + bitcoind.on('block', function(block) { + block.should.equal(message); + done(); }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind._registerEventHandlers = sinon.stub(); - var result = {}; - var readyCallCount = 0; - bitcoind.on('ready', function() { - readyCallCount++; + bitcoind._zmqBlockHandler(node, message); + }); + it('will not emit same block twice', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + bitcoind._rapidProtectedUpdateTip = sinon.stub(); + bitcoind.on('block', function(block) { + block.should.equal(message); + done(); }); - bitcoind._onReady(result, function(err) { - if (err) { - throw err; - } - bitcoind._registerEventHandlers.callCount.should.equal(1); - readyCallCount.should.equal(1); - bitcoind.genesisBuffer.should.equal(genesisBuffer); - bitcoind.height.should.equal(101); + bitcoind._zmqBlockHandler(node, message); + bitcoind._zmqBlockHandler(node, message); + }); + it('will call function to update tip', function() { + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + bitcoind._rapidProtectedUpdateTip = sinon.stub(); + bitcoind._zmqBlockHandler(node, message); + bitcoind._rapidProtectedUpdateTip.callCount.should.equal(1); + bitcoind._rapidProtectedUpdateTip.args[0][0].should.equal(node); + bitcoind._rapidProtectedUpdateTip.args[0][1].should.equal(message); + }); + it('will emit to subscribers', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + bitcoind._rapidProtectedUpdateTip = sinon.stub(); + var emitter = new EventEmitter(); + bitcoind.subscriptions.block.push(emitter); + emitter.on('bitcoind/block', function(blockHash) { + blockHash.should.equal(message.toString('hex')); done(); }); + bitcoind._zmqBlockHandler(node, message); }); }); - describe('#start', function() { - it('call bindings start with the correct arguments', function(done) { - var startCallCount = 0; - var start = function(obj, cb) { - startCallCount++; - obj.datadir.should.equal('testdir'); - obj.network.should.equal('regtest'); - cb(); + + describe('#_rapidProtectedUpdateTip', function() { + it('will limit tip updates with rapid calls', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var callCount = 0; + bitcoind._updateTip = function() { + callCount++; + callCount.should.be.within(1, 2); + if (callCount > 1) { + done(); + } }; - var onBlocksReady = sinon.stub().callsArg(0); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - start: start, - onBlocksReady: onBlocksReady - }; + var node = {}; + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + var count = 0; + function repeat() { + bitcoind._rapidProtectedUpdateTip(node, message); + count++; + if (count < 50) { + repeat(); } - }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind._loadConfiguration = sinon.stub(); - bitcoind._onReady = sinon.stub().callsArg(1); - bitcoind.start(function(err) { - should.not.exist(err); - bitcoind._loadConfiguration.callCount.should.equal(1); - startCallCount.should.equal(1); - onBlocksReady.callCount.should.equal(1); - bitcoind._onReady.callCount.should.equal(1); + } + repeat(); + }); + }); + + describe('#_updateTip', function() { + var sandbox = sinon.sandbox.create(); + var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); + beforeEach(function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('log and emit rpc error from get block', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub(); + bitcoind.on('error', function(err) { + err.code.should.equal(-1); + err.message.should.equal('Test error'); + log.error.callCount.should.equal(1); done(); }); + var node = { + client: { + getBlock: sinon.stub().callsArgWith(1, {message: 'Test error', code: -1}) + } + }; + bitcoind._updateTip(node, message); }); - it('will give an error from bindings.start', function(done) { - var start = sinon.stub().callsArgWith(1, new Error('test')); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - start: start - }; + it('emit synced if percentage is 100', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 100); + bitcoind.on('synced', function() { + done(); + }); + var node = { + client: { + getBlock: sinon.stub() } + }; + bitcoind._updateTip(node, message); + }); + it('NOT emit synced if percentage is less than 100', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 99); + bitcoind.on('synced', function() { + throw new Error('Synced called'); }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind._loadConfiguration = sinon.stub(); - bitcoind.start(function(err) { - should.exist(err); + var node = { + client: { + getBlock: sinon.stub() + } + }; + bitcoind._updateTip(node, message); + log.info.callCount.should.equal(1); + done(); + }); + it('log and emit error from syncPercentage', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, new Error('test')); + bitcoind.on('error', function(err) { + log.error.callCount.should.equal(1); err.message.should.equal('test'); done(); }); + var node = { + client: { + getBlock: sinon.stub() + } + }; + bitcoind._updateTip(node, message); }); - it('will give an error from bindings.onBlocksReady', function(done) { - var start = sinon.stub().callsArgWith(1, null); - var onBlocksReady = sinon.stub().callsArgWith(0, new Error('test')); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - start: start, - onBlocksReady: onBlocksReady - }; + it('reset caches and set height', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub(); + bitcoind._resetCaches = sinon.stub(); + bitcoind.on('tip', function(height) { + bitcoind._resetCaches.callCount.should.equal(1); + height.should.equal(10); + bitcoind.height.should.equal(10); + done(); + }); + var node = { + client: { + getBlock: sinon.stub().callsArgWith(1, null, { + result: { + height: 10 + } + }) } + }; + bitcoind._updateTip(node, message); + }); + it('will NOT update twice for the same hash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub(); + bitcoind._resetCaches = sinon.stub(); + bitcoind.on('tip', function() { + done(); }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind._onReady = sinon.stub().callsArg(1); - bitcoind._loadConfiguration = sinon.stub(); - bitcoind.start(function(err) { - should.exist(err); - err.message.should.equal('test'); + var node = { + client: { + getBlock: sinon.stub().callsArgWith(1, null, { + result: { + height: 10 + } + }) + } + }; + bitcoind._updateTip(node, message); + bitcoind._updateTip(node, message); + }); + }); + + describe('#_zmqTransactionHandler', function() { + it('will emit to subscribers', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedBuffer = new Buffer('abcdef', 'hex'); + var emitter = new EventEmitter(); + bitcoind.subscriptions.transaction.push(emitter); + emitter.on('bitcoind/transaction', function(hex) { + hex.should.be.a('string'); + hex.should.equal(expectedBuffer.toString('hex')); done(); }); + var node = {}; + bitcoind._zmqTransactionHandler(node, expectedBuffer); }); - describe('reindex', function() { - var log = require('../../lib/').log; - var info; - beforeEach(function() { - info = sinon.stub(log, 'info'); + it('will NOT emit to subscribers more than once for the same tx', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedBuffer = new Buffer('abcdef', 'hex'); + var emitter = new EventEmitter(); + bitcoind.subscriptions.transaction.push(emitter); + emitter.on('bitcoind/transaction', function() { + done(); }); - afterEach(function() { - info.restore(); + var node = {}; + bitcoind._zmqTransactionHandler(node, expectedBuffer); + bitcoind._zmqTransactionHandler(node, expectedBuffer); + }); + it('will emit "tx" event', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedBuffer = new Buffer('abcdef', 'hex'); + bitcoind.on('tx', function(buffer) { + buffer.should.be.instanceof(Buffer); + buffer.toString('hex').should.equal(expectedBuffer.toString('hex')); + done(); }); - it('will wait for a reindex to complete before calling the callback.', function(done) { - var start = sinon.stub().callsArgWith(1, null); - var onBlocksReady = sinon.stub().callsArg(0); - var percentage = 98; - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - return { - start: start, - onBlocksReady: onBlocksReady, - syncPercentage: function() { - return percentage; - } - }; - } - }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind._reindex = true; - bitcoind._reindexWait = 1; - bitcoind._onReady = sinon.stub().callsArg(1); - bitcoind._loadConfiguration = sinon.stub(); - bitcoind.start(function() { - info.callCount.should.be.within(2,3); - bitcoind._reindex.should.be.false; - done(); - }); - setTimeout(function() { - percentage = 100; - }, 2); + var node = {}; + bitcoind._zmqTransactionHandler(node, expectedBuffer); + }); + it('will NOT emit "tx" event more than once for the same tx', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedBuffer = new Buffer('abcdef', 'hex'); + bitcoind.on('tx', function() { + done(); }); + var node = {}; + bitcoind._zmqTransactionHandler(node, expectedBuffer); + bitcoind._zmqTransactionHandler(node, expectedBuffer); }); }); - describe('#stop', function() { - it('will call bindings stop', function() { - var stop = sinon.stub().callsArgWith(0, null, 'status'); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - stop: stop - }; + + describe('#_subscribeZmqEvents', function() { + it('will call subscribe on zmq socket', function() { + var bitcoind = new BitcoinService(baseConfig); + var node = { + zmqSubSocket: { + subscribe: sinon.stub(), + on: sinon.stub() } + }; + bitcoind._subscribeZmqEvents(node); + node.zmqSubSocket.subscribe.callCount.should.equal(2); + node.zmqSubSocket.subscribe.args[0][0].should.equal('hashblock'); + node.zmqSubSocket.subscribe.args[1][0].should.equal('rawtx'); + }); + it('will call relevant handler for rawtx topics', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._zmqTransactionHandler = sinon.stub(); + var node = { + zmqSubSocket: new EventEmitter() + }; + node.zmqSubSocket.subscribe = sinon.stub(); + bitcoind._subscribeZmqEvents(node); + node.zmqSubSocket.on('message', function() { + bitcoind._zmqTransactionHandler.callCount.should.equal(1); + done(); }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind.stop(function(err, status) { - stop.callCount.should.equal(1); - should.not.exist(err); + var topic = new Buffer('rawtx', 'utf8'); + var message = new Buffer('abcdef', 'hex'); + node.zmqSubSocket.emit('message', topic, message); + }); + it('will call relevant handler for hashblock topics', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._zmqBlockHandler = sinon.stub(); + var node = { + zmqSubSocket: new EventEmitter() + }; + node.zmqSubSocket.subscribe = sinon.stub(); + bitcoind._subscribeZmqEvents(node); + node.zmqSubSocket.on('message', function() { + bitcoind._zmqBlockHandler.callCount.should.equal(1); + done(); }); + var topic = new Buffer('hashblock', 'utf8'); + var message = new Buffer('abcdef', 'hex'); + node.zmqSubSocket.emit('message', topic, message); }); - it('will give an error from bindings stop', function() { - var stop = sinon.stub().callsArgWith(0, new Error('test')); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - return { - stop: stop - }; + }); + + describe('#_initZmqSubSocket', function() { + }); + + describe('#_checkReindex', function() { + var sandbox = sinon.sandbox.create(); + before(function() { + sandbox.stub(log, 'info'); + }); + after(function() { + sandbox.restore(); + }); + it('give error from client syncpercentage', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._reindexWait = 1; + var node = { + _reindex: true, + client: { + syncPercentage: sinon.stub().callsArgWith(0, {code: -1 , message: 'Test error'}) } - }); - var bitcoind = new TestBitcoin(baseConfig); - bitcoind.stop(function(err) { - stop.callCount.should.equal(1); + }; + bitcoind._checkReindex(node, function(err) { should.exist(err); - err.message.should.equal('test'); + err.should.be.instanceof(errors.RPCError); + done(); }); }); - }); - describe('proxy methods', function() { - - var proxyMethods = [ - ['isSynced', 0], - ['syncPercentage', 0], - ['getBlock', 2], - ['isSpent', 2], - ['getBlockIndex', 1], - ['isMainChain', 1], - ['estimateFee', 1], - ['sendTransaction', 2], - ['getTransaction', 3], - ['getTransactionWithBlockInfo', 3], - ['getMempoolTransactions', 0], - ['addMempoolUncheckedTransaction', 1], - ['getBestBlockHash', 0], - ['getNextBlockHash', 1], - ['getInfo', 0] - ]; - - proxyMethods.forEach(function(x) { - it('pass ' + x[1] + ' argument(s) to ' + x[0], function() { - - var stub = sinon.stub(); - var TestBitcoin = proxyquire('../../lib/services/bitcoind', { - fs: { - readFileSync: readFileSync - }, - bindings: function(name) { - name.should.equal('bitcoind.node'); - var methods = {}; - methods[x[0]] = stub; - return methods; + it('will wait until syncpercentage is 100 percent', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._reindexWait = 1; + var percent = 90; + var node = { + _reindex: true, + client: { + syncPercentage: function(callback) { + callback(null, percent++); } - }); - - var bitcoind = new TestBitcoin(baseConfig); - var args = []; - for (var i = 0; i < x[1]; i++) { - args.push(i); } + }; + bitcoind._checkReindex(node, function() { + node._reindex.should.equal(false); + log.info.callCount.should.equal(11); + done(); + }); + }); + }); - bitcoind[x[0]].apply(bitcoind, args); - stub.callCount.should.equal(1); - stub.args[0].length.should.equal(x[1]); + describe('#_loadTipFromNode', function() { + }); + + describe('#_spawnChildProcess', function() { + }); + + describe('#_connectProcess', function() { + }); + + describe('#start', function() { + }); + + describe('#isSynced', function() { + it('will give error from syncPercentage', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, new Error('test')); + bitcoind.isSynced(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will give "true" if percentage is 100.00', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 100.00); + bitcoind.isSynced(function(err, synced) { + if (err) { + return done(err); + } + synced.should.equal(true); + done(); + }); + }); + it('will give "true" if percentage is 99.98', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 99.98); + bitcoind.isSynced(function(err, synced) { + if (err) { + return done(err); + } + synced.should.equal(true); + done(); + }); + }); + it('will give "false" if percentage is 99.49', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 99.49); + bitcoind.isSynced(function(err, synced) { + if (err) { + return done(err); + } + synced.should.equal(false); + done(); + }); + }); + it('will give "false" if percentage is 1', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.syncPercentage = sinon.stub().callsArgWith(0, null, 1); + bitcoind.isSynced(function(err, synced) { + if (err) { + return done(err); + } + synced.should.equal(false); + done(); + }); + }); + }); + + describe('#syncPercentage', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockchainInfo = sinon.stub().callsArgWith(0, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getBlockchainInfo: getBlockchainInfo + } + }); + bitcoind.syncPercentage(function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will call client getInfo and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockchainInfo = sinon.stub().callsArgWith(0, null, { + result: { + verificationprogress: '0.983821387' + } + }); + bitcoind.nodes.push({ + client: { + getBlockchainInfo: getBlockchainInfo + } + }); + bitcoind.syncPercentage(function(err, percentage) { + if (err) { + return done(err); + } + percentage.should.equal(98.3821387); + done(); + }); + }); + }); + + describe('#_normalizeAddressArg', function() { + it('will turn single address into array', function() { + var bitcoind = new BitcoinService(baseConfig); + var args = bitcoind._normalizeAddressArg('address'); + args.should.deep.equal(['address']); + }); + it('will keep an array as an array', function() { + var bitcoind = new BitcoinService(baseConfig); + var args = bitcoind._normalizeAddressArg(['address', 'address']); + args.should.deep.equal(['address', 'address']); + }); + }); + + describe('#getAddressBalance', function() { + }); + + describe('#getAddressUnspentOutputs', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + var options = {}; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + done(); + }); + }); + it('will give results from client getaddressutxos', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: expectedUtxos + }) + } + }); + var options = {}; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(1); + utxos.should.deep.equal(expectedUtxos); + done(); + }); + }); + it('will use cache', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var expectedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + } + ]; + var getAddressUtxos = sinon.stub().callsArgWith(1, null, { + result: expectedUtxos + }); + bitcoind.nodes.push({ + client: { + getAddressUtxos: getAddressUtxos + } + }); + var options = {}; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(1); + utxos.should.deep.equal(expectedUtxos); + getAddressUtxos.callCount.should.equal(1); + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(1); + utxos.should.deep.equal(expectedUtxos); + getAddressUtxos.callCount.should.equal(1); + done(); + }); + }); + }); + }); + + describe('#_getBalanceFromMempool', function() { + }); + + describe('#_getTxidsMempool', function() { + }); + + describe('#_getHeightRangeQuery', function() { + }); + + describe('#getAddressTxids', function() { + it('will give rpc error from mempool query', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + var options = {}; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + }); + }); + it('will give rpc error from txids query', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressTxids: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + var options = { + queryMempool: false + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + }); + }); + it('will get txid results', function(done) { + var expectedTxids = [ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47', + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579', + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a', + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693', + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681' + ]; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressTxids: sinon.stub().callsArgWith(1, null, { + result: expectedTxids.reverse() + }) + } + }); + var options = { + queryMempool: false + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, options, function(err, txids) { + if (err) { + return done(err); + } + txids.length.should.equal(expectedTxids.length); + txids.should.deep.equal(expectedTxids); + done(); + }); + }); + it('will get txid results from cache', function(done) { + var expectedTxids = [ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce' + ]; + var bitcoind = new BitcoinService(baseConfig); + var getAddressTxids = sinon.stub().callsArgWith(1, null, { + result: expectedTxids.reverse() + }); + bitcoind.nodes.push({ + client: { + getAddressTxids: getAddressTxids + } + }); + var options = { + queryMempool: false + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, options, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(1); + txids.should.deep.equal(expectedTxids); + + bitcoind.getAddressTxids(address, options, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(1); + txids.should.deep.equal(expectedTxids); + done(); + }); + }); + }); + it('will get txid results WITHOUT cache if rangeQuery and exclude mempool', function(done) { + var expectedTxids = [ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce' + ]; + var bitcoind = new BitcoinService(baseConfig); + var getAddressMempool = sinon.stub(); + var getAddressTxids = sinon.stub().callsArgWith(1, null, { + result: expectedTxids.reverse() + }); + bitcoind.nodes.push({ + client: { + getAddressTxids: getAddressTxids, + getAddressMempool: getAddressMempool + } + }); + var options = { + queryMempool: true, // start and end will exclude mempool + start: 4, + end: 2 + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, options, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(1); + getAddressMempool.callCount.should.equal(0); + txids.should.deep.equal(expectedTxids); + + bitcoind.getAddressTxids(address, options, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(2); + getAddressMempool.callCount.should.equal(0); + txids.should.deep.equal(expectedTxids); + done(); + }); + }); + }); + it('will get txid results from cache and live mempool', function(done) { + var expectedTxids = [ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce' + ]; + var bitcoind = new BitcoinService(baseConfig); + var getAddressTxids = sinon.stub().callsArgWith(1, null, { + result: expectedTxids.reverse() + }); + var getAddressMempool = sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2' + }, + { + txid: 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345' + }, + { + txid: 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9' + } + ] + }); + bitcoind.nodes.push({ + client: { + getAddressTxids: getAddressTxids, + getAddressMempool: getAddressMempool + } + }); + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressTxids(address, {queryMempool: false}, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(1); + txids.should.deep.equal(expectedTxids); + + bitcoind.getAddressTxids(address, {queryMempool: true}, function(err, txids) { + if (err) { + return done(err); + } + getAddressTxids.callCount.should.equal(1); + txids.should.deep.equal([ + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', // mempool + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', // mempool + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', // mempool + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce' // confirmed + ]); + done(); + }); + }); + }); + }); + + describe('#_getConfirmationDetail', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'warn'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('should get 1 confirmation', function() { + var tx = new Transaction(txhex); + tx.__height = 10; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.height = 10; + var confirmations = bitcoind._getConfirmationsDetail(tx); + confirmations.should.equal(1); + }); + it('should get 2 confirmation', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = new Transaction(txhex); + bitcoind.height = 11; + tx.__height = 10; + var confirmations = bitcoind._getConfirmationsDetail(tx); + confirmations.should.equal(2); + }); + it('should get 0 confirmation with overflow', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = new Transaction(txhex); + bitcoind.height = 3; + tx.__height = 10; + var confirmations = bitcoind._getConfirmationsDetail(tx); + log.warn.callCount.should.equal(1); + confirmations.should.equal(0); + }); + it('should get 1000 confirmation', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = new Transaction(txhex); + bitcoind.height = 1000; + tx.__height = 1; + var confirmations = bitcoind._getConfirmationsDetail(tx); + confirmations.should.equal(1000); + }); + }); + + describe('#_getAddressDetailsForTransaction', function() { + it('will calculate details for the transaction', function(done) { + /* jshint sub:true */ + var tx = bitcore.Transaction({ + 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', + 'version': 1, + 'inputs': [ + { + 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', + 'outputIndex': 0, + 'sequenceNumber': 4294967295, + 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'output': { + 'satoshis': 1000000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + } + ], + 'outputs': [ + { + 'satoshis': 100000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 200000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 50000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 300000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 349990000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + ], + 'nLockTime': 0 + }); + var bitcoind = new BitcoinService(baseConfig); + var addresses = ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']; + var details = bitcoind._getAddressDetailsForTransaction(tx, addresses); + should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([ + 0, 1, 2, 3, 4 + ]); + details.satoshis.should.equal(-10000); + done(); + }); + }); + + describe('#_getDetailedTransaction', function() { + it('will get detailed transaction info', function(done) { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var tx = { + populateInputs: sinon.stub().callsArg(2), + __height: 20, + __timestamp: 1453134151, + isCoinbase: sinon.stub().returns(false), + getFee: sinon.stub().returns(1000) + }; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, tx); + bitcoind.height = 300; + bitcoind._getAddressDetailsForTransaction = sinon.stub().returns({ + addresses: {}, + satoshis: 1000, + }); + bitcoind._getDetailedTransaction(txid, {}, function(err) { + if (err) { + return done(err); + } + done(); + }); + }); + it('give error from getTransactionWithBlockInfo', function(done) { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind._getDetailedTransaction(txid, {}, function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + it('give error from populateInputs', function(done) { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var tx = { + populateInputs: sinon.stub().callsArgWith(2, new Error('test')), + }; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, tx); + bitcoind._getDetailedTransaction(txid, {}, function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + + it('will correct detailed info', function(done) { + // block #314159 + // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e + // outputIndex 1 + var txAddress = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + var txString = '0100000001a08ee59fcd5d86fa170abb6d925d62d5c5c476359681b70877c04f270c4ef246000000008a47304402203fb9b476bb0c37c9b9ed5784ebd67ae589492be11d4ae1612be29887e3e4ce750220741ef83781d1b3a5df8c66fa1957ad0398c733005310d7d9b1d8c2310ef4f74c0141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02b0a75fac000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac20badc02000000001976a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac00000000'; + var previousTxString = '010000000155532fad2869bb951b0bd646a546887f6ee668c4c0ee13bf3f1c4bce6d6e3ed9000000008c4930460221008540795f4ef79b1d2549c400c61155ca5abbf3089c84ad280e1ba6db2a31abce022100d7d162175483d51174d40bba722e721542c924202a0c2970b07e680b51f3a0670141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02f0af3caf000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac80969800000000001976a91421277e65777760d1f3c7c982ba14ed8f934f005888ac00000000'; + var transaction = new Transaction(); + var previousTransaction = new Transaction(); + previousTransaction.fromString(previousTxString); + var previousTransactionTxid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + transaction.fromString(txString); + var txid = transaction.hash; + transaction.__blockHash = '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45'; + transaction.__height = 314159; + transaction.__timestamp = 1407292005; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.height = 314159; + bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, transaction); + bitcoind.getTransaction = function(prevTxid, callback) { + prevTxid.should.equal(previousTransactionTxid); + setImmediate(function() { + callback(null, previousTransaction); + }); + }; + var transactionInfo = { + addresses: {}, + txid: txid, + timestamp: 1407292005, + satoshis: 48020000, + address: txAddress + }; + transactionInfo.addresses[txAddress] = {}; + transactionInfo.addresses[txAddress].outputIndexes = [1]; + transactionInfo.addresses[txAddress].inputIndexes = []; + bitcoind._getAddressDetailsForTransaction = sinon.stub().returns(transactionInfo); + bitcoind._getDetailedTransaction(txid, {}, function(err, info) { + if (err) { + return done(err); + } + info.addresses[txAddress].should.deep.equal({ + outputIndexes: [1], + inputIndexes: [] + }); + info.satoshis.should.equal(48020000); + info.height.should.equal(314159); + info.confirmations.should.equal(1); + info.timestamp.should.equal(1407292005); + info.fees.should.equal(20000); + info.tx.should.equal(transaction); + done(); + }); + }); + }); + + describe('#_getAddressStrings', function() { + }); + + describe('#_paginateTxids', function() { + it('slice txids based on "from" and "to" (3 to 30)', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var paginated = bitcoind._paginateTxids(txids, 3, 30); + paginated.should.deep.equal([3, 4, 5, 6, 7, 8, 9, 10]); + }); + it('slice txids based on "from" and "to" (0 to 3)', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var paginated = bitcoind._paginateTxids(txids, 0, 3); + paginated.should.deep.equal([0, 1, 2]); + }); + it('slice txids based on "from" and "to" (0 to 1)', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var paginated = bitcoind._paginateTxids(txids, 0, 1); + paginated.should.deep.equal([0]); + }); + it('will throw error if "from" is greater than "to"', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + (function() { + var paginated = bitcoind._paginateTxids(txids, 1, 0); + }).should.throw('"from" is expected to be less than "to"'); + }); + }); + + describe('#getAddressHistory', function() { + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + it('will give an error if length of addresses is too long', function(done) { + var addresses = []; + for (var i = 0; i < 101; i++) { + addresses.push(address); + } + var bitcoind = new BitcoinService(baseConfig); + bitcoind.maxAddressesQuery = 100; + bitcoind.getAddressHistory(addresses, {}, function(err) { + should.exist(err); + err.message.match(/Maximum/); + done(); + }); + }); + it('give error from getAddressTxids', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, new Error('test')); + bitcoind.getAddressHistory('address', {}, function(err) { + should.exist(err); + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will paginate', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._getDetailedTransaction = function(txid, options, callback) { + callback(null, txid); + }; + var txids = ['one', 'two', 'three', 'four']; + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, txids); + bitcoind.getAddressHistory('address', {from: 1, to: 3}, function(err, data) { + if (err) { + return done(err); + } + data.items.length.should.equal(2); + data.items.should.deep.equal(['two', 'three']); + done(); + }); + }); + }); + + describe('#getAddressSummary', function() { + var txid1 = '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8'; + var txid2 = '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d'; + var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; + var memtxid1 = 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92'; + var memtxid2 = 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce'; + + it('will handle error from getAddressTxids', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + } + ] + }) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, new Error('test')); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, {}); + var address = ''; + var options = {}; + bitcoind.getAddressSummary(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will handle error from getAddressBalance', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + } + ] + }) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, {}); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, new Error('test'), {}); + var address = ''; + var options = {}; + bitcoind.getAddressSummary(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will handle error from client getAddressMempool', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, {}); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, {}); + var address = ''; + var options = {}; + bitcoind.getAddressSummary(address, options, function(err) { + should.exist(err); + err.should.be.instanceof(Error); + err.message.should.equal('Test error'); + done(); + }); + }); + it('should set all properties', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: memtxid1, + satoshis: -1000000 + }, + { + txid: memtxid2, + satoshis: 99999 + } + ] + }) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { + received: 30 * 1e8, + balance: 20 * 1e8 + }); + var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; + var options = {}; + bitcoind.getAddressSummary(address, options, function(err, summary) { + summary.appearances.should.equal(3); + summary.totalReceived.should.equal(3000000000); + summary.totalSpent.should.equal(1000000000); + summary.balance.should.equal(2000000000); + summary.unconfirmedAppearances.should.equal(2); + summary.unconfirmedBalance.should.equal(-900001); + summary.txids.should.deep.equal([ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92', + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d', + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247' + ]); + done(); + }); + }); + it('will get from cache with noTxList', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: memtxid1, + satoshis: -1000000 + }, + { + txid: memtxid2, + satoshis: 99999 + } + ] + }) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { + received: 30 * 1e8, + balance: 20 * 1e8 + }); + var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; + var options = { + noTxList: true + }; + function checkSummary(summary) { + summary.appearances.should.equal(3); + summary.totalReceived.should.equal(3000000000); + summary.totalSpent.should.equal(1000000000); + summary.balance.should.equal(2000000000); + summary.unconfirmedAppearances.should.equal(2); + summary.unconfirmedBalance.should.equal(-900001); + should.not.exist(summary.txids); + } + bitcoind.getAddressSummary(address, options, function(err, summary) { + checkSummary(summary); + bitcoind.getAddressTxids.callCount.should.equal(1); + bitcoind.getAddressBalance.callCount.should.equal(1); + bitcoind.getAddressSummary(address, options, function(err, summary) { + checkSummary(summary); + bitcoind.getAddressTxids.callCount.should.equal(1); + bitcoind.getAddressBalance.callCount.should.equal(1); + done(); + }); + }); + }); + }); + + describe('#getRawBlock', function() { + var blockhash = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + var blockhex = '0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000'; + it('will give rcp error from client getblockhash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getBlockHash: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + bitcoind.getRawBlock(10, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + done(); + }); + }); + it('will give rcp error from client getblock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getBlock: sinon.stub().callsArgWith(2, {code: -1, message: 'Test error'}) + } + }); + bitcoind.getRawBlock(blockhash, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + done(); + }); + }); + it('will try all nodes for getblock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockWithError = sinon.stub().callsArgWith(2, {code: -1, message: 'Test error'}); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({ + client: { + getBlock: getBlockWithError + } + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlockWithError + } + }); + bitcoind.nodes.push({ + client: { + getBlock: sinon.stub().callsArgWith(2, null, { + result: blockhex + }) + } + }); + bitcoind.getRawBlock(blockhash, function(err, buffer) { + if (err) { + return done(err); + } + buffer.should.be.instanceof(Buffer); + getBlockWithError.callCount.should.equal(2); + done(); + }); + }); + it('will get block from cache', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlock + } + }); + bitcoind.getRawBlock(blockhash, function(err, buffer) { + if (err) { + return done(err); + } + buffer.should.be.instanceof(Buffer); + getBlock.callCount.should.equal(1); + bitcoind.getRawBlock(blockhash, function(err, buffer) { + if (err) { + return done(err); + } + buffer.should.be.instanceof(Buffer); + getBlock.callCount.should.equal(1); + done(); + }); + }); + }); + it('will get block by height', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getRawBlock(0, function(err, buffer) { + if (err) { + return done(err); + } + buffer.should.be.instanceof(Buffer); + getBlock.callCount.should.equal(1); + getBlockHash.callCount.should.equal(1); + done(); + }); + }); + }); + + describe('#getBlock', function() { + }); + + describe('#getBlockHashesByTimestamp', function() { + it('should give an rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHashes = sinon.stub().callsArgWith(2, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getBlockHashes: getBlockHashes + } + }); + bitcoind.getBlockHashesByTimestamp(1441911000, 1441914000, function(err, hashes) { + should.exist(err); + err.message.should.equal('error'); + done(); + }); + }); + it('should get the correct block hashes', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var block1 = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + var block2 = '000000000383752a55a0b2891ce018fd0fdc0b6352502772b034ec282b4a1bf6'; + var getBlockHashes = sinon.stub().callsArgWith(2, null, { + result: [block2, block1] + }); + bitcoind.nodes.push({ + client: { + getBlockHashes: getBlockHashes + } + }); + bitcoind.getBlockHashesByTimestamp(1441914000, 1441911000, function(err, hashes) { + should.not.exist(err); + hashes.should.deep.equal([block2, block1]); + done(); + }); + }); + }); + + describe('#getBlockHeader', function() { + }); + + describe('#estimateFee', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var estimateFee = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + estimateFee: estimateFee + } + }); + bitcoind.estimateFee(1, function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will call client estimateFee and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var estimateFee = sinon.stub().callsArgWith(1, null, { + result: -1 + }); + bitcoind.nodes.push({ + client: { + estimateFee: estimateFee + } + }); + bitcoind.estimateFee(1, function(err, feesPerKb) { + if (err) { + return done(err); + } + feesPerKb.should.equal(-1); + done(); + }); + }); + }); + + describe('#sendTransaction', function(done) { + var tx = bitcore.Transaction(txhex); + it('will give rpc error', function() { + var bitcoind = new BitcoinService(baseConfig); + var sendRawTransaction = sinon.stub().callsArgWith(2, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + sendRawTransaction: sendRawTransaction + } + }); + bitcoind.sendTransaction(txhex, function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + }); + }); + it('will send to client and get hash', function() { + var bitcoind = new BitcoinService(baseConfig); + var sendRawTransaction = sinon.stub().callsArgWith(2, null, { + result: tx.hash + }); + bitcoind.nodes.push({ + client: { + sendRawTransaction: sendRawTransaction + } + }); + bitcoind.sendTransaction(txhex, function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal(tx.hash); + }); + }); + it('will send to client with absurd fees and get hash', function() { + var bitcoind = new BitcoinService(baseConfig); + var sendRawTransaction = sinon.stub().callsArgWith(2, null, { + result: tx.hash + }); + bitcoind.nodes.push({ + client: { + sendRawTransaction: sendRawTransaction + } + }); + bitcoind.sendTransaction(txhex, {allowAbsurdFees: true}, function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal(tx.hash); + }); + }); + }); + + describe('#getRawTransaction', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getRawTransaction = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getRawTransaction('txid', function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will try all nodes', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.tryAllInterval = 1; + var getRawTransactionWithError = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + var getRawTransaction = sinon.stub().callsArgWith(1, null, { + result: txhex + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransactionWithError + } + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransactionWithError + } + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getRawTransaction('txid', function(err, tx) { + if (err) { + return done(err); + } + should.exist(tx); + tx.should.be.an.instanceof(Buffer); + done(); + }); + }); + it('will get from cache', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getRawTransaction = sinon.stub().callsArgWith(1, null, { + result: txhex + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getRawTransaction('txid', function(err, tx) { + if (err) { + return done(err); + } + should.exist(tx); + tx.should.be.an.instanceof(Buffer); + + bitcoind.getRawTransaction('txid', function(err, tx) { + should.exist(tx); + tx.should.be.an.instanceof(Buffer); + getRawTransaction.callCount.should.equal(1); + done(); + }); + }); + }); + }); + + describe('#getTransaction', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getRawTransaction = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getTransaction('txid', function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will try all nodes', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.tryAllInterval = 1; + var getRawTransactionWithError = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + var getRawTransaction = sinon.stub().callsArgWith(1, null, { + result: txhex + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransactionWithError + } + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransactionWithError + } + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getTransaction('txid', function(err, tx) { + if (err) { + return done(err); + } + should.exist(tx); + tx.should.be.an.instanceof(bitcore.Transaction); + done(); + }); + }); + it('will get from cache', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getRawTransaction = sinon.stub().callsArgWith(1, null, { + result: txhex + }); + bitcoind.nodes.push({ + client: { + getRawTransaction: getRawTransaction + } + }); + bitcoind.getTransaction('txid', function(err, tx) { + if (err) { + return done(err); + } + should.exist(tx); + tx.should.be.an.instanceof(bitcore.Transaction); + + bitcoind.getTransaction('txid', function(err, tx) { + should.exist(tx); + tx.should.be.an.instanceof(bitcore.Transaction); + getRawTransaction.callCount.should.equal(1); + done(); + }); + + }); + }); + }); + + describe('#getTransactionWithBlockInfo', function() { + var txBuffer = new Buffer('01000000016f95980911e01c2c664b3e78299527a47933aac61a515930a8fe0213d1ac9abe01000000da0047304402200e71cda1f71e087c018759ba3427eb968a9ea0b1decd24147f91544629b17b4f0220555ee111ed0fc0f751ffebf097bdf40da0154466eb044e72b6b3dcd5f06807fa01483045022100c86d6c8b417bff6cc3bbf4854c16bba0aaca957e8f73e19f37216e2b06bb7bf802205a37be2f57a83a1b5a8cc511dc61466c11e9ba053c363302e7b99674be6a49fc0147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a9148a31d53a448c18996e81ce67811e5fb7da21e4468738c9d6f90000000017a9148ce5408cfeaddb7ccb2545ded41ef478109454848700000000', 'hex'); + var info = { + blockHash: '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6', + height: 530482, + timestamp: 1439559434000, + buffer: txBuffer + }; + + it('should give a transaction with height and timestamp', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, {code: -1, message: 'Test error'}) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getTransactionWithBlockInfo(txid, function(err) { + should.exist(err); + err.should.be.instanceof(errors.RPCError); + done(); + }); + }); + it('should give a transaction with height and timestamp', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: { + hex: txBuffer.toString('hex'), + blockhash: info.blockHash, + height: info.height, + time: info.timestamp, + vout: [ + { + spentTxId: 'txid', + spentIndex: 2, + spentHeight: 100 + } + ] + } + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getTransactionWithBlockInfo(txid, function(err, tx) { + // TODO verify additional info + should.exist(tx); + done(); + }); + }); + }); + + describe('#getBestBlockHash', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash + } + }); + bitcoind.getBestBlockHash(function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will call client getInfo and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, { + result: 'besthash' + }); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash + } + }); + bitcoind.getBestBlockHash(function(err, hash) { + if (err) { + return done(err); + } + should.exist(hash); + hash.should.equal('besthash'); + done(); + }); + }); + }); + + describe('#getSpentInfo', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getSpentInfo = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getSpentInfo: getSpentInfo + } + }); + bitcoind.getSpentInfo({}, function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will empty object when not found', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getSpentInfo = sinon.stub().callsArgWith(1, {message: 'test', code: -5}); + bitcoind.nodes.push({ + client: { + getSpentInfo: getSpentInfo + } + }); + bitcoind.getSpentInfo({}, function(err, info) { + should.not.exist(err); + info.should.deep.equal({}); + done(); + }); + }); + it('will call client getSpentInfo and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getSpentInfo = sinon.stub().callsArgWith(1, null, { + result: { + txid: 'txid', + index: 10, + height: 101 + } + }); + bitcoind.nodes.push({ + client: { + getSpentInfo: getSpentInfo + } + }); + bitcoind.getSpentInfo({}, function(err, info) { + if (err) { + return done(err); + } + info.txid.should.equal('txid'); + info.index.should.equal(10); + info.height.should.equal(101); + done(); + }); + }); + }); + + describe('#getInfo', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getInfo = sinon.stub().callsArgWith(0, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + getInfo: getInfo + } + }); + bitcoind.getInfo(function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will call client getInfo and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.node.getNetworkName = sinon.stub().returns('testnet'); + var getInfo = sinon.stub().callsArgWith(0, null, { + result: {} + }); + bitcoind.nodes.push({ + client: { + getInfo: getInfo + } + }); + bitcoind.getInfo(function(err, info) { + if (err) { + return done(err); + } + should.exist(info); + should.exist(info.network); + info.network.should.equal('testnet'); + done(); + }); + }); + }); + + describe('#generateBlock', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var generate = sinon.stub().callsArgWith(1, {message: 'error', code: -1}); + bitcoind.nodes.push({ + client: { + generate: generate + } + }); + bitcoind.generateBlock(10, function(err) { + should.exist(err); + err.should.be.an.instanceof(errors.RPCError); + done(); + }); + }); + it('will call client generate and give result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var generate = sinon.stub().callsArgWith(1, null, { + result: ['hash'] + }); + bitcoind.nodes.push({ + client: { + generate: generate + } + }); + bitcoind.generateBlock(10, function(err, hashes) { + if (err) { + return done(err); + } + hashes.length.should.equal(1); + hashes[0].should.equal('hash'); + done(); + }); + }); + }); + + describe('#stop', function() { + it('will callback if spawn is not set', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.stop(done); + }); + it('will exit spawned process', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.spawn = {}; + bitcoind.spawn.process = new EventEmitter(); + bitcoind.spawn.process.kill = sinon.stub(); + bitcoind.stop(done); + bitcoind.spawn.process.kill.callCount.should.equal(1); + bitcoind.spawn.process.kill.args[0][0].should.equal('SIGINT'); + bitcoind.spawn.process.emit('exit', 0); + }); + it('will give error with non-zero exit status code', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.spawn = {}; + bitcoind.spawn.process = new EventEmitter(); + bitcoind.spawn.process.kill = sinon.stub(); + bitcoind.stop(function(err) { + err.should.be.instanceof(Error); + err.code.should.equal(1); + done(); }); + bitcoind.spawn.process.kill.callCount.should.equal(1); + bitcoind.spawn.process.kill.args[0][0].should.equal('SIGINT'); + bitcoind.spawn.process.emit('exit', 1); }); }); diff --git a/test/services/db.unit.js b/test/services/db.unit.js deleted file mode 100644 index 60568cce..00000000 --- a/test/services/db.unit.js +++ /dev/null @@ -1,1038 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var sinon = require('sinon'); -var EventEmitter = require('events').EventEmitter; -var proxyquire = require('proxyquire'); -var index = require('../../'); -var DB = index.services.DB; -var blockData = require('../data/livenet-345003.json'); -var bitcore = require('bitcore-lib'); -var Networks = bitcore.Networks; -var Block = bitcore.Block; -var BufferUtil = bitcore.util.buffer; -var Transaction = bitcore.Transaction; -var transactionData = require('../data/bitcoin-transactions.json'); -var chainHashes = require('../data/hashes.json'); -var chainData = require('../data/testnet-blocks.json'); -var errors = index.errors; -var memdown = require('memdown'); -var levelup = require('levelup'); - -describe('DB Service', function() { - - function hexlebuf(hexString){ - return BufferUtil.reverse(new Buffer(hexString, 'hex')); - } - - function lebufhex(buf) { - return BufferUtil.reverse(buf).toString('hex'); - } - - var baseConfig = { - node: { - network: Networks.testnet, - datadir: 'testdir' - }, - store: memdown - }; - - var genesisBuffer = new Buffer('0100000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000bac8b0fa927c0ac8234287e33c5f74d38d354820e24756ad709d7038fc5f31f020e7494dffff001d03e4b6720101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0420e7494d017f062f503253482fffffffff0100f2052a010000002321021aeaf2f8638a129a3156fbe7e5ef635226b0bafd495ff03afe2c843d7e3a4b51ac00000000', 'hex'); - - - describe('#_setDataPath', function() { - it('should set the database path', function() { - var config = { - node: { - network: Networks.livenet, - datadir: process.env.HOME + '/.bitcoin' - }, - store: memdown - }; - var db = new DB(config); - db.dataPath.should.equal(process.env.HOME + '/.bitcoin/bitcore-node.db'); - }); - it('should load the db for testnet', function() { - var config = { - node: { - network: Networks.testnet, - datadir: process.env.HOME + '/.bitcoin' - }, - store: memdown - }; - var db = new DB(config); - db.dataPath.should.equal(process.env.HOME + '/.bitcoin/testnet3/bitcore-node.db'); - }); - it('error with unknown network', function() { - var config = { - node: { - network: 'unknown', - datadir: process.env.HOME + '/.bitcoin' - }, - store: memdown - }; - (function() { - var db = new DB(config); - }).should.throw('Unknown network'); - }); - it('should load the db with regtest', function() { - // Switch to use regtest - Networks.enableRegtest(); - var regtest = Networks.get('regtest'); - var config = { - node: { - network: regtest, - datadir: process.env.HOME + '/.bitcoin' - }, - store: memdown - }; - var db = new DB(config); - db.dataPath.should.equal(process.env.HOME + '/.bitcoin/regtest/bitcore-node.db'); - Networks.disableRegtest(); - }); - }); - - describe('#_checkVersion', function() { - var config = { - node: { - network: Networks.get('testnet'), - datadir: 'testdir' - }, - store: memdown - }; - it('will handle an error while retrieving the tip', function() { - var db = new DB(config); - db.store = {}; - db.store.get = sinon.stub().callsArgWith(2, new Error('test')); - db._checkVersion(function(err) { - should.exist(err); - err.message.should.equal('test'); - }); - }); - it('will handle an error while retrieving the version', function() { - var db = new DB(config); - db.store = {}; - db.store.get = function() {}; - var callCount = 0; - sinon.stub(db.store, 'get', function(key, options, callback) { - if (callCount === 1) { - return callback(new Error('test')); - } - callCount++; - setImmediate(callback); - }); - db._checkVersion(function(err) { - should.exist(err); - err.message.should.equal('test'); - }); - }); - it('will NOT check the version if a tip is not found', function(done) { - var db = new DB(config); - db.store = {}; - db.store.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); - db._checkVersion(done); - }); - it('will NOT give an error if the versions match', function(done) { - var db = new DB(config); - db.store = {}; - db.store.get = function() {}; - var callCount = 0; - sinon.stub(db.store, 'get', function(key, options, callback) { - if (callCount === 1) { - var versionBuffer = new Buffer(new Array(4)); - versionBuffer.writeUInt32BE(2); - return callback(null, versionBuffer); - } - callCount++; - setImmediate(callback); - }); - db.version = 2; - db._checkVersion(done); - }); - it('will give an error if the versions do NOT match', function(done) { - var db = new DB(config); - db.store = {}; - db.store.get = function() {}; - var callCount = 0; - sinon.stub(db.store, 'get', function(key, options, callback) { - if (callCount === 1) { - var versionBuffer = new Buffer(new Array(4)); - versionBuffer.writeUInt32BE(2); - return callback(null, versionBuffer); - } - callCount++; - setImmediate(callback); - }); - db.version = 3; - db._checkVersion(function(err) { - should.exist(err); - err.message.should.match(/^The version of the database/); - done(); - }); - }); - it('will default to version 1 if the version is NOT found', function(done) { - var db = new DB(config); - db.store = {}; - db.store.get = function() {}; - var callCount = 0; - sinon.stub(db.store, 'get', function(key, options, callback) { - if (callCount === 1) { - return callback(new levelup.errors.NotFoundError()); - } - callCount++; - setImmediate(callback); - }); - db.version = 1; - db._checkVersion(done); - }); - }); - - describe('#_setVersion', function() { - var config = { - node: { - network: Networks.get('testnet'), - datadir: 'testdir' - }, - store: memdown - }; - it('will give an error from the store', function(done) { - var db = new DB(config); - db.store = {}; - db.store.put = sinon.stub().callsArgWith(2, new Error('test')); - db._setVersion(function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - it('will set the version', function(done) { - var db = new DB(config); - db.store = {}; - db.store.put = sinon.stub().callsArgWith(2, null); - db.version = 5; - db._setVersion(function(err) { - if (err) { - return done(err); - } - db.store.put.args[0][0].should.deep.equal(new Buffer('ff', 'hex')); - db.store.put.args[0][1].should.deep.equal(new Buffer('00000005', 'hex')); - done(); - }); - }); - }); - - describe('#start', function() { - var TestDB; - - before(function() { - TestDB = proxyquire('../../lib/services/db', { - fs: { - existsSync: sinon.stub().returns(true) - }, - levelup: sinon.stub() - }); - }); - - it('should emit ready', function(done) { - var db = new TestDB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - on: sinon.spy(), - genesisBuffer: genesisBuffer - }; - db.loadTip = sinon.stub().callsArg(0); - db.connectBlock = sinon.stub().callsArg(1); - db._checkVersion = sinon.stub().callsArg(0); - db._setVersion = sinon.stub().callsArg(0); - db.sync = sinon.stub(); - var readyFired = false; - db.on('ready', function() { - readyFired = true; - }); - db.start(function() { - readyFired.should.equal(true); - done(); - }); - }); - - it('will call sync when there is a new tip', function(done) { - var db = new TestDB(baseConfig); - db.node.services = {}; - db.node.services.bitcoind = new EventEmitter(); - db.node.services.bitcoind.genesisBuffer = genesisBuffer; - db.loadTip = sinon.stub().callsArg(0); - db.connectBlock = sinon.stub().callsArg(1); - db._checkVersion = sinon.stub().callsArg(0); - db._setVersion = sinon.stub().callsArg(0); - db.sync = sinon.stub(); - db.start(function() { - db.sync = function() { - done(); - }; - db.node.services.bitcoind.emit('tip', 10); - }); - }); - - it('will not call sync when there is a new tip and shutting down', function(done) { - var db = new TestDB(baseConfig); - db.node.services = {}; - db.node.services.bitcoind = new EventEmitter(); - db.node.services.bitcoind.syncPercentage = sinon.spy(); - db.node.services.bitcoind.genesisBuffer = genesisBuffer; - db.loadTip = sinon.stub().callsArg(0); - db.connectBlock = sinon.stub().callsArg(1); - db._checkVersion = sinon.stub().callsArg(0); - db._setVersion = sinon.stub().callsArg(0); - db.node.stopping = true; - db.sync = sinon.stub(); - db.start(function() { - db.sync.callCount.should.equal(1); - db.node.services.bitcoind.once('tip', function() { - db.sync.callCount.should.equal(1); - done(); - }); - db.node.services.bitcoind.emit('tip', 10); - }); - }); - - }); - - describe('#stop', function() { - it('should wait until db has stopped syncing before closing leveldb', function(done) { - var db = new DB(baseConfig); - db.store = { - close: sinon.stub().callsArg(0) - }; - db.bitcoindSyncing = true; - - db.stop(function(err) { - should.not.exist(err); - done(); - }); - - setTimeout(function() { - db.bitcoindSyncing = false; - }, 15); - }); - }); - - describe('#getTransaction', function() { - it('will return a NotFound error', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getTransaction: sinon.stub().callsArgWith(2, null, null) - }; - var txid = '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623'; - db.getTransaction(txid, true, function(err) { - err.should.be.instanceof(errors.Transaction.NotFound); - done(); - }); - }); - it('will return an error from bitcoind', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getTransaction: sinon.stub().callsArgWith(2, new Error('test error')) - }; - var txid = '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623'; - db.getTransaction(txid, true, function(err) { - err.message.should.equal('test error'); - done(); - }); - }); - it('will return an error from bitcoind', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getTransaction: sinon.stub().callsArgWith(2, null, new Buffer(transactionData[0].hex, 'hex')) - }; - var txid = '7426c707d0e9705bdd8158e60983e37d0f5d63529086d6672b07d9238d5aa623'; - db.getTransaction(txid, true, function(err, tx) { - if (err) { - throw err; - } - should.exist(tx); - done(); - }); - }); - }); - - describe('#loadTip', function() { - it('genesis block if no metadata is found in the db', function(done) { - var db = new DB(baseConfig); - db.genesis = Block.fromBuffer(genesisBuffer); - db.store = { - get: sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()) - }; - db.connectBlock = sinon.stub().callsArg(1); - db.sync = sinon.stub(); - db.loadTip(function() { - should.exist(db.tip); - db.tip.hash.should.equal('00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'); - done(); - }); - }); - - it('tip from the database if it exists', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({height: 1}) - } - } - }; - var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; - var tip = Block.fromBuffer(genesisBuffer); - var db = new DB({node: node}); - db.store = { - get: sinon.stub().callsArgWith(2, null, new Buffer(tipHash, 'hex')) - }; - db.getBlock = sinon.stub().callsArgWith(1, null, tip); - db.sync = sinon.stub(); - db.loadTip(function() { - should.exist(db.tip); - db.tip.hash.should.equal(tipHash); - db.tip.__height.should.equal(1); - done(); - }); - }); - - it('give error if levelup error', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub() - } - } - }; - var db = new DB({node: node}); - db.store = { - get: sinon.stub().callsArgWith(2, new Error('test')) - }; - db.loadTip(function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('should try 3 times before giving error from getBlock', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub(), - getBlockIndex: sinon.stub().returns({height: 1}) - } - } - }; - var db = new DB({node: node}); - db.retryInterval = 10; - var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; - db.store = { - get: sinon.stub().callsArgWith(2, null, new Buffer(tipHash, 'hex')) - }; - db.getBlock = sinon.stub().callsArgWith(1, new Error('test')); - db.loadTip(function(err) { - should.exist(err); - db.getBlock.callCount.should.equal(3); - err.message.should.equal('test'); - done(); - }); - }); - }); - - describe('#getBlock', function() { - var db = new DB(baseConfig); - var blockBuffer = new Buffer(blockData, 'hex'); - var expectedBlock = Block.fromBuffer(blockBuffer); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getBlock: sinon.stub().callsArgWith(1, null, blockBuffer) - }; - - it('should get the block from bitcoin daemon', function(done) { - db.getBlock('00000000000000000593b60d8b4f40fd1ec080bdb0817d475dae47b5f5b1f735', function(err, block) { - should.not.exist(err); - block.hash.should.equal(expectedBlock.hash); - done(); - }); - }); - it('should give an error when bitcoind.js gives an error', function(done) { - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = {}; - db.node.services.bitcoind.getBlock = sinon.stub().callsArgWith(1, new Error('error')); - db.getBlock('00000000000000000593b60d8b4f40fd1ec080bdb0817d475dae47b5f5b1f735', function(err, block) { - should.exist(err); - err.message.should.equal('error'); - done(); - }); - }); - }); - - describe('#getBlockHashesByTimestamp', function() { - it('should get the correct block hashes', function(done) { - var db = new DB(baseConfig); - var readStream = new EventEmitter(); - db.store = { - createReadStream: sinon.stub().returns(readStream) - }; - - var block1 = { - hash: '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b', - timestamp: 1441911909 - }; - - var block2 = { - hash: '000000000383752a55a0b2891ce018fd0fdc0b6352502772b034ec282b4a1bf6', - timestamp: 1441913112 - }; - - db.getBlockHashesByTimestamp(1441914000, 1441911000, function(err, hashes) { - should.not.exist(err); - hashes.should.deep.equal([block2.hash, block1.hash]); - done(); - }); - - readStream.emit('data', { - key: db._encodeBlockIndexKey(block2.timestamp), - value: db._encodeBlockIndexValue(block2.hash) - }); - - readStream.emit('data', { - key: db._encodeBlockIndexKey(block1.timestamp), - value: db._encodeBlockIndexValue(block1.hash) - }); - - readStream.emit('close'); - }); - - it('should give an error if the stream has an error', function(done) { - var db = new DB(baseConfig); - var readStream = new EventEmitter(); - db.store = { - createReadStream: sinon.stub().returns(readStream) - }; - - db.getBlockHashesByTimestamp(1441911000, 1441914000, function(err, hashes) { - should.exist(err); - err.message.should.equal('error'); - done(); - }); - - readStream.emit('error', new Error('error')); - - readStream.emit('close'); - }); - - it('should give an error if the timestamp is out of range', function(done) { - var db = new DB(baseConfig); - var readStream = new EventEmitter(); - db.store = { - createReadStream: sinon.stub().returns(readStream) - }; - - db.getBlockHashesByTimestamp(-1, -5, function(err, hashes) { - should.exist(err); - err.message.should.equal('Invalid Argument: timestamp out of bounds'); - done(); - }); - }); - }); - - describe('#getPrevHash', function() { - it('should return prevHash from bitcoind', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getBlockIndex: sinon.stub().returns({ - prevHash: 'prevhash' - }) - }; - - db.getPrevHash('hash', function(err, prevHash) { - should.not.exist(err); - prevHash.should.equal('prevhash'); - done(); - }); - }); - - it('should give an error if bitcoind could not find it', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getBlockIndex: sinon.stub().returns(null) - }; - - db.getPrevHash('hash', function(err, prevHash) { - should.exist(err); - done(); - }); - }); - }); - - describe('#getTransactionWithBlockInfo', function() { - it('should give a transaction with height and timestamp', function(done) { - var txBuffer = new Buffer('01000000016f95980911e01c2c664b3e78299527a47933aac61a515930a8fe0213d1ac9abe01000000da0047304402200e71cda1f71e087c018759ba3427eb968a9ea0b1decd24147f91544629b17b4f0220555ee111ed0fc0f751ffebf097bdf40da0154466eb044e72b6b3dcd5f06807fa01483045022100c86d6c8b417bff6cc3bbf4854c16bba0aaca957e8f73e19f37216e2b06bb7bf802205a37be2f57a83a1b5a8cc511dc61466c11e9ba053c363302e7b99674be6a49fc0147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a9148a31d53a448c18996e81ce67811e5fb7da21e4468738c9d6f90000000017a9148ce5408cfeaddb7ccb2545ded41ef478109454848700000000', 'hex'); - var info = { - blockHash: '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6', - height: 530482, - timestamp: 1439559434000, - buffer: txBuffer - }; - - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, info) - }; - - db.getTransactionWithBlockInfo('2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f', true, function(err, tx) { - should.not.exist(err); - tx.__blockHash.should.equal(info.blockHash); - tx.__height.should.equal(info.height); - tx.__timestamp.should.equal(info.timestamp); - done(); - }); - }); - it('should give an error if one occurred', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, new Error('error')) - }; - - db.getTransactionWithBlockInfo('tx', true, function(err, tx) { - should.exist(err); - done(); - }); - }); - }); - - describe('#sendTransaction', function() { - it('should handle a basic serialized transaction hex string', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - sendTransaction: sinon.stub().returns('txid') - }; - - var tx = 'hexstring'; - db.sendTransaction(tx, function(err, txid) { - should.not.exist(err); - txid.should.equal('txid'); - done(); - }); - }); - it('should give the txid on success', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - sendTransaction: sinon.stub().returns('txid') - }; - - var tx = new Transaction(); - tx.serialize = sinon.stub().returns('txstring'); - db.sendTransaction(tx, function(err, txid) { - should.not.exist(err); - tx.serialize.callCount.should.equal(1); - txid.should.equal('txid'); - done(); - }); - }); - it('should give an error if bitcoind threw an error', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - sendTransaction: sinon.stub().throws(new Error('error')) - }; - - var tx = new Transaction(); - tx.serialize = sinon.stub().returns('txstring'); - db.sendTransaction(tx, function(err, txid) { - tx.serialize.callCount.should.equal(1); - should.exist(err); - done(); - }); - }); - }); - - describe('#estimateFee', function() { - it('should pass along the fee from bitcoind', function(done) { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - db.node.services.bitcoind = { - estimateFee: sinon.stub().returns(1000) - }; - - db.estimateFee(5, function(err, fee) { - should.not.exist(err); - fee.should.equal(1000); - db.node.services.bitcoind.estimateFee.args[0][0].should.equal(5); - done(); - }); - }); - }); - - describe('#connectBlock', function() { - it('should remove block from mempool and call blockHandler with true', function(done) { - var db = new DB(baseConfig); - db.mempool = { - removeBlock: sinon.stub() - }; - db.runAllBlockHandlers = sinon.stub().callsArg(2); - db.connectBlock({hash: 'hash'}, function(err) { - should.not.exist(err); - db.runAllBlockHandlers.args[0][1].should.equal(true); - done(); - }); - }); - }); - - describe('#disconnectBlock', function() { - it('should call blockHandler with false', function(done) { - var db = new DB(baseConfig); - db.runAllBlockHandlers = sinon.stub().callsArg(2); - db.disconnectBlock({hash: 'hash'}, function(err) { - should.not.exist(err); - db.runAllBlockHandlers.args[0][1].should.equal(false); - done(); - }); - }); - }); - - describe('#runAllBlockHandlers', function() { - var db = new DB(baseConfig); - var Service1 = function() {}; - Service1.prototype.blockHandler = sinon.stub().callsArgWith(2, null, ['op1', 'op2', 'op3']); - var Service2 = function() {}; - Service2.prototype.blockHandler = sinon.stub().callsArgWith(2, null, ['op4', 'op5']); - var Service3 = function() {}; - var Service4 = function() {}; - Service4.prototype.blockHandler = sinon.stub().callsArgWith(2, null, 'bad-value'); - db.node = {}; - db.node.services = { - service1: new Service1(), - service2: new Service2() - }; - db.store = { - batch: sinon.stub().callsArg(1) - }; - - var block = { - hash: '00000000000000000d0aaf93e464ddeb503655a0750f8b9c6eed0bdf0ccfc863', - header: { - timestamp: 1441906365 - } - }; - - it('should call blockHandler in all services and perform operations', function(done) { - db.runAllBlockHandlers(block, true, function(err) { - should.not.exist(err); - var tipOp = { - type: 'put', - key: DB.PREFIXES.TIP, - value: new Buffer('00000000000000000d0aaf93e464ddeb503655a0750f8b9c6eed0bdf0ccfc863', 'hex') - } - var blockOp = { - type: 'put', - key: db._encodeBlockIndexKey(1441906365), - value: db._encodeBlockIndexValue('00000000000000000d0aaf93e464ddeb503655a0750f8b9c6eed0bdf0ccfc863') - }; - db.store.batch.args[0][0].should.deep.equal([tipOp, blockOp, 'op1', 'op2', 'op3', 'op4', 'op5']); - done(); - }); - }); - - it('should give an error if one of the services gives an error', function(done) { - var Service3 = function() {}; - Service3.prototype.blockHandler = sinon.stub().callsArgWith(2, new Error('error')); - db.node.services.service3 = new Service3(); - - db.runAllBlockHandlers(block, true, function(err) { - should.exist(err); - done(); - }); - }); - - it('should not give an error if a service does not have blockHandler', function(done) { - db.node = {}; - db.node.services = { - service3: new Service3() - }; - - db.runAllBlockHandlers(block, true, function(err) { - should.not.exist(err); - done(); - }); - }); - - it('should throw an error if blockHandler gives unexpected result', function() { - db.node = {}; - db.node.services = { - service4: new Service4() - }; - - (function() { - db.runAllBlockHandlers(block, true, function(err) { - should.not.exist(err); - }); - }).should.throw('bitcore.ErrorInvalidArgument'); - }); - }); - - describe('#getAPIMethods', function() { - it('should return the correct db methods', function() { - var db = new DB(baseConfig); - db.node = {}; - db.node.services = {}; - var methods = db.getAPIMethods(); - methods.length.should.equal(6); - }); - }); - - describe('#findCommonAncestor', function() { - it('will find an ancestor 6 deep', function(done) { - var db = new DB(baseConfig); - db.tip = { - hash: chainHashes[chainHashes.length - 1] - }; - - var expectedAncestor = chainHashes[chainHashes.length - 6]; - - var mainBlocks = {}; - for(var i = chainHashes.length - 1; i > chainHashes.length - 10; i--) { - var hash = chainHashes[i]; - var prevHash = hexlebuf(chainHashes[i - 1]); - mainBlocks[hash] = { - header: { - prevHash: prevHash - } - }; - } - - var forkedBlocks = { - 'd7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82': { - header: { - prevHash: hexlebuf('76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a') - }, - hash: 'd7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82' - }, - '76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a': { - header: { - prevHash: hexlebuf('f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c') - }, - hash: '76d920dbd83beca9fa8b2f346d5c5a81fe4a350f4b355873008229b1e6f8701a' - }, - 'f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c': { - header: { - prevHash: hexlebuf('2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31') - }, - hash: 'f0a0d76a628525243c8af7606ee364741ccd5881f0191bbe646c8a4b2853e60c' - }, - '2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31': { - header: { - prevHash: hexlebuf('adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453') - }, - hash: '2f72b809d5ccb750c501abfdfa8c4c4fad46b0b66c088f0568d4870d6f509c31' - }, - 'adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453': { - header: { - prevHash: hexlebuf('3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618') - }, - hash: 'adf66e6ae10bc28fc22bc963bf43e6b53ef4429269bdb65038927acfe66c5453' - }, - '3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618': { - header: { - prevHash: hexlebuf(expectedAncestor) - }, - hash: '3ea12707e92eed024acf97c6680918acc72560ec7112cf70ac213fb8bb4fa618' - } - }; - db.node.services = {}; - db.node.services.bitcoind = { - getBlockIndex: function(hash) { - var forkedBlock = forkedBlocks[hash]; - var mainBlock = mainBlocks[hash]; - var prevHash; - if (forkedBlock && forkedBlock.header.prevHash) { - prevHash = BufferUtil.reverse(forkedBlock.header.prevHash).toString('hex'); - } else if (mainBlock && mainBlock.header.prevHash){ - prevHash = BufferUtil.reverse(mainBlock.header.prevHash).toString('hex'); - } else { - return null; - } - return { - prevHash: prevHash - }; - } - }; - var block = forkedBlocks['d7fa6f3d5b2fe35d711e6aca5530d311b8c6e45f588a65c642b8baf4b4441d82']; - db.findCommonAncestor(block, function(err, ancestorHash) { - if (err) { - throw err; - } - ancestorHash.should.equal(expectedAncestor); - done(); - }); - }); - }); - - describe('#syncRewind', function() { - it('will undo blocks 6 deep', function() { - var db = new DB(baseConfig); - var ancestorHash = chainHashes[chainHashes.length - 6]; - db.tip = { - __height: 10, - hash: chainHashes[chainHashes.length], - header: { - prevHash: hexlebuf(chainHashes[chainHashes.length - 1]) - } - }; - db.emit = sinon.stub(); - db.getBlock = function(hash, callback) { - setImmediate(function() { - for(var i = chainHashes.length; i > 0; i--) { - var block = { - hash: chainHashes[i], - header: { - prevHash: hexlebuf(chainHashes[i - 1]) - } - }; - if (chainHashes[i] === hash) { - callback(null, block); - } - } - }); - }; - db.node.services = {}; - db.disconnectBlock = function(block, callback) { - setImmediate(callback); - }; - db.findCommonAncestor = function(block, callback) { - setImmediate(function() { - callback(null, ancestorHash); - }); - }; - var forkedBlock = {}; - db.syncRewind(forkedBlock, function(err) { - if (err) { - throw err; - } - db.tip.__height.should.equal(4); - db.tip.hash.should.equal(ancestorHash); - }); - }); - }); - - describe('#sync', function() { - var node = new EventEmitter(); - var syncConfig = { - node: node, - store: memdown - }; - syncConfig.node.network = Networks.testnet; - syncConfig.node.datadir = 'testdir'; - it('will get and add block up to the tip height', function(done) { - var db = new DB(syncConfig); - var blockBuffer = new Buffer(blockData, 'hex'); - var block = Block.fromBuffer(blockBuffer); - db.node.services = {}; - db.node.services.bitcoind = { - getBlock: sinon.stub().callsArgWith(1, null, blockBuffer), - isSynced: sinon.stub().returns(true), - height: 1 - }; - db.tip = { - __height: 0, - hash: lebufhex(block.header.prevHash) - }; - db.emit = sinon.stub(); - db.cache = { - hashes: {} - }; - db.connectBlock = function(block, callback) { - db.tip.__height += 1; - callback(); - }; - db.node.once('synced', function() { - done(); - }); - db.sync(); - }); - it('will exit and emit error with error from bitcoind.getBlock', function(done) { - var db = new DB(syncConfig); - db.node.services = {}; - db.node.services.bitcoind = { - getBlock: sinon.stub().callsArgWith(1, new Error('test error')), - height: 1 - }; - db.tip = { - __height: 0 - }; - db.node.on('error', function(err) { - err.message.should.equal('test error'); - done(); - }); - db.sync(); - }); - it('will stop syncing when the node is stopping', function(done) { - var db = new DB(syncConfig); - var blockBuffer = new Buffer(blockData, 'hex'); - var block = Block.fromBuffer(blockBuffer); - db.node.services = {}; - db.node.services.bitcoind = { - getBlock: sinon.stub().callsArgWith(1, null, blockBuffer), - isSynced: sinon.stub().returns(true), - height: 1 - }; - db.tip = { - __height: 0, - hash: block.prevHash - }; - db.emit = sinon.stub(); - db.cache = { - hashes: {} - }; - db.connectBlock = function(block, callback) { - db.tip.__height += 1; - callback(); - }; - db.node.stopping = true; - var synced = false; - db.node.once('synced', function() { - synced = true; - }); - db.sync(); - setTimeout(function() { - synced.should.equal(false); - done(); - }, 10); - }); - }); - -}); diff --git a/test/transaction.unit.js b/test/transaction.unit.js index eee529a4..0a78c8c5 100644 --- a/test/transaction.unit.js +++ b/test/transaction.unit.js @@ -4,9 +4,44 @@ var should = require('chai').should(); var sinon = require('sinon'); var bitcoinlib = require('../'); var Transaction = bitcoinlib.Transaction; -var levelup = require('levelup'); describe('Bitcoin Transaction', function() { + + describe('#populateSpentInfo', function() { + it('will call db.getSpentInfo with correct arguments', function(done) { + var tx = new Transaction(); + tx.to('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i', 1000); + tx.to('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', 2000); + var expectedHash = tx.hash; + var expectedIndex = 2; + var expectedHeight = 300000; + var db = { + getSpentInfo: sinon.stub().callsArgWith(1, null, { + txid: expectedHash, + index: expectedIndex, + height: expectedHeight + }) + }; + tx.populateSpentInfo(db, {}, function(err) { + if (err) { + return done(err); + } + db.getSpentInfo.args[0][0].txid.should.equal(tx.hash); + db.getSpentInfo.args[0][0].index.should.equal(0); + tx.outputs[0].__spentTxId.should.equal(expectedHash); + tx.outputs[0].__spentIndex.should.equal(expectedIndex); + tx.outputs[0].__spentHeight.should.equal(expectedHeight); + + db.getSpentInfo.args[1][0].txid.should.equal(tx.hash); + db.getSpentInfo.args[1][0].index.should.equal(1); + tx.outputs[1].__spentTxId.should.equal(expectedHash); + tx.outputs[1].__spentIndex.should.equal(expectedIndex); + tx.outputs[1].__spentHeight.should.equal(expectedHeight); + done(); + }); + }); + }); + describe('#populateInputs', function() { it('will call _populateInput with transactions', function() { var tx = new Transaction(); @@ -22,6 +57,17 @@ describe('Bitcoin Transaction', function() { tx._populateInput.args[0][2].should.equal(transactions); }); }); + it('will skip coinbase transactions', function() { + var tx = new Transaction(); + tx.isCoinbase = sinon.stub().returns(true); + tx._populateInput = sinon.stub().callsArg(3); + tx.inputs = ['input']; + var transactions = []; + var db = {}; + tx.populateInputs(db, transactions, function(err) { + tx._populateInput.callCount.should.equal(0); + }); + }); }); describe('#_populateInput', function() { @@ -29,6 +75,15 @@ describe('Bitcoin Transaction', function() { prevTxId: new Buffer('d6cffbb343a6a41eeaa199478c985493843bfe6a59d674a5c188787416cbcda3', 'hex'), outputIndex: 0 }; + it('should give an error if the input does not have a prevTxId', function(done) { + var badInput = {}; + var tx = new Transaction(); + tx._populateInput({}, badInput, [], function(err) { + should.exist(err); + err.message.should.equal('Input is expected to have prevTxId as a buffer'); + done(); + }); + }); it('should give an error if the input does not have a valid prevTxId', function(done) { var badInput = { prevTxId: 'bad' @@ -43,7 +98,7 @@ describe('Bitcoin Transaction', function() { it('if an error happened it should pass it along', function(done) { var tx = new Transaction(); var db = { - getTransaction: sinon.stub().callsArgWith(2, new Error('error')) + getTransaction: sinon.stub().callsArgWith(1, new Error('error')) }; tx._populateInput(db, input, [], function(err) { should.exist(err); @@ -54,7 +109,7 @@ describe('Bitcoin Transaction', function() { it('should return an error if the transaction for the input does not exist', function(done) { var tx = new Transaction(); var db = { - getTransaction: sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()) + getTransaction: sinon.stub().callsArgWith(1, null, null) }; tx._populateInput(db, input, [], function(err) { should.exist(err); @@ -65,7 +120,7 @@ describe('Bitcoin Transaction', function() { it('should look through poolTransactions if database does not have transaction', function(done) { var tx = new Transaction(); var db = { - getTransaction: sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()) + getTransaction: sinon.stub().callsArgWith(1, null, null) }; var transactions = [ { @@ -79,12 +134,12 @@ describe('Bitcoin Transaction', function() { done(); }); }); - it('should not return an error if an error did not occur', function(done) { + it('should set the output on the input', function(done) { var prevTx = new Transaction(); prevTx.outputs = ['output']; var tx = new Transaction(); var db = { - getTransaction: sinon.stub().callsArgWith(2, null, prevTx) + getTransaction: sinon.stub().callsArgWith(1, null, prevTx) }; tx._populateInput(db, input, [], function(err) { should.not.exist(err); @@ -94,27 +149,4 @@ describe('Bitcoin Transaction', function() { }); }); - describe('#_checkSpent', function() { - it('should return an error if input was spent', function(done) { - var tx = new Transaction(); - var db = { - isSpentDB: sinon.stub().callsArgWith(1, true) - }; - tx._checkSpent(db, [], 'input', function(err) { - should.exist(err); - err.message.should.equal('Input already spent'); - done(); - }); - }); - it('should not return an error if input was unspent', function(done) { - var tx = new Transaction(); - var db = { - isSpentDB: sinon.stub().callsArgWith(1, false) - }; - tx._checkSpent(db, [], 'input', function(err) { - should.not.exist(err); - done(); - }); - }); - }); }); From 69ff5423c26cdf249490c7b5f7b4bebab4563414 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Apr 2016 10:29:35 -0400 Subject: [PATCH 108/299] bitcoind: rename exported events to rawtransaction and hashblock --- docs/services/bitcoind.md | 12 +++---- lib/services/bitcoind.js | 24 ++++++------- regtest/node.js | 6 ++-- test/services/bitcoind.unit.js | 62 +++++++++++++++++----------------- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index d535d52b..8846a53c 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -297,20 +297,20 @@ node.services.bitcoind.on('block', function(blockHash) { ``` For details on instantiating a bus for a node, see the [Bus Documentation](../bus.md). -- Name: `bitcoind/transaction`, Arguments: `[address, address...]` -- Name: `bitcoind/balance`, Arguments: `[address, address...]` +- Name: `bitcoind/rawtransaction` +- Name: `bitcoind/hashblock` **Examples:** ```js -bus.subscribe('bitcoind/transaction', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); -bus.subscribe('bitcoind/balance', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); +bus.subscribe('bitcoind/rawtransaction'); +bus.subscribe('bitcoind/hashblock'); -bus.on('bitcoind/transaction', function(transaction) { +bus.on('bitcoind/rawtransaction', function(transactionHex) { //... }); -bus.on('bitcoind/balance', function(balance) { +bus.on('bitcoind/hashblock', function(blockhashHex) { //... }); ``` diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fe2d89d4..babf61da 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -44,8 +44,8 @@ function Bitcoin(options) { // event subscribers this.subscriptions = {}; - this.subscriptions.transaction = []; - this.subscriptions.block = []; + this.subscriptions.rawtransaction = []; + this.subscriptions.hashblock = []; // limits this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; @@ -146,16 +146,16 @@ Bitcoin.prototype.getAPIMethods = function() { Bitcoin.prototype.getPublishEvents = function() { return [ { - name: 'bitcoind/transaction', + name: 'bitcoind/rawtransaction', scope: this, - subscribe: this.subscribe.bind(this, 'transaction'), - unsubscribe: this.unsubscribe.bind(this, 'transaction') + subscribe: this.subscribe.bind(this, 'rawtransaction'), + unsubscribe: this.unsubscribe.bind(this, 'rawtransaction') }, { - name: 'bitcoind/block', + name: 'bitcoind/hashblock', scope: this, - subscribe: this.subscribe.bind(this, 'block'), - unsubscribe: this.unsubscribe.bind(this, 'block') + subscribe: this.subscribe.bind(this, 'hashblock'), + unsubscribe: this.unsubscribe.bind(this, 'hashblock') } ]; }; @@ -352,8 +352,8 @@ Bitcoin.prototype._zmqBlockHandler = function(node, message) { self.zmqKnownBlocks.set(id, true); self.emit('block', message); - for (var i = 0; i < this.subscriptions.block.length; i++) { - this.subscriptions.block[i].emit('bitcoind/block', message.toString('hex')); + for (var i = 0; i < this.subscriptions.hashblock.length; i++) { + this.subscriptions.hashblock[i].emit('bitcoind/hashblock', message.toString('hex')); } } @@ -421,8 +421,8 @@ Bitcoin.prototype._zmqTransactionHandler = function(node, message) { self.emit('tx', message); // Notify transaction subscribers - for (var i = 0; i < this.subscriptions.transaction.length; i++) { - this.subscriptions.transaction[i].emit('bitcoind/transaction', message.toString('hex')); + for (var i = 0; i < this.subscriptions.rawtransaction.length; i++) { + this.subscriptions.rawtransaction[i].emit('bitcoind/rawtransaction', message.toString('hex')); } } }; diff --git a/regtest/node.js b/regtest/node.js index 9e9e281f..7fa59ad4 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -199,9 +199,9 @@ describe('Node Functionality', function() { var bus = node.openBus(); var blockExpected; var blockReceived; - bus.subscribe('bitcoind/block'); - bus.on('bitcoind/block', function(data) { - bus.unsubscribe('bitcoind/block'); + bus.subscribe('bitcoind/hashblock'); + bus.on('bitcoind/hashblock', function(data) { + bus.unsubscribe('bitcoind/hashblock'); if (blockExpected) { data.should.be.equal(blockExpected); done(); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index f80b5f60..a994b9d9 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -73,8 +73,8 @@ describe('Bitcoin Service', function() { it('will set subscriptions', function() { var bitcoind = new BitcoinService(baseConfig); bitcoind.subscriptions.should.deep.equal({ - transaction: [], - block: [] + rawtransaction: [], + hashblock: [] }); }); }); @@ -100,11 +100,11 @@ describe('Bitcoin Service', function() { var events = bitcoind.getPublishEvents(); should.exist(events); events.length.should.equal(2); - events[0].name.should.equal('bitcoind/transaction'); + events[0].name.should.equal('bitcoind/rawtransaction'); events[0].scope.should.equal(bitcoind); events[0].subscribe.should.be.a('function'); events[0].unsubscribe.should.be.a('function'); - events[1].name.should.equal('bitcoind/block'); + events[1].name.should.equal('bitcoind/hashblock'); events[1].scope.should.equal(bitcoind); events[1].subscribe.should.be.a('function'); events[1].unsubscribe.should.be.a('function'); @@ -116,19 +116,19 @@ describe('Bitcoin Service', function() { var events = bitcoind.getPublishEvents(); events[0].subscribe('test'); - bitcoind.subscribe.args[0][0].should.equal('transaction'); + bitcoind.subscribe.args[0][0].should.equal('rawtransaction'); bitcoind.subscribe.args[0][1].should.equal('test'); events[0].unsubscribe('test'); - bitcoind.unsubscribe.args[0][0].should.equal('transaction'); + bitcoind.unsubscribe.args[0][0].should.equal('rawtransaction'); bitcoind.unsubscribe.args[0][1].should.equal('test'); events[1].subscribe('test'); - bitcoind.subscribe.args[1][0].should.equal('block'); + bitcoind.subscribe.args[1][0].should.equal('hashblock'); bitcoind.subscribe.args[1][1].should.equal('test'); events[1].unsubscribe('test'); - bitcoind.unsubscribe.args[1][0].should.equal('block'); + bitcoind.unsubscribe.args[1][0].should.equal('hashblock'); bitcoind.unsubscribe.args[1][1].should.equal('test'); }); }); @@ -137,12 +137,12 @@ describe('Bitcoin Service', function() { it('will push to subscriptions', function() { var bitcoind = new BitcoinService(baseConfig); var emitter = {}; - bitcoind.subscribe('block', emitter); - bitcoind.subscriptions.block[0].should.equal(emitter); + bitcoind.subscribe('hashblock', emitter); + bitcoind.subscriptions.hashblock[0].should.equal(emitter); var emitter2 = {}; - bitcoind.subscribe('transaction', emitter2); - bitcoind.subscriptions.transaction[0].should.equal(emitter2); + bitcoind.subscribe('rawtransaction', emitter2); + bitcoind.subscriptions.rawtransaction[0].should.equal(emitter2); }); }); @@ -154,19 +154,19 @@ describe('Bitcoin Service', function() { var emitter3 = {}; var emitter4 = {}; var emitter5 = {}; - bitcoind.subscribe('block', emitter1); - bitcoind.subscribe('block', emitter2); - bitcoind.subscribe('block', emitter3); - bitcoind.subscribe('block', emitter4); - bitcoind.subscribe('block', emitter5); - bitcoind.subscriptions.block.length.should.equal(5); - - bitcoind.unsubscribe('block', emitter3); - bitcoind.subscriptions.block.length.should.equal(4); - bitcoind.subscriptions.block[0].should.equal(emitter1); - bitcoind.subscriptions.block[1].should.equal(emitter2); - bitcoind.subscriptions.block[2].should.equal(emitter4); - bitcoind.subscriptions.block[3].should.equal(emitter5); + bitcoind.subscribe('hashblock', emitter1); + bitcoind.subscribe('hashblock', emitter2); + bitcoind.subscribe('hashblock', emitter3); + bitcoind.subscribe('hashblock', emitter4); + bitcoind.subscribe('hashblock', emitter5); + bitcoind.subscriptions.hashblock.length.should.equal(5); + + bitcoind.unsubscribe('hashblock', emitter3); + bitcoind.subscriptions.hashblock.length.should.equal(4); + bitcoind.subscriptions.hashblock[0].should.equal(emitter1); + bitcoind.subscriptions.hashblock[1].should.equal(emitter2); + bitcoind.subscriptions.hashblock[2].should.equal(emitter4); + bitcoind.subscriptions.hashblock[3].should.equal(emitter5); }); }); @@ -475,8 +475,8 @@ describe('Bitcoin Service', function() { var message = new Buffer('00000000002e08fc7ae9a9aa5380e95e2adcdc5752a4a66a7d3a22466bd4e6aa', 'hex'); bitcoind._rapidProtectedUpdateTip = sinon.stub(); var emitter = new EventEmitter(); - bitcoind.subscriptions.block.push(emitter); - emitter.on('bitcoind/block', function(blockHash) { + bitcoind.subscriptions.hashblock.push(emitter); + emitter.on('bitcoind/hashblock', function(blockHash) { blockHash.should.equal(message.toString('hex')); done(); }); @@ -625,8 +625,8 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var expectedBuffer = new Buffer('abcdef', 'hex'); var emitter = new EventEmitter(); - bitcoind.subscriptions.transaction.push(emitter); - emitter.on('bitcoind/transaction', function(hex) { + bitcoind.subscriptions.rawtransaction.push(emitter); + emitter.on('bitcoind/rawtransaction', function(hex) { hex.should.be.a('string'); hex.should.equal(expectedBuffer.toString('hex')); done(); @@ -638,8 +638,8 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var expectedBuffer = new Buffer('abcdef', 'hex'); var emitter = new EventEmitter(); - bitcoind.subscriptions.transaction.push(emitter); - emitter.on('bitcoind/transaction', function() { + bitcoind.subscriptions.rawtransaction.push(emitter); + emitter.on('bitcoind/rawtransaction', function() { done(); }); var node = {}; From 24ca5ce053fcc21fff92dd50865e583bed5866f7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Apr 2016 10:44:56 -0400 Subject: [PATCH 109/299] web: option to enable/disable socket rpc handling --- lib/services/web.js | 16 +++++++++++++--- test/services/web.unit.js | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/lib/services/web.js b/lib/services/web.js index af773b87..5f809806 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -1,15 +1,19 @@ 'use strict'; +var fs = require('fs'); var http = require('http'); var https = require('https'); var express = require('express'); var bodyParser = require('body-parser'); var socketio = require('socket.io'); -var BaseService = require('../service'); var inherits = require('util').inherits; + +var BaseService = require('../service'); +var bitcore = require('bitcore-lib'); +var _ = bitcore.deps._; var index = require('../'); var log = index.log; -var fs = require('fs'); + /** * This service represents a hub for combining several services over a single HTTP port. Services @@ -32,6 +36,9 @@ var WebService = function(options) { this.httpsOptions = options.httpsOptions || this.node.httpsOptions; this.port = options.port || this.node.port || 3456; + this.enableSocketRPC = _.isUndefined(options.enableSocketRPC) ? + WebService.DEFAULT_SOCKET_RPC : options.enableSocketRPC; + this.node.on('ready', function() { self.eventNames = self.getEventNames(); self.setupAllRoutes(); @@ -43,6 +50,7 @@ var WebService = function(options) { inherits(WebService, BaseService); WebService.dependencies = []; +WebService.DEFAULT_SOCKET_RPC = true; /** * Called by Node to start the service @@ -157,7 +165,9 @@ WebService.prototype.getEventNames = function() { WebService.prototype.socketHandler = function(socket) { var bus = this.node.openBus(); - socket.on('message', this.socketMessageHandler.bind(this)); + if (this.enableSocketRPC) { + socket.on('message', this.socketMessageHandler.bind(this)); + } socket.on('subscribe', function(name, params) { bus.subscribe(name, params); diff --git a/test/services/web.unit.js b/test/services/web.unit.js index 4bc5760f..5415af2d 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -36,6 +36,19 @@ var WebService = proxyquire('../../lib/services/web', {http: httpStub, https: ht describe('WebService', function() { var defaultNode = new EventEmitter(); + describe('@constructor', function() { + it('will set socket rpc settings', function() { + var web = new WebService({node: defaultNode, enableSocketRPC: false}); + web.enableSocketRPC.should.equal(false); + + var web2 = new WebService({node: defaultNode, enableSocketRPC: true}); + web2.enableSocketRPC.should.equal(true); + + var web3 = new WebService({node: defaultNode}); + web3.enableSocketRPC.should.equal(WebService.DEFAULT_SOCKET_RPC); + }); + }); + describe('#start', function() { beforeEach(function() { httpStub.createServer.reset(); @@ -236,6 +249,19 @@ describe('WebService', function() { socket.emit('message', 'data'); }); + it('on message should NOT call socketMessageHandler if not enabled', function(done) { + web = new WebService({node: node, enableSocketRPC: false}); + web.eventNames = web.getEventNames(); + web.socketMessageHandler = sinon.stub(); + socket = new EventEmitter(); + web.socketHandler(socket); + socket.on('message', function() { + web.socketMessageHandler.callCount.should.equal(0); + done(); + }); + socket.emit('message', 'data'); + }); + it('on subscribe should call bus.subscribe', function(done) { bus.subscribe = function(param1) { param1.should.equal('data'); From bb726bac8b9ee845add38a07915261ceb5c675a4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Apr 2016 16:35:31 -0400 Subject: [PATCH 110/299] test: bitcoind getaddressbalance unit test --- test/services/bitcoind.unit.js | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index a994b9d9..b638d1ff 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -882,6 +882,43 @@ describe('Bitcoin Service', function() { }); describe('#getAddressBalance', function() { + it('will give rpc error', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressBalance: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) + } + }); + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + var options = {}; + bitcoind.getAddressBalance(address, options, function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + it('will give balance', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressBalance: sinon.stub().callsArgWith(1, null, { + result: { + received: 100000, + balance: 10000 + } + }) + } + }); + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + var options = {}; + bitcoind.getAddressBalance(address, options, function(err, data) { + if (err) { + return done(err); + } + data.balance.should.equal(10000); + data.received.should.equal(100000); + done(); + }); + }); }); describe('#getAddressUnspentOutputs', function() { From dab49aef399ac41d18caa3361887f3ed6464e85c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Apr 2016 16:51:56 -0400 Subject: [PATCH 111/299] docs: various updates - remove build and update bitcoind - remove outdated error documentation - update bus docs --- README.md | 1 - docs/build.md | 117 -------------------------------------- docs/bus.md | 8 +-- docs/errors.md | 16 ------ docs/services/bitcoind.md | 2 +- 5 files changed, 5 insertions(+), 139 deletions(-) delete mode 100644 docs/build.md delete mode 100644 docs/errors.md diff --git a/README.md b/README.md index 4b87327e..9bf4f4fc 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ There are several add-on services available to extend the functionality of Bitco - [Testing & Development](docs/testing.md) - Developer guide for testing - [Node](docs/node.md) - Details on the node constructor - [Bus](docs/bus.md) - Overview of the event bus constructor -- [Errors](docs/errors.md) - Reference for error handling and types - [Release Process](docs/release.md) - Information about verifying a release and the release process. ## Contributing diff --git a/docs/build.md b/docs/build.md deleted file mode 100644 index d22b8411..00000000 --- a/docs/build.md +++ /dev/null @@ -1,117 +0,0 @@ -# Build & Install -This includes a detailed instructions for compiling. There are two main parts of the build, compiling Bitcoin Core as a static library and the Node.js bindings. - -## Ubuntu 14.04 (Unix/Linux) -If git is not already installed, it can be installed by running: - -```bash -sudo apt-get install git -``` - -If Node.js v0.12 isn't installed, it can be installed using "nvm", it can be done by following the installation script at [https://github.com/creationix/nvm#install-script](https://github.com/creationix/nvm#install-script) and then install version v0.12 - -```bash -nvm install v0.12 -``` - -To build Bitcoin Core and bindings development packages are needed: - -```bash -sudo apt-get install build-essential libtool autotools-dev automake autoconf pkg-config libssl-dev -``` - -Clone the bitcore-node repository locally: - -```bash -git clone https://github.com/bitpay/bitcore-node.git -cd bitcore-node -``` - -And finally run the build which will take several minutes. A script in the "bin" directory will download Bitcoin Core v0.11, apply a patch (see more info below), and compile the static library and Node.js bindings. You can start this by running: - -```bash -npm install -``` - -Once everything is built, you can run bitcore-node via: - -```bash -npm start -``` - -This will then start the syncing process for Bitcoin Core and the extended capabilities as provided by the built-in Address Module (details below). - -## Fedora -Later versions of Fedora (>= 22) should also work with this project. The directions for Ubuntu should generally work except the installation of system utilities and libraries is a bit different. Git is already installed and ready for use without installation. - -```bash -yum install libtool automake autoconf pkgconfig openssl make gcc gcc-c++ kernel-devel openssl-devel.x86_64 patch -``` - -## Mac OS X Yosemite -If Xcode is not already installed, it can be installed via the Mac App Store (will take several minutes). XCode includes "Clang", "git" and other build tools. Once Xcode is installed, you'll then need to install "xcode-select" via running in a terminal and following the prompts: - -```bash -xcode-select --install -``` - -If "Homebrew" is not yet installed, it's needed to install "autoconf" and others. You can install it using the script at [http://brew.sh](http://brew.sh) and following the directions at [https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/Installation.md](https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/Installation.md) And then run in a terminal: - -```bash -brew install autoconf automake libtool openssl pkg-config -``` - -If Node.js v0.12 and associated commands "node", "npm" and "nvm" are not already installed, you can use "nvm" by running the script at [https://github.com/creationix/nvm#install-script](https://github.com/creationix/nvm#install-script) And then run this command to install Node.js v0.12 - -```bash -nvm install v0.12 -``` - -Clone the bitcore-node repository locally: - -```bash -git clone https://github.com/bitpay/bitcore-node.git -cd bitcore-node -``` - -And finally run the build which will take several minutes. A script in the "bin" directory will download Bitcoin Core v0.11, apply a patch (see more info below), and compile the static library and Node.js bindings. You can start this by running: - -```bash -npm install -``` - -## Cross Compilation -If you desire to cross compile to ARM or Windows from a system that has cross compilation tools available for use, please use the following directions: - -Using a Debian (Jessie) system as the host system (the system that will be doing the compiling): - -```bash -echo -n "deb http://emdebian.org/tools/debian/ jessie main" | sudo tee -a /etc/apt/sources.list -sudo dpkg --add-architecture armhf #or whatever arch you are interested in compiling for -sudo apt-get update #you will get GPG KEY warnings, you can decide if you would like to trust the key -sudo apt-get install crossbuild-essential-armhf -``` - -Next is to use the cross compilation toolchain instead of the defaults: - -```bash -CXX=arm-linux-gnueabihf-g++ CC=arm-linux-gnueabihf-gcc npm install -``` - -The only thing different is the setting of CC/CXX environment variables. Please make sure those compilers (arm-linux-gnueabihf-gcc) actually exist and are on your path. - -```bash -arm-linux-gnueabihf-g++ -v -arm-linux-gnueabihf-gcc -v -``` - -You should get output with the last line ending with something like this: -gcc version 4.9.2 ( 4.9.2-10) - -Once everything is built, you can run bitcore-node via: - -```bash -npm start -``` - -This will then start the syncing process for Bitcoin Core and the extended capabilities as provided by the built-in Address Module (details below). diff --git a/docs/bus.md b/docs/bus.md index 153b9a9c..8746d682 100644 --- a/docs/bus.md +++ b/docs/bus.md @@ -20,11 +20,11 @@ bus.close(); ```javascript // subscribe to all transaction events -bus.subscribe('db/transaction'); +bus.subscribe('bitcoind/rawtransaction'); -// only subscribe to events relevant to a bitcoin address -bus.subscribe('address/transaction', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); +// to subscribe to new block hashes +bus.subscribe('bitcoind/hashblock'); // unsubscribe -bus.unsubscribe('db/transaction'); +bus.unsubscribe('bitcoind/rawtransaction'); ``` diff --git a/docs/errors.md b/docs/errors.md deleted file mode 100644 index e033466b..00000000 --- a/docs/errors.md +++ /dev/null @@ -1,16 +0,0 @@ -# Errors -Many times there are cases where an error condition can be gracefully handled depending on a particular use. To assist in better error handling, errors will have different types so that it's possible to determine the type of error and handle appropriately. - -```js -node.services.address.getUnspentOutputs('00000000839a8...', function(err, outputs) { - - if (err instanceof errors.NoOutputs) { - // the address hasn't received any transactions - } - - // otherwise the address has outputs (which may be unspent/spent) - -}); -``` - -For more information about different types of errors, please see `lib/errors.js`. diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index 8846a53c..8c49624a 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -292,7 +292,7 @@ node.services.bitcoind.on('tx', function(transactionBuffer) { }); node.services.bitcoind.on('block', function(blockHash) { - // a new transaction has left the mempool + // a new block has been added }); ``` From 552abf77cfc2160736c0b71fa2480c90983d111c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 15 Apr 2016 17:00:03 -0400 Subject: [PATCH 112/299] docs: symlink docs/index.md -> README.md --- docs/index.md | 55 +-------------------------------------------------- 1 file changed, 1 insertion(+), 54 deletions(-) mode change 100644 => 120000 docs/index.md diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index aec8f91c..00000000 --- a/docs/index.md +++ /dev/null @@ -1,54 +0,0 @@ -A Bitcoin full node for building applications and services with Node.js. A node is extensible and can be configured to run additional services. At the minimum a node has native bindings to Bitcoin Core with the [Bitcoin Service](services/bitcoind.md). Additional services can be enabled to make a node more useful such as exposing new APIs, adding new indexes for addresses with the [Address Service](services/address.md), running a block explorer, wallet service, and other customizations. - -# Install - -```bash -npm install -g bitcore-node -bitcore-node start -``` - -Note: For your convenience, we distribute binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the [Build & Install](build.md) documentation to build the project from source. - -# Prerequisites -- Node.js v0.12 or v4.2 -- ~100GB of disk storage -- ~4GB of RAM -- Mac OS X >= 10.9, Ubuntu >= 12.04 (libc >= 2.15 and libstdc++ >= 6.0.16) - -# Configuration -Bitcore includes a Command Line Interface (CLI) for managing, configuring and interfacing with your Bitcore Node. - -```bash -bitcore-node create -d mynode -cd mynode -bitcore-node install -bitcore-node install https://github.com/yourname/helloworld -``` - -This will create a directory with configuration files for your node and install the necessary dependencies. For more information about (and developing) services, please see the [Service Documentation](services.md). - -To start bitcore as a daemon: - -```bash -bitcore start --daemon -``` - -# Add-on Services -There are several add-on services available to extend the functionality of Bitcore Node: -- [Insight API](https://github.com/bitpay/insight-api/tree/v0.3.0) -- [Insight UI](https://github.com/bitpay/insight/tree/v0.3.0) - -# Documentation -- [Services](services.md) - - [Bitcoind](services/bitcoind.md) - Native bindings to Bitcoin Core - - [Database](services/db.md) - The foundation API methods for getting information about blocks and transactions. - - [Address](services/address.md) - Adds additional API methods for querying and subscribing to events with bitcoin addresses. - - [Web](services/web.md) - Creates an express application over which services can expose their web/API content - -- [Build & Install](build.md) - How to build and install from source -- [Testing & Development](testing.md) - Developer guide for testing -- [Node](node.md) - Details on the node constructor -- [Bus](bus.md) - Overview of the event bus constructor -- [Errors](errors.md) - Reference for error handling and types -- [Patch](patch.md) - Information about the patch applied to Bitcoin Core -- [Release Process](release.md) - Information about verifying a release and the release process. diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file From a4f5a6fa829f069dc585cf757368ba6948ac7b53 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 10:37:33 -0400 Subject: [PATCH 113/299] test: getblock unit tests --- lib/services/bitcoind.js | 52 +++++++------- test/services/bitcoind.unit.js | 128 +++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 26 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index babf61da..df5f5ab0 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1160,41 +1160,41 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { var self = this; function queryBlock(blockhash) { - self._tryAll(function(done) { - self.client.getBlock(blockhash, false, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var blockObj = bitcore.Block.fromString(response.result); - self.blockCache.set(blockhash, blockObj); - done(null, blockObj); + var cachedBlock = self.blockCache.get(blockhash); + if (cachedBlock) { + return setImmediate(function() { + callback(null, cachedBlock); }); - }, callback); - } - - var cachedBlock = self.blockCache.get(blockArg); - if (cachedBlock) { - return setImmediate(function() { - callback(null, cachedBlock); - }); - } else { - if (_.isNumber(blockArg)) { + } else { self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { + self.client.getBlock(blockhash, false, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } - done(null, response.result); + var blockObj = bitcore.Block.fromString(response.result); + self.blockCache.set(blockhash, blockObj); + done(null, blockObj); }); - }, function(err, blockhash) { + }, callback); + } + } + + if (_.isNumber(blockArg)) { + self._tryAll(function(done) { + self.client.getBlockHash(blockArg, function(err, response) { if (err) { - return callback(err); + return done(self._wrapRPCError(err)); } - queryBlock(blockhash); + done(null, response.result); }); - } else { - queryBlock(blockArg); - } + }, function(err, blockhash) { + if (err) { + return callback(err); + } + queryBlock(blockhash); + }); + } else { + queryBlock(blockArg); } }; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index b638d1ff..85573427 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1760,6 +1760,134 @@ describe('Bitcoin Service', function() { }); describe('#getBlock', function() { + var blockhex = '0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000'; + it('will give an rpc error from client getblock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, {code: -1, message: 'Test error'}); + var getBlockHash = sinon.stub().callsArgWith(1, null, {}); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlock(0, function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + it('will give an rpc error from client getblockhash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind.getBlock(0, function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + it('will getblock as bitcore object from height', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b' + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlock(0, function(err, block) { + should.not.exist(err); + getBlock.args[0][0].should.equal('00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'); + getBlock.args[0][1].should.equal(false); + block.should.be.instanceof(bitcore.Block); + done(); + }); + }); + it('will getblock as bitcore object', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlock('00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b', function(err, block) { + should.not.exist(err); + getBlockHash.callCount.should.equal(0); + getBlock.callCount.should.equal(1); + getBlock.args[0][0].should.equal('00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'); + getBlock.args[0][1].should.equal(false); + block.should.be.instanceof(bitcore.Block); + done(); + }); + }); + it('will get block from cache', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + var hash = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + bitcoind.getBlock(hash, function(err, block) { + should.not.exist(err); + getBlockHash.callCount.should.equal(0); + getBlock.callCount.should.equal(1); + block.should.be.instanceof(bitcore.Block); + bitcoind.getBlock(hash, function(err, block) { + should.not.exist(err); + getBlockHash.callCount.should.equal(0); + getBlock.callCount.should.equal(1); + block.should.be.instanceof(bitcore.Block); + done(); + }); + }); + }); + it('will get block from cache with height (but not height)', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockhex + }); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b' + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlock(0, function(err, block) { + should.not.exist(err); + getBlockHash.callCount.should.equal(1); + getBlock.callCount.should.equal(1); + block.should.be.instanceof(bitcore.Block); + bitcoind.getBlock(0, function(err, block) { + should.not.exist(err); + getBlockHash.callCount.should.equal(2); + getBlock.callCount.should.equal(1); + block.should.be.instanceof(bitcore.Block); + done(); + }); + }); + }); }); describe('#getBlockHashesByTimestamp', function() { From 8fd405eedf85b01b3be6f9057808c049d76dfce7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 10:48:32 -0400 Subject: [PATCH 114/299] test: getBlockHeader unit tests --- test/services/bitcoind.unit.js | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 85573427..9945ac95 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1926,6 +1926,73 @@ describe('Bitcoin Service', function() { }); describe('#getBlockHeader', function() { + var blockhash = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + it('it will give rpc error from client getblockheader', function() { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHeader = sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}); + bitcoind.nodes.push({ + client: { + getBlockHeader: getBlockHeader + } + }); + bitcoind.getBlockHeader(blockhash, function(err) { + err.should.be.instanceof(Error); + }); + }); + it('it will give rpc error from client getblockhash', function() { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHeader = sinon.stub(); + var getBlockHash = sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}); + bitcoind.nodes.push({ + client: { + getBlockHeader: getBlockHeader, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlockHeader(0, function(err) { + err.should.be.instanceof(Error); + }); + }); + it('will give result from client getblockheader (from height)', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + var getBlockHeader = sinon.stub().callsArgWith(1, null, { + result: result + }); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: blockhash + }); + bitcoind.nodes.push({ + client: { + getBlockHeader: getBlockHeader, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlockHeader(0, function(err, blockHeader) { + should.not.exist(err); + getBlockHeader.args[0][0].should.equal(blockhash); + blockHeader.should.equal(result); + }); + }); + it('will give result from client getblockheader (from hash)', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + var getBlockHeader = sinon.stub().callsArgWith(1, null, { + result: result + }); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlockHeader: getBlockHeader, + getBlockHash: getBlockHash + } + }); + bitcoind.getBlockHeader(blockhash, function(err, blockHeader) { + should.not.exist(err); + getBlockHash.callCount.should.equal(0); + blockHeader.should.equal(result); + }); + }); }); describe('#estimateFee', function() { From 317fdbbdd8208cc9a22a694e58f77b8edf2434f8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 10:54:50 -0400 Subject: [PATCH 115/299] test: bitcoind _getAddressStrings unit tests --- test/services/bitcoind.unit.js | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 9945ac95..658eba0d 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1410,6 +1410,46 @@ describe('Bitcoin Service', function() { }); describe('#_getAddressStrings', function() { + it('will get address strings from bitcore addresses', function() { + var addresses = [ + bitcore.Address('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'), + bitcore.Address('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou'), + ]; + var bitcoind = new BitcoinService(baseConfig); + var strings = bitcoind._getAddressStrings(addresses); + strings[0].should.equal('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'); + strings[1].should.equal('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou'); + }); + it('will get address strings from strings', function() { + var addresses = [ + '1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i', + '3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', + ]; + var bitcoind = new BitcoinService(baseConfig); + var strings = bitcoind._getAddressStrings(addresses); + strings[0].should.equal('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'); + strings[1].should.equal('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou'); + }); + it('will get address strings from mixture of types', function() { + var addresses = [ + bitcore.Address('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'), + '3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', + ]; + var bitcoind = new BitcoinService(baseConfig); + var strings = bitcoind._getAddressStrings(addresses); + strings[0].should.equal('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'); + strings[1].should.equal('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou'); + }); + it('will give error with unknown', function() { + var addresses = [ + bitcore.Address('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i'), + 0, + ]; + var bitcoind = new BitcoinService(baseConfig); + (function() { + bitcoind._getAddressStrings(addresses); + }).should.throw(TypeError); + }); }); describe('#_paginateTxids', function() { From e09cc3d1fc6e273e13c8c8f9e275e4c46e024a5c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 11:11:17 -0400 Subject: [PATCH 116/299] test: bitcoind start unit tests --- test/services/bitcoind.unit.js | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 658eba0d..62829e4d 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -773,6 +773,73 @@ describe('Bitcoin Service', function() { }); describe('#start', function() { + it('will give error if "spawn" and "connect" are both not configured', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.options = {}; + bitcoind.start(function(err) { + err.should.be.instanceof(Error); + err.message.should.match(/Bitcoin configuration options/); + }); + done(); + }); + it('will give error from spawnChildProcess', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._spawnChildProcess = sinon.stub().callsArgWith(0, new Error('test')); + bitcoind.options = { + spawn: {} + }; + bitcoind.start(function(err) { + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will give error from connectProcess', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._connectProcess = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind.options = { + connect: [ + {} + ] + }; + bitcoind.start(function(err) { + bitcoind._connectProcess.callCount.should.equal(1); + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will push node from spawnChildProcess', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + bitcoind._initChain = sinon.stub().callsArg(0); + bitcoind._spawnChildProcess = sinon.stub().callsArgWith(0, null, node); + bitcoind.options = { + spawn: {} + }; + bitcoind.start(function(err) { + should.not.exist(err); + bitcoind.nodes.length.should.equal(1); + done(); + }); + }); + it('will push node from connectProcess', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._initChain = sinon.stub().callsArg(0); + var nodes = [{}]; + bitcoind._connectProcess = sinon.stub().callsArgWith(1, null, nodes); + bitcoind.options = { + connect: [ + {} + ] + }; + bitcoind.start(function(err) { + should.not.exist(err); + bitcoind._connectProcess.callCount.should.equal(1); + bitcoind.nodes.length.should.equal(1); + done(); + }); + }); }); describe('#isSynced', function() { From 7c37eba91e271557a0f00bcaf3b6bf687b28f910 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 13:08:57 -0400 Subject: [PATCH 117/299] test: unit tests for connect and spawn processes --- lib/services/bitcoind.js | 10 +- test/services/bitcoind.unit.js | 237 ++++++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 7 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index df5f5ab0..58de911f 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -52,6 +52,7 @@ function Bitcoin(options) { // try all interval this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; + this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; // available bitcoind nodes this._initClients(); @@ -62,6 +63,7 @@ Bitcoin.dependencies = []; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; +Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_CONFIG_SETTINGS = { server: 1, whitelist: '127.0.0.1', @@ -528,7 +530,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { } var options = [ - '--conf=' + path.resolve(this.spawn.configPath), + '--conf=' + this.spawn.configPath, '--datadir=' + this.spawn.datadir, ]; @@ -541,7 +543,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { log.error(err); }); - async.retry({times: 60, interval: 5000}, function(done) { + async.retry({times: 60, interval: self.startRetryInterval}, function(done) { if (self.node.stopping) { return done(new Error('Stopping while trying to connect to bitcoind.')); } @@ -563,7 +565,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); - self._checkReindex(node, function() { + self._checkReindex(node, function(err) { if (err) { return callback(err); } @@ -578,7 +580,7 @@ Bitcoin.prototype._connectProcess = function(config, callback) { var self = this; var node = {}; - async.retry({times: 60, interval: 5000}, function(done) { + async.retry({times: 60, interval: self.startRetryInterval}, function(done) { if (self.node.stopping) { return done(new Error('Stopping while trying to connect to bitcoind.')); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 62829e4d..dddad6bb 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -264,12 +264,12 @@ describe('Bitcoin Service', function() { }); describe('#_checkConfigIndexes', function() { - var stub; + var sandbox = sinon.sandbox.create(); beforeEach(function() { - stub = sinon.stub(log, 'warn'); + sandbox.stub(log, 'warn'); }); after(function() { - stub.restore(); + sandbox.restore(); }); it('should warn the user if reindex is set to 1 in the bitcoin.conf file', function() { var bitcoind = new BitcoinService(baseConfig); @@ -764,12 +764,243 @@ describe('Bitcoin Service', function() { }); describe('#_loadTipFromNode', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'warn'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('will give rpc from client getbestblockhash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, {code: -1, message: 'Test error'}); + var node = { + client: { + getBestBlockHash: getBestBlockHash + } + }; + bitcoind._loadTipFromNode(node, function(err) { + err.should.be.instanceof(Error); + log.warn.callCount.should.equal(0); + done(); + }); + }); + it('will give rpc from client getblock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, { + result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' + }); + var getBlock = sinon.stub().callsArgWith(1, new Error('Test error')); + var node = { + client: { + getBestBlockHash: getBestBlockHash, + getBlock: getBlock + } + }; + bitcoind._loadTipFromNode(node, function(err) { + getBlock.args[0][0].should.equal('00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45'); + err.should.be.instanceof(Error); + log.warn.callCount.should.equal(0); + done(); + }); + }); + it('will log when error is RPC_IN_WARMUP', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, {code: -28, message: 'Verifying blocks...'}); + var node = { + client: { + getBestBlockHash: getBestBlockHash + } + }; + bitcoind._loadTipFromNode(node, function(err) { + err.should.be.instanceof(Error); + log.warn.callCount.should.equal(1); + done(); + }); + }); + it('will set height and emit tip', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, { + result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' + }); + var getBlock = sinon.stub().callsArgWith(1, null, { + result: { + height: 100 + } + }); + var node = { + client: { + getBestBlockHash: getBestBlockHash, + getBlock: getBlock + } + }; + bitcoind.on('tip', function(height) { + height.should.equal(100); + bitcoind.height.should.equal(100); + done(); + }); + bitcoind._loadTipFromNode(node, function(err) { + if (err) { + return done(err); + } + }); + }); }); describe('#_spawnChildProcess', function() { + it('will give error from spawn config', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._loadSpawnConfiguration = sinon.stub().throws(new Error('test')); + bitcoind._spawnChildProcess(function(err) { + err.should.be.instanceof(Error); + err.message.should.equal('test'); + done(); + }); + }); + it('will include network with spawn command and init zmq/rpc on node', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'testexec'; + bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.datadir = 'testdir'; + bitcoind.spawn.config = {}; + bitcoind.spawn.config.rpcport = 20001; + bitcoind.spawn.config.rpcuser = 'bitcoin'; + bitcoind.spawn.config.rpcpassword = 'password'; + bitcoind.spawn.config.zmqpubrawtx = 'tcp://127.0.0.1:30001'; + + bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._checkReindex = sinon.stub().callsArgWith(1, null); + bitcoind._spawnChildProcess(function(err, node) { + should.not.exist(err); + spawn.callCount.should.equal(1); + spawn.args[0][0].should.equal('testexec'); + spawn.args[0][1].should.deep.equal([ + '--conf=testdir/bitcoin.conf', + '--datadir=testdir', + '--testnet' + ]); + spawn.args[0][2].should.deep.equal({ + stdio: 'inherit' + }); + bitcoind._loadTipFromNode.callCount.should.equal(1); + bitcoind._initZmqSubSocket.callCount.should.equal(1); + should.exist(bitcoind._initZmqSubSocket.args[0][0].client); + bitcoind._initZmqSubSocket.args[0][1].should.equal('tcp://127.0.0.1:30001'); + bitcoind._subscribeZmqEvents.callCount.should.equal(1); + should.exist(bitcoind._subscribeZmqEvents.args[0][0].client); + should.exist(node); + should.exist(node.client); + done(); + }); + }); + it('will give error after 60 retries', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind.startRetryInterval = 1; + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'testexec'; + bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.datadir = 'testdir'; + bitcoind.spawn.config = {}; + bitcoind.spawn.config.rpcport = 20001; + bitcoind.spawn.config.rpcuser = 'bitcoin'; + bitcoind.spawn.config.rpcpassword = 'password'; + bitcoind.spawn.config.zmqpubrawtx = 'tcp://127.0.0.1:30001'; + bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind._spawnChildProcess(function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); + it('will give error from check reindex', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'testexec'; + bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.datadir = 'testdir'; + bitcoind.spawn.config = {}; + bitcoind.spawn.config.rpcport = 20001; + bitcoind.spawn.config.rpcuser = 'bitcoin'; + bitcoind.spawn.config.rpcpassword = 'password'; + bitcoind.spawn.config.zmqpubrawtx = 'tcp://127.0.0.1:30001'; + + bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._checkReindex = sinon.stub().callsArgWith(1, new Error('test')); + + bitcoind._spawnChildProcess(function(err) { + err.should.be.instanceof(Error); + done(); + }); + }); }); describe('#_connectProcess', function() { + it('will give error from loadTipFromNode after 60 retries', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind.startRetryInterval = 1; + var config = {}; + bitcoind._connectProcess(config, function(err) { + err.should.be.instanceof(Error); + bitcoind._loadTipFromNode.callCount.should.equal(60); + done(); + }); + }); + it('will init zmq/rpc on node', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); + var config = {}; + bitcoind._connectProcess(config, function(err, node) { + should.not.exist(err); + bitcoind._loadTipFromNode.callCount.should.equal(1); + bitcoind._initZmqSubSocket.callCount.should.equal(1); + bitcoind._loadTipFromNode.callCount.should.equal(1); + should.exist(node); + should.exist(node.client); + done(); + }); + }); }); describe('#start', function() { From afda35962b3513b93523367ef44c157c3ff0ca0d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 13:37:37 -0400 Subject: [PATCH 118/299] test: mempool helper method unit tests --- lib/services/bitcoind.js | 2 +- test/services/bitcoind.unit.js | 65 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 58de911f..73c1ab74 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -769,7 +769,7 @@ Bitcoin.prototype._getTxidsFromMempool = function(deltas) { }; Bitcoin.prototype._getHeightRangeQuery = function(options, clone) { - if (options.start >= 0 && options.end >=0) { + if (options.start >= 0 && options.end >= 0) { if (options.end > options.start) { throw new TypeError('"end" is expected to be less than or equal to "start"'); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index dddad6bb..fb2fc7ca 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1308,12 +1308,77 @@ describe('Bitcoin Service', function() { }); describe('#_getBalanceFromMempool', function() { + it('will sum satoshis', function() { + var bitcoind = new BitcoinService(baseConfig); + var deltas = [ + { + satoshis: -1000, + }, + { + satoshis: 2000, + }, + { + satoshis: -10, + } + ]; + var sum = bitcoind._getBalanceFromMempool(deltas); + sum.should.equal(990); + }); }); describe('#_getTxidsMempool', function() { + it('will filter to txids', function() { + var bitcoind = new BitcoinService(baseConfig); + var deltas = [ + { + txid: 'txid0', + }, + { + txid: 'txid1', + }, + { + txid: 'txid2', + } + ]; + var txids = bitcoind._getTxidsFromMempool(deltas); + txids.length.should.equal(3); + txids[0].should.equal('txid0'); + txids[1].should.equal('txid1'); + txids[2].should.equal('txid2'); + }); }); describe('#_getHeightRangeQuery', function() { + it('will detect range query', function() { + var bitcoind = new BitcoinService(baseConfig); + var options = { + start: 20, + end: 0 + }; + var rangeQuery = bitcoind._getHeightRangeQuery(options); + rangeQuery.should.equal(true); + }); + it('will get range properties', function() { + var bitcoind = new BitcoinService(baseConfig); + var options = { + start: 20, + end: 0 + }; + var clone = {}; + bitcoind._getHeightRangeQuery(options, clone); + clone.end.should.equal(20); + clone.start.should.equal(0); + }); + it('will throw error with invalid range', function() { + var bitcoind = new BitcoinService(baseConfig); + var options = { + start: 0, + end: 20 + }; + (function() { + bitcoind._getHeightRangeQuery(options); + }).should.throw('"end" is expected'); + }); }); describe('#getAddressTxids', function() { From c8ba4eaa8f9a77499d150540975ec9905f8a2d14 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 13:45:06 -0400 Subject: [PATCH 119/299] test: remove outdated regtest --- regtest/node.js | 78 ------------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/regtest/node.js b/regtest/node.js index 7fa59ad4..368fa2a6 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -116,84 +116,6 @@ describe('Node Functionality', function() { }); }); - var invalidatedBlockHash; - - it.skip('will handle a reorganization', function(done) { - - var count; - var blockHash; - - async.series([ - function(next) { - client.getBlockCount(function(err, response) { - if (err) { - return next(err); - } - count = response.result; - next(); - }); - }, - function(next) { - client.getBlockHash(count, function(err, response) { - if (err) { - return next(err); - } - invalidatedBlockHash = response.result; - next(); - }); - }, - function(next) { - client.invalidateBlock(invalidatedBlockHash, next); - }, - function(next) { - client.getBlockCount(function(err, response) { - if (err) { - return next(err); - } - response.result.should.equal(count - 1); - next(); - }); - } - ], function(err) { - if (err) { - throw err; - } - var blocksRemoved = 0; - var blocksAdded = 0; - - var removeBlock = function() { - blocksRemoved++; - }; - - node.services.db.on('removeblock', removeBlock); - - var addBlock = function() { - blocksAdded++; - if (blocksAdded === 2 && blocksRemoved === 1) { - node.services.db.removeListener('addblock', addBlock); - node.services.db.removeListener('removeblock', removeBlock); - done(); - } - }; - - node.services.db.on('addblock', addBlock); - - // We need to add a transaction to the mempool so that the next block will - // have a different hash as the hash has been invalidated. - client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { - if (err) { - throw err; - } - client.generate(2, function(err, response) { - if (err) { - throw err; - } - }); - }); - }); - - }); - describe('Bus Functionality', function() { it('subscribes and unsubscribes to an event on the bus', function(done) { var bus = node.openBus(); From 3fed348cf7fe32df06289750f6454a1ce8c86812 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 15:27:12 -0400 Subject: [PATCH 120/299] docs: update development guide --- README.md | 2 +- docs/development.md | 156 ++++++++++++++++++++++++++++++++++++++++++++ docs/testing.md | 37 ----------- 3 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 docs/development.md delete mode 100644 docs/testing.md diff --git a/README.md b/README.md index 9bf4f4fc..ce74df1b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ There are several add-on services available to extend the functionality of Bitco - [Services](docs/services.md) - [Bitcoind](docs/services/bitcoind.md) - Interface to Bitcoin Core - [Web](docs/services/web.md) - Creates an express application over which services can expose their web/API content -- [Testing & Development](docs/testing.md) - Developer guide for testing +- [Development Environment](docs/development.md) - Guide for setting up a development environment - [Node](docs/node.md) - Details on the node constructor - [Bus](docs/bus.md) - Overview of the event bus constructor - [Release Process](docs/release.md) - Information about verifying a release and the release process. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..7c867a72 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,156 @@ +# Setting up Development Environment + +## Install Node.js + +Install Node.js by your favorite method, or use Node Version Manager by following directions at https://github.com/creationix/nvm + +```bash +nvm install v4 +``` + +## Fork and Download Repositories + +To develop bitcore-node: + +```bash +cd ~ +git clone git@github.com:/bitcore-node.git +git clone git@github.com:/bitcore-lib.git +``` + +To develop bitcoin or to compile from source: + +```bash +git clone git@github.com:/bitcoin.git +git fetch origin : +git checkout +``` +**Note**: See bitcoin documentation for building bitcoin on your platform. + + +## Install ZeroMQ Development Dependency + +For Ubuntu: +```bash +sudo apt-get install libzmq-dev +``` + +For Mac OS X: +```bash +brew install zeromq +``` + +## Install and Symlink + +```bash +cd bitcore-lib +npm install +cd ../bitcore-node +npm install +``` +**Note**: If you get a message about not being able to download bitcoin distribution, you'll need to compile bitcoind from source, and setup your configuration to use that version. + + +We now will setup symlinks in `bitcore-node` *(repeat this for any other modules you're planning on developing)*: +```bash +cd node_modules +rm -rf bitcore-lib +ln -s ~/bitcore-lib +rm -rf bitcoind-rpc +ln -s ~/bitcoind-rpc +``` + +And if you're compiling or developing bitcoin: +```bash +cd ../bin +ln -sf ~/bitcoin/src/bitcoind +``` + +## Run Tests + +If you do not already have mocha installed: +```bash +npm install mocha -g +``` + +To run all test suites: +```bash +cd bitcore-node +npm run regtest +npm run test +``` + +To run a specific unit test in watch mode: +```bash +mocha -w -R spec test/services/bitcoind.unit.js +``` + +To run a specific regtest: +```bash +mocha -R spec regtest/bitcoind.js +``` + +## Running a Development Node + +To test running the node, you can setup a configuration that will specify development versions of all of the services: + +```bash +cd ~ +mkdir devnode +cd devnode +mkdir node_modules +touch bitcore-node.json +touch package.json +``` + +Edit `bitcore-node.json` with something similar to: +```json +{ + "network": "livenet", + "port": 3001, + "services": [ + "bitcoind", + "web", + "insight-api", + "insight-ui" + ], + "servicesConfig": { + "bitcoind": { + "spawn": { + "datadir": "/home//.bitcoin", + "exec": "/home//bitcoin/src/bitcoind" + } + } + } +} +``` + +Setup symlinks for all of the services and dependencies: + +```bash +cd node_modules +ln -s ~/bitcore-lib +ln -s ~/bitcore-node +ln -s ~/insight-api +ln -s ~/insight-ui +``` + +Make sure that the `/bitcoin.conf` has the necessary settings, for example: +``` +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +spentindex=1 +zmqpubrawtx=tcp://127.0.0.1:28332 +zmqpubhashblock=tcp://127.0.0.1:28332 +rpcallowip=127.0.0.1 +rpcuser=bitcoin +rpcpassword=local321 +``` + +From within the `devnode` directory with the configuration file, start the node: +```bash +../bitcore-node/bin/bitcore-node start +``` \ No newline at end of file diff --git a/docs/testing.md b/docs/testing.md deleted file mode 100644 index a1a6844e..00000000 --- a/docs/testing.md +++ /dev/null @@ -1,37 +0,0 @@ -# Development & Testing -To run all of the JavaScript tests: - -```bash -npm run test -``` - -If you do not already have mocha installed: - -```bash -npm install mocha -g -``` - -To run the regression tests: - -```bash -mocha -R spec regtest/bitcoind.js -``` - -To be able to debug bitcoind you'll need to have `gdb` and `node` compiled for debugging with gdb using `--gdb` (sometimes called node_g), and you can then run: - -```bash -$ gdb --args node examples/node.js -``` - -To run mocha from within gdb (notice `_mocha` and not `mocha` so that the tests run in the same process): - -```bash -$ gdb --args node /path/to/_mocha -R spec integration/regtest.js -``` - -To run the benchmarks: - -```bash -$ cd benchmarks -$ node index.js -``` From bf67b932de1eb3eedc1286deb7f7c5e76bf76359 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 15:52:35 -0400 Subject: [PATCH 121/299] bitcoind: fix check reindex method not found --- lib/services/bitcoind.js | 5 ++++- test/services/bitcoind.unit.js | 17 +++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 73c1ab74..4e96d2ea 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -479,11 +479,14 @@ Bitcoin.prototype._checkReindex = function(node, callback) { } if (node._reindex) { interval = setInterval(function() { - node.client.syncPercentage(function(err, percentSynced) { + node.client.getBlockchainInfo(function(err, response) { if (err) { return finish(self._wrapRPCError(err)); } + var percentSynced = response.result.verificationprogress * 100; + log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); + if (Math.round(percentSynced) >= 100) { node._reindex = false; finish(); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index fb2fc7ca..50ef9a5d 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -728,13 +728,13 @@ describe('Bitcoin Service', function() { after(function() { sandbox.restore(); }); - it('give error from client syncpercentage', function(done) { + it('give error from client getblockchaininfo', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._reindexWait = 1; var node = { _reindex: true, client: { - syncPercentage: sinon.stub().callsArgWith(0, {code: -1 , message: 'Test error'}) + getBlockchainInfo: sinon.stub().callsArgWith(0, {code: -1 , message: 'Test error'}) } }; bitcoind._checkReindex(node, function(err) { @@ -743,15 +743,20 @@ describe('Bitcoin Service', function() { done(); }); }); - it('will wait until syncpercentage is 100 percent', function(done) { + it('will wait until sync is 100 percent', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._reindexWait = 1; - var percent = 90; + var percent = 0.89; var node = { _reindex: true, client: { - syncPercentage: function(callback) { - callback(null, percent++); + getBlockchainInfo: function(callback) { + percent += 0.01; + callback(null, { + result: { + verificationprogress: percent + } + }); } } }; From feb8038da69c7bd95e89d92c05b6c4349baa652c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 16:05:09 -0400 Subject: [PATCH 122/299] bitcoind: fix check reindex interval --- lib/services/bitcoind.js | 3 ++- test/services/bitcoind.unit.js | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4e96d2ea..53bc6c72 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -63,6 +63,7 @@ Bitcoin.dependencies = []; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; +Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_CONFIG_SETTINGS = { server: 1, @@ -492,7 +493,7 @@ Bitcoin.prototype._checkReindex = function(node, callback) { finish(); } }); - }, self._reindexWait); + }, node._reindexWait || Bitcoin.DEFAULT_REINDEX_INTERVAL); } else { callback(); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 50ef9a5d..dccfddb1 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -730,9 +730,9 @@ describe('Bitcoin Service', function() { }); it('give error from client getblockchaininfo', function(done) { var bitcoind = new BitcoinService(baseConfig); - bitcoind._reindexWait = 1; var node = { _reindex: true, + _reindexWait: 1, client: { getBlockchainInfo: sinon.stub().callsArgWith(0, {code: -1 , message: 'Test error'}) } @@ -745,10 +745,10 @@ describe('Bitcoin Service', function() { }); it('will wait until sync is 100 percent', function(done) { var bitcoind = new BitcoinService(baseConfig); - bitcoind._reindexWait = 1; var percent = 0.89; var node = { _reindex: true, + _reindexWait: 1, client: { getBlockchainInfo: function(callback) { percent += 0.01; From 033a62387f3351ed393675db0a7cf298a6f24c6d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 17:22:49 -0400 Subject: [PATCH 123/299] docs: include upgrade notes for bitcore 3 -> 4 --- README.md | 1 + docs/upgrade.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 docs/upgrade.md diff --git a/README.md b/README.md index ce74df1b..388a9383 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ There are several add-on services available to extend the functionality of Bitco ## Documentation +- [Upgrade Notes](docs/upgrade.md) - [Services](docs/services.md) - [Bitcoind](docs/services/bitcoind.md) - Interface to Bitcoin Core - [Web](docs/services/web.md) - Creates an express application over which services can expose their web/API content diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 00000000..19d88a53 --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,75 @@ +# Upgrade Notes + +## From Bitcore 3.0.0 to 4.0.0 + +`bitcore-node@2.1.1` to `bitcore-node@3.0.0` + +This major upgrade includes changes to indexes, API methods and services. Please review below details before upgrading. + +### Indexes + +Indexes include *more information* and are now also *faster*. Because of this a **reindex will be necessary** when upgrading as the address and database indexes are now a part of bitcoind with three new `bitcoin.conf` options: +- `-addressindex` +- `-timestampindex` +- `-spentindex` + +### Configuration Options + +- The `bitcoin.conf` file in will need to be updated to include additional indexes *(see below)*. +- The `datadir` option is now a part of `bitcoind` spawn configuration, and there is a new option to connect to multiple bitcoind processes (Please see [Bitcoin Service Docs](docs/services/bitcoind.md) for more details). The services `db` and `address` are now a part of the `bitcoind` service. Here is how to update `bitcore-node.json` configuration options: + +**Before**: +```json +{ + "datadir": "/home//.bitcoin", + "network": "livenet", + "port": 3001, + "services": [ + "address", + "bitcoind", + "db", + "web" + ] +} +``` + +**After**: +```json +{ + "network": "livenet", + "port": 3001, + "services": [ + "bitcoind", + "web" + ], + "servicesConfig": { + "bitcoind": { + "spawn": { + "datadir": "/home//.bitcoin", + "exec": "/home//bitcore-node/bin/bitcoind" + } + } + } +} +``` + +It will also be necessary to update `bitcoin.conf` settings, to include these fields: +``` +server=1 +whitelist=127.0.0.1 +txindex=1 +addressindex=1 +timestampindex=1 +spentindex=1 +zmqpubrawtx=tcp://127.0.0.1: +zmqpubhashblock=tcp://127.0.0.1: +rpcallowip=127.0.0.1 +rpcuser= +rpcpassword= +``` + +**Important**: Once changes have been made you'll also need to add the `reindex=1` option **only for the first startup** to regenerate the indexes. Once this is complete you should be able to remove the `bitcore-node.db` directory with the old indexes. + +### API and Service Changes +- Many API methods that were a part of the `db` and `address` services are now a part of the `bitcoind` service. Please see [Bitcoin Service Docs](docs/services/bitcoind.md) for more details. +- The `db` and `address` services are deprecated, most of the functionality still exists. Any services that were extending indexes with the `db` service, will need to manage chain state itself, or build the indexes within `bitcoind`. \ No newline at end of file From 2975f27a8ddefec7f7d56db8c5227cae0a6139aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 18 Apr 2016 18:28:27 -0400 Subject: [PATCH 124/299] bitcoind: add uacomment option to default config --- lib/services/bitcoind.js | 3 ++- test/data/default.bitcoin.conf | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 53bc6c72..21658092 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -76,7 +76,8 @@ Bitcoin.DEFAULT_CONFIG_SETTINGS = { zmqpubhashblock: 'tcp://127.0.0.1:28332', rpcallowip: '127.0.0.1', rpcuser: 'bitcoin', - rpcpassword: 'local321' + rpcpassword: 'local321', + uacomment: 'bitcore' }; Bitcoin.prototype._initCaches = function() { diff --git a/test/data/default.bitcoin.conf b/test/data/default.bitcoin.conf index 0f1bfde7..3665db54 100644 --- a/test/data/default.bitcoin.conf +++ b/test/data/default.bitcoin.conf @@ -9,3 +9,4 @@ zmqpubhashblock=tcp://127.0.0.1:28332 rpcallowip=127.0.0.1 rpcuser=bitcoin rpcpassword=local321 +uacomment=bitcore From 7dabd8c4abcd04510d51333282a0c1dee6f42e45 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 14:00:22 -0400 Subject: [PATCH 125/299] docs: correct development environment docs --- docs/development.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/development.md b/docs/development.md index 7c867a72..8dfdc8b0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,11 +28,12 @@ git checkout **Note**: See bitcoin documentation for building bitcoin on your platform. -## Install ZeroMQ Development Dependency +## Install Development Dependencies For Ubuntu: ```bash -sudo apt-get install libzmq-dev +sudo apt-get install libzmq3-dev +sudo apt-get install build-essential ``` For Mac OS X: @@ -112,7 +113,8 @@ Edit `bitcore-node.json` with something similar to: "bitcoind", "web", "insight-api", - "insight-ui" + "insight-ui", + "" ], "servicesConfig": { "bitcoind": { @@ -125,6 +127,8 @@ Edit `bitcore-node.json` with something similar to: } ``` +**Note**: To install services [insight-api](https://github.com/bitpay/insight-api) and [insight-ui](https://github.com/bitpay/insight-ui) you'll need to clone the repositories locally. + Setup symlinks for all of the services and dependencies: ```bash From 2b38f081756b146cc1edbe3ea79ae12007a0455f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 16:00:17 -0400 Subject: [PATCH 126/299] bitcoind: subscribe to zmq events once synced prevents flooding tx and and block events that can cause issues --- lib/services/bitcoind.js | 33 ++++++++++++++++++++-- test/services/bitcoind.unit.js | 51 ++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 21658092..0ce7b480 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -65,6 +65,7 @@ Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; +Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; Bitcoin.DEFAULT_CONFIG_SETTINGS = { server: 1, whitelist: '127.0.0.1', @@ -431,6 +432,34 @@ Bitcoin.prototype._zmqTransactionHandler = function(node, message) { } }; +Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { + var self = this; + var interval; + interval = setInterval(function() { + // update tip + node.client.getBestBlockHash(function(err, response) { + if (err) { + return log.error(self._wrapRPCError(err)); + } + var blockhash = new Buffer(response.result, 'hex'); + self._updateTip(node, blockhash); + + // check if synced + node.client.getBlockchainInfo(function(err, response) { + if (err) { + return log.error(self._wrapRPCError(err)); + } + var percentSynced = response.result.verificationprogress * 100; + if (Math.round(percentSynced) >= 99) { + // subscribe to events for further updates + self._subscribeZmqEvents(node); + clearInterval(interval); + } + }); + }); + }, node._tipUpdateInterval || Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL).unref(); +}; + Bitcoin.prototype._subscribeZmqEvents = function(node) { var self = this; node.zmqSubSocket.subscribe('hashblock'); @@ -574,7 +603,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { if (err) { return callback(err); } - self._subscribeZmqEvents(node); + self._checkSyncedAndSubscribeZmqEvents(node); callback(null, node); }); @@ -606,7 +635,7 @@ Bitcoin.prototype._connectProcess = function(config, callback) { } self._initZmqSubSocket(node, config.zmqpubrawtx); - self._subscribeZmqEvents(node); + self._checkSyncedAndSubscribeZmqEvents(node); callback(null, node); }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index dccfddb1..b8a2e778 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -669,6 +669,47 @@ describe('Bitcoin Service', function() { }); }); + describe('#_checkSyncedAndSubscribeZmqEvents', function() { + var sandbox = sinon.sandbox.create(); + before(function() { + sandbox.stub(log, 'error'); + }); + after(function() { + sandbox.restore(); + }); + it('log errors, update tip and subscribe to zmq events', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._updateTip = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, { + result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' + }); + getBestBlockHash.onCall(0).callsArgWith(0, {code: -1 , message: 'Test error'}); + var getBlockchainInfo = sinon.stub().callsArgWith(0, null, { + result: { + verificationprogress: 0.99 + } + }); + getBlockchainInfo.onCall(0).callsArgWith(0, {code: -1, message: 'Test error'}); + var node = { + _reindex: true, + _reindexWait: 1, + _tipUpdateInterval: 1, + client: { + getBestBlockHash: getBestBlockHash, + getBlockchainInfo: getBlockchainInfo + } + }; + bitcoind._checkSyncedAndSubscribeZmqEvents(node); + setTimeout(function() { + log.error.callCount.should.equal(2); + bitcoind._updateTip.callCount.should.equal(2); + bitcoind._subscribeZmqEvents.callCount.should.equal(1); + done(); + }, 10); + }); + }); + describe('#_subscribeZmqEvents', function() { it('will call subscribe on zmq socket', function() { var bitcoind = new BitcoinService(baseConfig); @@ -888,7 +929,7 @@ describe('Bitcoin Service', function() { bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); bitcoind._initZmqSubSocket = sinon.stub(); - bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); bitcoind._checkReindex = sinon.stub().callsArgWith(1, null); bitcoind._spawnChildProcess(function(err, node) { should.not.exist(err); @@ -906,8 +947,8 @@ describe('Bitcoin Service', function() { bitcoind._initZmqSubSocket.callCount.should.equal(1); should.exist(bitcoind._initZmqSubSocket.args[0][0].client); bitcoind._initZmqSubSocket.args[0][1].should.equal('tcp://127.0.0.1:30001'); - bitcoind._subscribeZmqEvents.callCount.should.equal(1); - should.exist(bitcoind._subscribeZmqEvents.args[0][0].client); + bitcoind._checkSyncedAndSubscribeZmqEvents.callCount.should.equal(1); + should.exist(bitcoind._checkSyncedAndSubscribeZmqEvents.args[0][0].client); should.exist(node); should.exist(node.client); done(); @@ -968,7 +1009,7 @@ describe('Bitcoin Service', function() { bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); bitcoind._initZmqSubSocket = sinon.stub(); - bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); bitcoind._checkReindex = sinon.stub().callsArgWith(1, new Error('test')); bitcoind._spawnChildProcess(function(err) { @@ -993,7 +1034,7 @@ describe('Bitcoin Service', function() { it('will init zmq/rpc on node', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._initZmqSubSocket = sinon.stub(); - bitcoind._subscribeZmqEvents = sinon.stub(); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); var config = {}; bitcoind._connectProcess(config, function(err, node) { From b092adcc2173f346806b420dc553a6e8b0718c33 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 16:40:54 -0400 Subject: [PATCH 127/299] bitcoind: subscribe to zmq events without interval if already synced --- lib/services/bitcoind.js | 28 ++++++++++++++++++++++++---- test/services/bitcoind.unit.js | 19 ++++++++++++++----- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 0ce7b480..7cba2e03 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -435,11 +435,12 @@ Bitcoin.prototype._zmqTransactionHandler = function(node, message) { Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { var self = this; var interval; - interval = setInterval(function() { + + function checkAndSubscribe(callback) { // update tip node.client.getBestBlockHash(function(err, response) { if (err) { - return log.error(self._wrapRPCError(err)); + return callback(self._wrapRPCError(err)); } var blockhash = new Buffer(response.result, 'hex'); self._updateTip(node, blockhash); @@ -447,17 +448,36 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { // check if synced node.client.getBlockchainInfo(function(err, response) { if (err) { - return log.error(self._wrapRPCError(err)); + return callback(self._wrapRPCError(err)); } var percentSynced = response.result.verificationprogress * 100; if (Math.round(percentSynced) >= 99) { // subscribe to events for further updates self._subscribeZmqEvents(node); clearInterval(interval); + callback(null, true); + } else { + callback(null, false); } }); }); - }, node._tipUpdateInterval || Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL).unref(); + } + + checkAndSubscribe(function(err, synced) { + if (err) { + log.error(err); + } + if (!synced) { + interval = setInterval(function() { + checkAndSubscribe(function(err) { + if (err) { + log.error(err); + } + }); + }, node._tipUpdateInterval || Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL).unref(); + } + }); + }; Bitcoin.prototype._subscribeZmqEvents = function(node) { diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index b8a2e778..7a2ab905 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -685,11 +685,20 @@ describe('Bitcoin Service', function() { result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' }); getBestBlockHash.onCall(0).callsArgWith(0, {code: -1 , message: 'Test error'}); - var getBlockchainInfo = sinon.stub().callsArgWith(0, null, { - result: { - verificationprogress: 0.99 + var progress = 0.90; + function getProgress() { + progress = progress + 0.01; + return progress; + } + var info = {}; + Object.defineProperty(info, 'result', { + get: function() { + return { + verificationprogress: getProgress() + }; } }); + var getBlockchainInfo = sinon.stub().callsArgWith(0, null, info); getBlockchainInfo.onCall(0).callsArgWith(0, {code: -1, message: 'Test error'}); var node = { _reindex: true, @@ -703,10 +712,10 @@ describe('Bitcoin Service', function() { bitcoind._checkSyncedAndSubscribeZmqEvents(node); setTimeout(function() { log.error.callCount.should.equal(2); - bitcoind._updateTip.callCount.should.equal(2); + bitcoind._updateTip.callCount.should.equal(10); bitcoind._subscribeZmqEvents.callCount.should.equal(1); done(); - }, 10); + }, 20); }); }); From 458fe2f2b6262e5f001996646b0e27a60f80811d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 16:53:00 -0400 Subject: [PATCH 128/299] bitcoind: emit block events while polling before subscribing to zmq events --- lib/services/bitcoind.js | 1 + test/services/bitcoind.unit.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 7cba2e03..570bab42 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -443,6 +443,7 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { return callback(self._wrapRPCError(err)); } var blockhash = new Buffer(response.result, 'hex'); + self.emit('block', blockhash); self._updateTip(node, blockhash); // check if synced diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 7a2ab905..d0ee746a 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -681,6 +681,10 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); bitcoind._updateTip = sinon.stub(); bitcoind._subscribeZmqEvents = sinon.stub(); + var blockEvents = 0; + bitcoind.on('block', function() { + blockEvents++; + }); var getBestBlockHash = sinon.stub().callsArgWith(0, null, { result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' }); @@ -712,6 +716,7 @@ describe('Bitcoin Service', function() { bitcoind._checkSyncedAndSubscribeZmqEvents(node); setTimeout(function() { log.error.callCount.should.equal(2); + blockEvents.should.equal(10); bitcoind._updateTip.callCount.should.equal(10); bitcoind._subscribeZmqEvents.callCount.should.equal(1); done(); From 7d878adcf0196a645a687e2f14171bb005f503d9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 17:23:27 -0400 Subject: [PATCH 129/299] bitcoind: immediately subscribe with connect option --- lib/services/bitcoind.js | 2 +- test/services/bitcoind.unit.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 570bab42..2d1f84ab 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -656,7 +656,7 @@ Bitcoin.prototype._connectProcess = function(config, callback) { } self._initZmqSubSocket(node, config.zmqpubrawtx); - self._checkSyncedAndSubscribeZmqEvents(node); + self._subscribeZmqEvents(node); callback(null, node); }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index d0ee746a..f7a228f9 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1048,7 +1048,7 @@ describe('Bitcoin Service', function() { it('will init zmq/rpc on node', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._initZmqSubSocket = sinon.stub(); - bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, null); var config = {}; bitcoind._connectProcess(config, function(err, node) { From 40e7b24ea993a2c837b5895ee5ee3bd5166ea225 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 20:10:32 -0400 Subject: [PATCH 130/299] test: fix unstubbed uncaughException --- test/scaffold/start.integration.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/scaffold/start.integration.js b/test/scaffold/start.integration.js index b2f4f61c..d61f4484 100644 --- a/test/scaffold/start.integration.js +++ b/test/scaffold/start.integration.js @@ -28,6 +28,8 @@ describe('#start', function() { '../node': TestNode }); + starttest.registerExitHandlers = sinon.stub(); + node = starttest({ path: __dirname, config: { @@ -51,6 +53,8 @@ describe('#start', function() { '../node': TestNode }); starttest.cleanShutdown = sinon.stub(); + starttest.registerExitHandlers = sinon.stub(); + starttest({ path: __dirname, config: { @@ -83,6 +87,7 @@ describe('#start', function() { var starttest = proxyquire('../../lib/scaffold/start', { '../node': TestNode }); + starttest.registerExitHandlers = sinon.stub(); node = starttest({ path: __dirname, From c3dab07b3086a82e9186e1b54e61dc8bfdf5964d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 19 Apr 2016 20:27:52 -0400 Subject: [PATCH 131/299] bitcoind: fix clearInterval issue with Node.js 0.12 --- lib/services/bitcoind.js | 5 ++++- test/services/bitcoind.unit.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 2d1f84ab..9d59f48d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -470,12 +470,15 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { } if (!synced) { interval = setInterval(function() { + if (self.node.stopping) { + return clearInterval(interval); + } checkAndSubscribe(function(err) { if (err) { log.error(err); } }); - }, node._tipUpdateInterval || Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL).unref(); + }, node._tipUpdateInterval || Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL); } }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index f7a228f9..62131046 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -720,7 +720,7 @@ describe('Bitcoin Service', function() { bitcoind._updateTip.callCount.should.equal(10); bitcoind._subscribeZmqEvents.callCount.should.equal(1); done(); - }, 20); + }, 40); }); }); From 019bc2a58cf03c5ee09393e28d30c89630bfd519 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 20 Apr 2016 11:41:02 -0400 Subject: [PATCH 132/299] bitcoind: load network bitcoin.conf and set defaults --- lib/services/bitcoind.js | 61 +++++++++++++++++----- test/services/bitcoind.unit.js | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 14 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 9d59f48d..d94e49ea 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -185,6 +185,26 @@ Bitcoin.prototype._getDefaultConfig = function() { return config; }; +Bitcoin.prototype._parseBitcoinConf = function(configPath) { + var options = {}; + var file = fs.readFileSync(configPath); + var unparsed = file.toString().split('\n'); + for(var i = 0; i < unparsed.length; i++) { + var line = unparsed[i]; + if (!line.match(/^\#/) && line.match(/\=/)) { + var option = line.split('='); + var value; + if (!Number.isNaN(Number(option[1]))) { + value = Number(option[1]); + } else { + value = option[1]; + } + options[option[0]] = value; + } + } + return options; +}; + Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ @@ -210,20 +230,12 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { fs.writeFileSync(configPath, defaultConfig); } - var file = fs.readFileSync(configPath); - var unparsed = file.toString().split('\n'); - for(var i = 0; i < unparsed.length; i++) { - var line = unparsed[i]; - if (!line.match(/^\#/) && line.match(/\=/)) { - var option = line.split('='); - var value; - if (!Number.isNaN(Number(option[1]))) { - value = Number(option[1]); - } else { - value = option[1]; - } - this.spawn.config[option[0]] = value; - } + _.extend(this.spawn.config, this._getDefaultConf()); + _.extend(this.spawn.config, this._parseBitcoinConf(configPath)); + + var networkConfigPath = this._getNetworkConfigPath(); + if (networkConfigPath && fs.existsSync(networkConfigPath)) { + _.extend(this.spawn.config, this._parseBitcoinConf(networkConfigPath)); } var spawnConfig = this.spawn.config; @@ -334,6 +346,27 @@ Bitcoin.prototype._initChain = function(callback) { }); }; +Bitcoin.prototype._getDefaultConf = function() { + var networkOptions = { + rpcport: 8332 + }; + if (this.node.network === bitcore.Networks.testnet) { + networkOptions.rpcport = 18332; + } + return networkOptions; +}; + +Bitcoin.prototype._getNetworkConfigPath = function() { + var networkPath; + if (this.node.network === bitcore.Networks.testnet) { + networkPath = 'testnet3/bitcoin.conf'; + if (this.node.network.regtestEnabled) { + networkPath = 'regtest/bitcoin.conf'; + } + } + return networkPath; +}; + Bitcoin.prototype._getNetworkOption = function() { var networkOption; if (this.node.network === bitcore.Networks.testnet) { diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 62131046..7f9136cf 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -411,6 +411,100 @@ describe('Bitcoin Service', function() { }); }); + describe('#_getDefaultConf', function() { + afterEach(function() { + bitcore.Networks.disableRegtest(); + baseConfig.node.network = bitcore.Networks.testnet; + }); + it('will get default rpc port for livenet', function() { + var config = { + node: { + network: bitcore.Networks.livenet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind._getDefaultConf().rpcport.should.equal(8332); + }); + it('will get default rpc port for testnet', function() { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind._getDefaultConf().rpcport.should.equal(18332); + }); + it('will get default rpc port for regtest', function() { + bitcore.Networks.enableRegtest(); + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind._getDefaultConf().rpcport.should.equal(18332); + }); + }); + + describe('#_getNetworkConfigPath', function() { + afterEach(function() { + bitcore.Networks.disableRegtest(); + baseConfig.node.network = bitcore.Networks.testnet; + }); + it('will get default config path for livenet', function() { + var config = { + node: { + network: bitcore.Networks.livenet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + should.equal(bitcoind._getNetworkConfigPath(), undefined); + }); + it('will get default rpc port for testnet', function() { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind._getNetworkConfigPath().should.equal('testnet3/bitcoin.conf'); + }); + it('will get default rpc port for regtest', function() { + bitcore.Networks.enableRegtest(); + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind._getNetworkConfigPath().should.equal('regtest/bitcoin.conf'); + }); + }); + describe('#_getNetworkOption', function() { afterEach(function() { bitcore.Networks.disableRegtest(); From 3e2492e6d47ea4356e00cae56ce63fa363758ff3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 20 Apr 2016 11:55:45 -0400 Subject: [PATCH 133/299] scaffold: detect incompatible config --- lib/scaffold/start.js | 9 +++++++++ test/scaffold/start.integration.js | 28 ++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 47c61307..fa1365dc 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -201,6 +201,15 @@ function start(options) { } fullConfig.path = path.resolve(options.path, './bitcore-node.json'); + + if (fullConfig.datadir) { + throw new TypeError( + 'Configuration file (' + fullConfig.path + ') is not compatible with this version.' + + ' Please see https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md' + + ' for upgrade details.' + ); + } + fullConfig.services = start.setupServices(require, servicesPath, options.config); var node = new BitcoreNode(fullConfig); diff --git a/test/scaffold/start.integration.js b/test/scaffold/start.integration.js index d61f4484..5523f29d 100644 --- a/test/scaffold/start.integration.js +++ b/test/scaffold/start.integration.js @@ -15,7 +15,11 @@ describe('#start', function() { options.services[0].should.deep.equal({ name: 'bitcoind', module: BitcoinService, - config: {} + config: { + spawn: { + datadir: './data' + } + } }); }; TestNode.prototype.start = sinon.stub().callsArg(0); @@ -36,7 +40,13 @@ describe('#start', function() { services: [ 'bitcoind' ], - datadir: './data' + servicesConfig: { + bitcoind: { + spawn: { + datadir: './data' + } + } + } } }); node.should.be.instanceof(TestNode); @@ -59,7 +69,7 @@ describe('#start', function() { path: __dirname, config: { services: [], - datadir: './testdir' + servicesConfig: {} } }); setImmediate(function() { @@ -74,7 +84,10 @@ describe('#start', function() { name: 'bitcoind', module: BitcoinService, config: { - param: 'test' + param: 'test', + spawn: { + datadir: './data' + } } }); }; @@ -97,10 +110,13 @@ describe('#start', function() { ], servicesConfig: { 'bitcoind': { - param: 'test' + param: 'test', + spawn: { + datadir: './data' + } } }, - datadir: './data' + } }); node.should.be.instanceof(TestNode); From 2015514e783bc79a05cd50f840ddd835c81e14e9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 20 Apr 2016 12:05:15 -0400 Subject: [PATCH 134/299] test: increase timeout for check synced test --- test/services/bitcoind.unit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 7f9136cf..39df9972 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -814,7 +814,7 @@ describe('Bitcoin Service', function() { bitcoind._updateTip.callCount.should.equal(10); bitcoind._subscribeZmqEvents.callCount.should.equal(1); done(); - }, 40); + }, 200); }); }); From 587602d080cc643a5350462ce7581621df335223 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 20 Apr 2016 13:03:18 -0400 Subject: [PATCH 135/299] bitcoind: stop failsafe timeout --- lib/services/bitcoind.js | 24 ++++++++++++++++++------ test/services/bitcoind.unit.js | 13 +++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index d94e49ea..96a17344 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -49,6 +49,7 @@ function Bitcoin(options) { // limits this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; + this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; // try all interval this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; @@ -61,6 +62,7 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; +Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; @@ -1570,16 +1572,26 @@ Bitcoin.prototype.generateBlock = function(num, callback) { */ Bitcoin.prototype.stop = function(callback) { if (this.spawn && this.spawn.process) { + var exited = false; this.spawn.process.once('exit', function(code) { - if (code !== 0) { - var error = new Error('bitcoind spawned process exited with status code: ' + code); - error.code = code; - return callback(error); - } else { - return callback(); + if (!exited) { + exited = true; + if (code !== 0) { + var error = new Error('bitcoind spawned process exited with status code: ' + code); + error.code = code; + return callback(error); + } else { + return callback(); + } } }); this.spawn.process.kill('SIGINT'); + setTimeout(function() { + if (!exited) { + exited = true; + return callback(new Error('bitcoind process did not exit')); + } + }, this.shutdownTimeout).unref(); } else { callback(); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 39df9972..88cea586 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3032,6 +3032,19 @@ describe('Bitcoin Service', function() { bitcoind.spawn.process.kill.args[0][0].should.equal('SIGINT'); bitcoind.spawn.process.emit('exit', 1); }); + it('will stop after timeout', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.shutdownTimeout = 300; + bitcoind.spawn = {}; + bitcoind.spawn.process = new EventEmitter(); + bitcoind.spawn.process.kill = sinon.stub(); + bitcoind.stop(function(err) { + err.should.be.instanceof(Error); + done(); + }); + bitcoind.spawn.process.kill.callCount.should.equal(1); + bitcoind.spawn.process.kill.args[0][0].should.equal('SIGINT'); + }); }); }); From d1cf9deef00bfeefd0be13a765d2e58d1280d728 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 20 Apr 2016 15:35:43 -0400 Subject: [PATCH 136/299] bitcoind: parse ints for pagination --- lib/services/bitcoind.js | 6 ++++-- test/services/bitcoind.unit.js | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 96a17344..db33350d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1059,10 +1059,12 @@ Bitcoin.prototype._getAddressStrings = function(addresses) { return addressStrings; }; -Bitcoin.prototype._paginateTxids = function(fullTxids, from, to) { +Bitcoin.prototype._paginateTxids = function(fullTxids, fromArg, toArg) { var txids; + var from = parseInt(fromArg); + var to = parseInt(toArg); if (from >= 0 && to >= 0) { - $.checkState(from < to, '"from" is expected to be less than "to"'); + $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); txids = fullTxids.slice(from, to); } else { txids = fullTxids; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 88cea586..80df5fc0 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1992,8 +1992,14 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; (function() { - var paginated = bitcoind._paginateTxids(txids, 1, 0); - }).should.throw('"from" is expected to be less than "to"'); + bitcoind._paginateTxids(txids, 1, 0); + }).should.throw('"from" (1) is expected to be less than "to"'); + }); + it('will handle string numbers', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var paginated = bitcoind._paginateTxids(txids, '1', '3'); + paginated.should.deep.equal([1, 2]); }); }); From b901e10c9d7403a7fa5581b6d89a8f18980ae668 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 17:13:24 -0400 Subject: [PATCH 137/299] bitcoind: update unspentoutputs with mempool --- lib/services/bitcoind.js | 75 ++++++++++++++++++++++++++++++---- scripts/install | 2 +- test/services/bitcoind.unit.js | 12 ++++-- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index db33350d..59f20cf1 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -822,22 +822,81 @@ Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { */ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callback) { var self = this; + var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); var utxos = self.utxosCache.get(cacheKey); - if (utxos) { - return setImmediate(function() { - callback(null, utxos); - }); - } else { - self.client.getAddressUtxos({addresses: addresses}, function(err, response) { + + function transformUnspentOutput(delta) { + var script = bitcore.Script.fromAddress(delta.address); + return { + address: delta.address, + txid: delta.txid, + height: -1, // unconfirmed + outputIndex: delta.index, + script: script.toHex(), + satoshis: delta.satoshis + }; + } + + function updateWithMempool(utxos, mempoolDeltas) { + if (!mempoolDeltas || !mempoolDeltas.length) { + return utxos; + } + var isSpentOutputs = false; + var spentOutputs = []; + for (var i = 0; i < mempoolDeltas.length; i++) { + var delta = mempoolDeltas[i]; + if (delta.satoshis > 0) { + utxos.push(transformUnspentOutput(delta)); + } else if (delta.satoshis < 0) { + if (!spentOutputs[delta.prevtxid]) { + spentOutputs[delta.prevtxid] = [delta.prevout]; + } else { + spentOutputs[delta.prevtxid].push(delta.prevout); + } + isSpentOutputs = true; + } + } + if (isSpentOutputs) { + return utxos.filter(function(utxo) { + if (!spentOutputs[utxo.txid]) { + return true; + } else { + return (spentOutputs[utxo.txid].indexOf(utxo.outputIndex) === -1); + } + }); + } + return utxos; + } + + function finish(mempoolDeltas) { + if (utxos) { + return setImmediate(function() { + callback(null, updateWithMempool(utxos, mempoolDeltas).reverse()); + }); + } else { + self.client.getAddressUtxos({addresses: addresses}, function(err, response) { + if (err) { + return callback(self._wrapRPCError(err)); + } + self.utxosCache.set(cacheKey, response.result); + callback(null, updateWithMempool(response.result, mempoolDeltas).reverse()); + }); + } + } + + if (queryMempool) { + self.client.getAddressMempool({addresses: addresses}, function(err, response) { if (err) { return callback(self._wrapRPCError(err)); } - self.utxosCache.set(cacheKey, response.result); - callback(null, response.result); + finish(response.result); }); + } else { + finish(); } + }; Bitcoin.prototype._getBalanceFromMempool = function(deltas) { diff --git a/scripts/install b/scripts/install index f1f7bd9d..aa76aed6 100755 --- a/scripts/install +++ b/scripts/install @@ -5,7 +5,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12.0-bitcore-beta3" +tag="v0.12.0-bitcore-beta4" cd "${root_dir}/bin" diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 80df5fc0..e5532a06 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1381,7 +1381,9 @@ describe('Bitcoin Service', function() { getAddressUtxos: sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}) } }); - var options = {}; + var options = { + queryMempool: false + }; var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; bitcoind.getAddressUnspentOutputs(address, options, function(err) { should.exist(err); @@ -1408,7 +1410,9 @@ describe('Bitcoin Service', function() { }) } }); - var options = {}; + var options = { + queryMempool: false + }; var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { if (err) { @@ -1439,7 +1443,9 @@ describe('Bitcoin Service', function() { getAddressUtxos: getAddressUtxos } }); - var options = {}; + var options = { + queryMempool: false + }; var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { if (err) { From 0272b17f0e133846f1cc34110eaf00339c895afe Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 17:30:42 -0400 Subject: [PATCH 138/299] test: fix regtest amount check --- regtest/node.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/regtest/node.js b/regtest/node.js index 368fa2a6..40de6de2 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -243,6 +243,8 @@ describe('Node Functionality', function() { var address5; var testKey6; var address6; + var tx2Amount; + var tx2Hash; before(function(done) { /* jshint maxstatements: 50 */ @@ -313,10 +315,12 @@ describe('Node Functionality', function() { async.series([ function(next) { var tx2 = new Transaction(); + tx2Amount = results[0].satoshis - 10000; tx2.from(results[0]); - tx2.to(address2, results[0].satoshis - 10000); + tx2.to(address2, tx2Amount); tx2.change(address); tx2.sign(testKey); + tx2Hash = tx2.hash; node.sendTransaction(tx2.serialize(), function(err) { if (err) { return next(err); @@ -399,7 +403,8 @@ describe('Node Functionality', function() { should.exist(history[2].addresses[address3]); history[3].height.should.equal(156); should.exist(history[3].addresses[address2]); - history[3].satoshis.should.equal(99990000); + history[3].satoshis.should.equal(tx2Amount); + history[3].tx.hash.should.equal(tx2Hash); history[3].confirmations.should.equal(4); done(); }); From 7f17dd4a4c1402f92d948c07c66790723b829234 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 22 Apr 2016 12:09:57 -0400 Subject: [PATCH 139/299] bitcoind: fixed issue with cache mempool updates --- lib/services/bitcoind.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 59f20cf1..c952f9b3 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -832,23 +832,25 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb return { address: delta.address, txid: delta.txid, - height: -1, // unconfirmed outputIndex: delta.index, script: script.toHex(), - satoshis: delta.satoshis + satoshis: delta.satoshis, + timestamp: delta.timestamp }; } - function updateWithMempool(utxos, mempoolDeltas) { + function updateWithMempool(confirmedUtxos, mempoolDeltas) { if (!mempoolDeltas || !mempoolDeltas.length) { - return utxos; + return confirmedUtxos; } var isSpentOutputs = false; + var mempoolUnspentOutputs = []; var spentOutputs = []; + for (var i = 0; i < mempoolDeltas.length; i++) { var delta = mempoolDeltas[i]; if (delta.satoshis > 0) { - utxos.push(transformUnspentOutput(delta)); + mempoolUnspentOutputs.push(transformUnspentOutput(delta)); } else if (delta.satoshis < 0) { if (!spentOutputs[delta.prevtxid]) { spentOutputs[delta.prevtxid] = [delta.prevout]; @@ -858,6 +860,9 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb isSpentOutputs = true; } } + + var utxos = mempoolUnspentOutputs.reverse().concat(confirmedUtxos); + if (isSpentOutputs) { return utxos.filter(function(utxo) { if (!spentOutputs[utxo.txid]) { @@ -867,21 +872,23 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb } }); } + return utxos; } function finish(mempoolDeltas) { if (utxos) { return setImmediate(function() { - callback(null, updateWithMempool(utxos, mempoolDeltas).reverse()); + callback(null, updateWithMempool(utxos, mempoolDeltas)); }); } else { self.client.getAddressUtxos({addresses: addresses}, function(err, response) { if (err) { return callback(self._wrapRPCError(err)); } - self.utxosCache.set(cacheKey, response.result); - callback(null, updateWithMempool(response.result, mempoolDeltas).reverse()); + var utxos = response.result.reverse(); + self.utxosCache.set(cacheKey, utxos); + callback(null, updateWithMempool(utxos, mempoolDeltas)); }); } } From 5e6600162afe609dde19c9fff2b8d6b6c0fdf529 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 22 Apr 2016 12:48:16 -0400 Subject: [PATCH 140/299] test: add unit test for getaddressunspentoutputs with mempool --- lib/services/bitcoind.js | 1 + test/services/bitcoind.unit.js | 78 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index c952f9b3..bc45a1b8 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -840,6 +840,7 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb } function updateWithMempool(confirmedUtxos, mempoolDeltas) { + /* jshint maxstatements: 20 */ if (!mempoolDeltas || !mempoolDeltas.length) { return confirmedUtxos; } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index e5532a06..d030cf8b 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1465,6 +1465,84 @@ describe('Bitcoin Service', function() { }); }); }); + it('will update with mempool results', function(done) { + var deltas = [ + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 1 + }, + { + txid: 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', + satoshis: 100000, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342833133 + }, + { + txid: 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', + satoshis: 400000, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342954813 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + } + ]; + var expectedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + outputIndex: 1, + satoshis: 400000, + script: '76a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac', + timestamp: 1461342954813, + txid: 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345' + }, + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + outputIndex: 0, + satoshis: 100000, + script: '76a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac', + timestamp: 1461342833133, + txid: 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f' + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(2); + utxos.should.deep.equal(expectedUtxos); + done(); + }); + }); }); describe('#_getBalanceFromMempool', function() { From c6e543c2a181a0d34cb1c036da66678429993f22 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 22 Apr 2016 16:13:57 -0400 Subject: [PATCH 141/299] bitcoind: fix noTxList caching issue --- lib/services/bitcoind.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index bc45a1b8..b72e2345 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1247,9 +1247,12 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { self.summaryCache.set(cacheKey, summary); if (!options.noTxList) { var allTxids = mempoolTxids.reverse().concat(summaryTxids); - summary.txids = allTxids; + var allSummary = _.clone(summary); + allSummary.txids = allTxids; + callback(null, allSummary); + } else { + callback(null, summary); } - callback(null, summary); }); } From c63e98f0616d6218a744454a6526014b1b4be0ab Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 22 Apr 2016 16:51:56 -0400 Subject: [PATCH 142/299] bitcoind: limit tx history range --- lib/services/bitcoind.js | 6 ++++++ test/services/bitcoind.unit.js | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b72e2345..f99f30bd 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -48,6 +48,7 @@ function Bitcoin(options) { this.subscriptions.hashblock = []; // limits + this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; @@ -62,6 +63,7 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; +Bitcoin.DEFAULT_MAX_HISTORY = 10; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; @@ -1132,6 +1134,10 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, fromArg, toArg) { var to = parseInt(toArg); if (from >= 0 && to >= 0) { $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); + $.checkState( + (to - from) <= this.maxTransactionHistory, + '"from" (' + from + ') and "to" (' + to + ') range should be less than or equal to ' + this.maxTransactionHistory + ); txids = fullTxids.slice(from, to); } else { txids = fullTxids; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index d030cf8b..7d3d227a 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2054,12 +2054,19 @@ describe('Bitcoin Service', function() { }); describe('#_paginateTxids', function() { - it('slice txids based on "from" and "to" (3 to 30)', function() { + it('slice txids based on "from" and "to" (3 to 13)', function() { var bitcoind = new BitcoinService(baseConfig); var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - var paginated = bitcoind._paginateTxids(txids, 3, 30); + var paginated = bitcoind._paginateTxids(txids, 3, 13); paginated.should.deep.equal([3, 4, 5, 6, 7, 8, 9, 10]); }); + it('slice txids based on "from" and "to" (3 to 30)', function() { + var bitcoind = new BitcoinService(baseConfig); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + (function() { + bitcoind._paginateTxids(txids, 3, 30); + }).should.throw(Error); + }); it('slice txids based on "from" and "to" (0 to 3)', function() { var bitcoind = new BitcoinService(baseConfig); var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; From a61f43a5843841266faa463a57dfb1f151b3126c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 22 Apr 2016 17:09:13 -0400 Subject: [PATCH 143/299] build: upgrade socket.io nodejs binary addons have been removed as optional dependencies from ws, however they will still be used if available: https://github.com/websockets/ws/commit/49b11093e9a009e5305dcde7003d3a896b2811dc --- package.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4f4848bb..397eef17 100644 --- a/package.json +++ b/package.json @@ -53,10 +53,14 @@ "mkdirp": "0.5.0", "npm": "^2.14.1", "semver": "^5.0.1", - "socket.io": "bitpay/socket.io#bitpay-1.3.7", - "socket.io-client": "bitpay/socket.io-client#bitpay-1.3.7", + "socket.io": "^1.4.5", + "socket.io-client": "^1.4.5", "zmq": "^2.14.0" }, + "optionalDependencies": { + "bufferutil": "~1.2.1", + "utf-8-validate": "~1.2.1" + }, "devDependencies": { "benchmark": "1.0.0", "bitcore-p2p": "^1.1.0", From 3f34fb6ea0b28683758a9a7092b25d09582b0c13 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 11:02:37 -0400 Subject: [PATCH 144/299] bitcoind: always log errors emitted instead of being uncaught exceptions --- lib/services/bitcoind.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index f99f30bd..e3314b7a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -58,6 +58,10 @@ function Bitcoin(options) { // available bitcoind nodes this._initClients(); + + this.on('error', function(err) { + log.error(err.stack); + }); } util.inherits(Bitcoin, Service); @@ -429,7 +433,6 @@ Bitcoin.prototype._updateTip = function(node, message) { node.client.getBlock(self.tiphash, function(err, response) { if (err) { var error = self._wrapRPCError(err); - log.error(error); self.emit('error', error); } else { self.height = response.result.height; @@ -441,7 +444,6 @@ Bitcoin.prototype._updateTip = function(node, message) { if(!self.node.stopping) { self.syncPercentage(function(err, percentage) { if (err) { - log.error(err); self.emit('error', err); } else { if (Math.round(percentage) >= 100) { From 944c44ed745103d660a668b4f81bf4d748b26632 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 11:12:03 -0400 Subject: [PATCH 145/299] bitcoind: return selected set of info for getinfo --- lib/services/bitcoind.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index e3314b7a..c79bde47 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1631,8 +1631,20 @@ Bitcoin.prototype.getInfo = function(callback) { return callback(self._wrapRPCError(err)); } var result = response.result; - result.network = self.node.getNetworkName(); - callback(null, result); + var info = { + version: result.version, + protocolversion: result.protocolversion, + blocks: result.blocks, + timeoffset: result.timeoffset, + connections: result.connections, + proxy: result.proxy, + difficulty: result.difficulty, + testnet: result.testnet, + relayfee: result.relayfee, + errors: result.errors, + network: self.node.getNetworkName() + }; + callback(null, info); }); }; From 76eeba59999307fc8e031167e6500d8267fd98c2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 16:04:49 -0400 Subject: [PATCH 146/299] build: verify bitcoin download --- .gitignore | 1 + package.json | 3 +- scripts/download | 111 +++++++++++++++++++++++++++++++++++++++++++++++ scripts/install | 53 ---------------------- 4 files changed, 114 insertions(+), 54 deletions(-) create mode 100755 scripts/download delete mode 100755 scripts/install diff --git a/.gitignore b/.gitignore index 8132c320..0e43f951 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ coverage/* *.log .DS_Store bin/bitcoin* +bin/SHA256SUMS.asc regtest/data/node1/regtest regtest/data/node2/regtest regtest/data/node3/regtest diff --git a/package.json b/package.json index 397eef17..fb4d1b2a 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "bitcore-node": "./bin/bitcore-node" }, "scripts": { - "install": "./scripts/install", + "preinstall": "./scripts/download", + "verify": "./scripts/download --skip-bitcoin-download --verify-bitcoin-download", "test": "NODE_ENV=test mocha -R spec --recursive", "regtest": "./scripts/regtest", "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" diff --git a/scripts/download b/scripts/download new file mode 100755 index 00000000..220bf560 --- /dev/null +++ b/scripts/download @@ -0,0 +1,111 @@ +#!/bin/bash + +set -e + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." +platform=`uname -a | awk '{print tolower($1)}'` +arch=`uname -m` +version="0.12.0" +url="https://github.com/braydonf/bitcoin/releases/download" +tag="v0.12.0-bitcore-beta6" + +if [ "${platform}" == "linux" ]; then + if [ "${arch}" == "x86_64" ]; then + tarball_name="bitcoin-${version}-linux64.tar.gz" + elif [ "${arch}" == "x86_32" ]; then + tarball_name="bitcoin-${version}-linux32.tar.gz" + fi +elif [ "${platform}" == "darwin" ]; then + tarball_name="bitcoin-${version}-osx64.tar.gz" +else + echo "Bitcoin binary distribution not available for platform and architecture" + exit -1 +fi + +binary_url="${url}/${tag}/${tarball_name}" +shasums_url="${url}/${tag}/SHA256SUMS.asc" + +download_bitcoind() { + + cd "${root_dir}/bin" + + echo "Downloading bitcoin: ${binary_url}" + + is_curl=true + if hash curl 2>/dev/null; then + curl --fail -I $binary_url >/dev/null 2>&1 + else + is_curl=false + wget --server-response --spider $binary_url >/dev/null 2>&1 + fi + + if test $? -eq 0; then + if [ "${is_curl}" = true ]; then + curl -L $binary_url > $tarball_name + curl -L $shasums_url > SHA256SUMS.asc + else + wget $binary_url + wget $shasums_url + fi + if test -e "${tarball_name}"; then + echo "Unpacking bitcoin distribution" + tar -xvzf $tarball_name + if test $? -eq 0; then + ln -sf "bitcoin-${version}/bin/bitcoind" + return; + fi + fi + fi + echo "Bitcoin binary distribution could not be downloaded" + exit -1 +} + +verify_download() { + echo "Verifying signatures of bitcoin download" + gpg --verify "${root_dir}/bin/SHA256SUMS.asc" + + if hash shasum 2>/dev/null; then + shasum_cmd="shasum -a 256" + else + shasum_cmd="sha256sum" + fi + + download_sha=$(${shasum_cmd} "${root_dir}/bin/${tarball_name}" | awk '{print $1}') + expected_sha=$(cat "${root_dir}/bin/SHA256SUMS.asc" | grep "${tarball_name}" | awk '{print $1}') + echo "Checksum (download): ${download_sha}" + echo "Checksum (verified): ${expected_sha}" + if [ "${download_sha}" != "${expected_sha}" ]; then + echo -e "\033[1;31mChecksums did NOT match!\033[0m\n" + exit 1 + else + echo -e "\033[1;32mChecksums matched!\033[0m\n" + fi +} + +download=1 +verify=0 + +while [ -n "$1" ]; do + param="$1" + value="$2" + + case $param in + --skip-bitcoin-download) + download=0 + ;; + --verify-bitcoin-download) + verify=1 + ;; + esac + shift +done + +if [ "${download}" = 1 ]; then + download_bitcoind +fi + +if [ "${verify}" = 1 ]; then + verify_download +fi + +exit 0 diff --git a/scripts/install b/scripts/install deleted file mode 100755 index aa76aed6..00000000 --- a/scripts/install +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -platform=`uname -a | awk '{print tolower($1)}'` -arch=`uname -m` -version="0.12.0" -url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12.0-bitcore-beta4" - -cd "${root_dir}/bin" - -if [ "${platform}" == "linux" ]; then - if [ "${arch}" == "x86_64" ]; then - tarball_name="bitcoin-${version}-linux64.tar.gz" - elif [ "${arch}" == "x86_32" ]; then - tarball_name="bitcoin-${version}-linux32.tar.gz" - fi -elif [ "${platform}" == "darwin" ]; then - tarball_name="bitcoin-${version}-osx64.tar.gz" -else - echo "Bitcoin binary distribution not available for platform and architecture" - exit -1 -fi - -binary_url="${url}/${tag}/${tarball_name}" - -echo "Downloading bitcoin: ${binary_url}" - -is_curl=true -if hash curl 2>/dev/null; then - curl --fail -I $binary_url >/dev/null 2>&1 -else - is_curl=false - wget --server-response --spider $binary_url >/dev/null 2>&1 -fi - -if test $? -eq 0; then - if [ "${is_curl}" = true ]; then - curl -L $binary_url > $tarball_name - else - wget $binary_url - fi - if test -e "${tarball_name}"; then - echo "Unpacking bitcoin distribution" - tar -xvzf $tarball_name - if test $? -eq 0; then - ln -sf "bitcoin-${version}/bin/bitcoind" - exit 0 - fi - fi -fi -echo "Bitcoin binary distribution could not be downloaded" -exit -1 From 9e0e9a2c89c855ceeacca72d5ab74d5551707350 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 16:27:00 -0400 Subject: [PATCH 147/299] build: include environment variables for downloading bitcoin for parent modules to specify npm rebuild and install behavior --- scripts/download | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/download b/scripts/download index 220bf560..93a04775 100755 --- a/scripts/download +++ b/scripts/download @@ -85,6 +85,14 @@ verify_download() { download=1 verify=0 +if [ "${SKIP_BITCOIN_DOWNLOAD}" = 1 ]; then + download=0; +fi + +if [ "${VERIFY_BITCOIN_DOWNLOAD}" = 1 ]; then + verify=1; +fi + while [ -n "$1" ]; do param="$1" value="$2" From d958e83f1d73f546280156d3f638dab41411bd02 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 17:20:46 -0400 Subject: [PATCH 148/299] build: add support for nodejs 0.10 For Ubuntu 14.04 Node.js compatibility: http://packages.ubuntu.com/trusty/nodejs --- .travis.yml | 1 + lib/scaffold/add.js | 7 ++++--- lib/scaffold/find-config.js | 3 ++- lib/scaffold/remove.js | 9 +++++---- lib/utils.js | 5 +++++ package.json | 1 + 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e1fcec0..c9912b43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ addons: - gcc-4.8 - libzmq3-dev node_js: + - "v0.10.25" - "v0.12.7" - "v4" script: diff --git a/lib/scaffold/add.js b/lib/scaffold/add.js index 5d0e6eea..76f24bdb 100644 --- a/lib/scaffold/add.js +++ b/lib/scaffold/add.js @@ -5,6 +5,7 @@ var fs = require('fs'); var path = require('path'); var spawn = require('child_process').spawn; var bitcore = require('bitcore-lib'); +var utils = require('../utils'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; @@ -14,7 +15,7 @@ var _ = bitcore.deps._; * @param {Function} done */ function addConfig(configFilePath, service, done) { - $.checkState(path.isAbsolute(configFilePath), 'An absolute path is expected'); + $.checkState(utils.isAbsolutePath(configFilePath), 'An absolute path is expected'); fs.readFile(configFilePath, function(err, data) { if (err) { return done(err); @@ -39,7 +40,7 @@ function addConfig(configFilePath, service, done) { * @param {Function} done */ function addService(configDir, service, done) { - $.checkState(path.isAbsolute(configDir), 'An absolute path is expected'); + $.checkState(utils.isAbsolutePath(configDir), 'An absolute path is expected'); var npm = spawn('npm', ['install', service, '--save'], {cwd: configDir}); npm.stdout.on('data', function(data) { @@ -69,7 +70,7 @@ function add(options, done) { $.checkArgument(_.isObject(options)); $.checkArgument(_.isFunction(done)); $.checkArgument( - _.isString(options.path) && path.isAbsolute(options.path), + _.isString(options.path) && utils.isAbsolutePath(options.path), 'An absolute path is expected' ); $.checkArgument(Array.isArray(options.services)); diff --git a/lib/scaffold/find-config.js b/lib/scaffold/find-config.js index f1c70b53..a4306e8b 100644 --- a/lib/scaffold/find-config.js +++ b/lib/scaffold/find-config.js @@ -5,6 +5,7 @@ var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var path = require('path'); var fs = require('fs'); +var utils = require('../utils'); /** * Will return the path and bitcore-node configuration @@ -12,7 +13,7 @@ var fs = require('fs'); */ function findConfig(cwd) { $.checkArgument(_.isString(cwd), 'Argument should be a string'); - $.checkArgument(path.isAbsolute(cwd), 'Argument should be an absolute path'); + $.checkArgument(utils.isAbsolutePath(cwd), 'Argument should be an absolute path'); var directory = String(cwd); while (!fs.existsSync(path.resolve(directory, 'bitcore-node.json'))) { directory = path.resolve(directory, '../'); diff --git a/lib/scaffold/remove.js b/lib/scaffold/remove.js index d17dabf4..6d866d6f 100644 --- a/lib/scaffold/remove.js +++ b/lib/scaffold/remove.js @@ -8,6 +8,7 @@ var spawn = require('child_process').spawn; var bitcore = require('bitcore-lib'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; +var utils = require('../utils'); /** * Will remove a service from bitcore-node.json @@ -16,7 +17,7 @@ var _ = bitcore.deps._; * @param {Function} done */ function removeConfig(configFilePath, service, done) { - $.checkArgument(path.isAbsolute(configFilePath), 'An absolute path is expected'); + $.checkArgument(utils.isAbsolutePath(configFilePath), 'An absolute path is expected'); fs.readFile(configFilePath, function(err, data) { if (err) { return done(err); @@ -47,7 +48,7 @@ function removeConfig(configFilePath, service, done) { * @param {Function} done */ function uninstallService(configDir, service, done) { - $.checkArgument(path.isAbsolute(configDir), 'An absolute path is expected'); + $.checkArgument(utils.isAbsolutePath(configDir), 'An absolute path is expected'); $.checkArgument(_.isString(service), 'A string is expected for the service argument'); var child = spawn('npm', ['uninstall', service, '--save'], {cwd: configDir}); @@ -76,7 +77,7 @@ function uninstallService(configDir, service, done) { * @param {Function} done */ function removeService(configDir, service, done) { - $.checkArgument(path.isAbsolute(configDir), 'An absolute path is expected'); + $.checkArgument(utils.isAbsolutePath(configDir), 'An absolute path is expected'); $.checkArgument(_.isString(service), 'A string is expected for the service argument'); // check if the service is installed @@ -109,7 +110,7 @@ function remove(options, done) { $.checkArgument(_.isObject(options)); $.checkArgument(_.isFunction(done)); $.checkArgument( - _.isString(options.path) && path.isAbsolute(options.path), + _.isString(options.path) && utils.isAbsolutePath(options.path), 'An absolute path is expected' ); $.checkArgument(Array.isArray(options.services)); diff --git a/lib/utils.js b/lib/utils.js index 72cc58c1..cae2a5fe 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -21,4 +21,9 @@ utils.startAtZero = function startAtZero(obj, key) { } }; +utils.isAbsolutePath = require('path').isAbsolute; +if (!utils.isAbsolutePath) { + utils.isAbsolutePath = require('path-is-absolute'); +} + module.exports = utils; diff --git a/package.json b/package.json index fb4d1b2a..94fbf0bf 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "lru-cache": "^4.0.1", "mkdirp": "0.5.0", "npm": "^2.14.1", + "path-is-absolute": "^1.0.0", "semver": "^5.0.1", "socket.io": "^1.4.5", "socket.io-client": "^1.4.5", From d28f8567f12ad6de5529b2984dd075011f747a61 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Apr 2016 14:32:51 -0400 Subject: [PATCH 149/299] bitcoind: handle unexpected process exits --- lib/services/bitcoind.js | 108 ++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index c79bde47..90614201 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -52,6 +52,9 @@ function Bitcoin(options) { this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; + // spawn restart setting + this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME; + // try all interval this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; @@ -70,6 +73,7 @@ Bitcoin.dependencies = []; Bitcoin.DEFAULT_MAX_HISTORY = 10; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; +Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; @@ -613,6 +617,39 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) { }); }; +Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { + var spawnOptions = this.options.spawn; + var pidPath = spawnOptions.datadir + '/bitcoind.pid'; + + function stopProcess() { + fs.readFile(pidPath, 'utf8', function(err, pid) { + if (err && err.code === 'ENOENT') { + // pid file doesn't exist we can continue + return callback(null); + } else if (err) { + return callback(err); + } + pid = parseInt(pid); + log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); + try { + process.kill(pid, 'SIGINT'); + } catch(err) { + if (err && err.code === 'ESRCH') { + log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid); + return callback(null); + } else if(err) { + return callback(err); + } + } + setTimeout(function() { + stopProcess(); + }, 10000); + }); + } + + stopProcess(); +}; + Bitcoin.prototype._spawnChildProcess = function(callback) { var self = this; @@ -634,43 +671,68 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { if (self._getNetworkOption()) { options.push(self._getNetworkOption()); } - self.spawn.process = spawn(this.spawn.exec, options, {stdio: 'inherit'}); - self.spawn.process.on('error', function(err) { - log.error(err); - }); - - async.retry({times: 60, interval: self.startRetryInterval}, function(done) { - if (self.node.stopping) { - return done(new Error('Stopping while trying to connect to bitcoind.')); + self._stopSpawnedBitcoin(function(err) { + if (err) { + return callback(err); } - node.client = new BitcoinRPC({ - protocol: 'http', - host: '127.0.0.1', - port: self.spawn.config.rpcport, - user: self.spawn.config.rpcuser, - pass: self.spawn.config.rpcpassword + log.info('Starting bitcoin process'); + self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'}); + + self.spawn.process.on('error', function(err) { + self.emit('error', err); }); - self._loadTipFromNode(node, done); + self.spawn.process.once('exit', function(code) { + if (!self.node.stopping) { + log.warn('Bitcoin process unexpectedly exited with code:', code); + log.warn('Restarting bitcoin child process in ' + self.spawnRestartTime + 'ms'); + setTimeout(function() { + self._spawnChildProcess(function(err) { + if (err) { + return self.emit('error', err); + } + log.warn('Bitcoin process restarted'); + }); + }, self.spawnRestartTime); + } + }); - }, function(err) { - if (err) { - return callback(err); - } + async.retry({times: 60, interval: self.startRetryInterval}, function(done) { + if (self.node.stopping) { + return done(new Error('Stopping while trying to connect to bitcoind.')); + } - self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); + node.client = new BitcoinRPC({ + protocol: 'http', + host: '127.0.0.1', + port: self.spawn.config.rpcport, + user: self.spawn.config.rpcuser, + pass: self.spawn.config.rpcpassword + }); - self._checkReindex(node, function(err) { + self._loadTipFromNode(node, done); + + }, function(err) { if (err) { return callback(err); } - self._checkSyncedAndSubscribeZmqEvents(node); - callback(null, node); + + self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); + + self._checkReindex(node, function(err) { + if (err) { + return callback(err); + } + self._checkSyncedAndSubscribeZmqEvents(node); + callback(null, node); + }); + }); }); + }; Bitcoin.prototype._connectProcess = function(config, callback) { From c1e9d5a3d9aebe5293a1b35164e5b845c199660b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Apr 2016 17:29:40 -0400 Subject: [PATCH 150/299] test: added tests for stopSpawnedBitcoin --- lib/services/bitcoind.js | 12 +++++-- regtest/bitcoind.js | 1 + regtest/p2p.js | 1 + test/services/bitcoind.unit.js | 60 ++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 90614201..da702678 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -54,6 +54,7 @@ function Bitcoin(options) { // spawn restart setting this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME; + this.spawnStopTime = options.spawnStopTime || Bitcoin.DEFAULT_SPAWN_STOP_TIME; // try all interval this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; @@ -62,6 +63,9 @@ function Bitcoin(options) { // available bitcoind nodes this._initClients(); + // for testing purposes + this._process = options.process || process; + this.on('error', function(err) { log.error(err.stack); }); @@ -74,6 +78,7 @@ Bitcoin.DEFAULT_MAX_HISTORY = 10; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000; +Bitcoin.DEFAULT_SPAWN_STOP_TIME = 10000; Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; @@ -618,6 +623,7 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) { }; Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { + var self = this; var spawnOptions = this.options.spawn; var pidPath = spawnOptions.datadir + '/bitcoind.pid'; @@ -630,9 +636,9 @@ Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { return callback(err); } pid = parseInt(pid); - log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); try { - process.kill(pid, 'SIGINT'); + log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); + self._process.kill(pid, 'SIGINT'); } catch(err) { if (err && err.code === 'ESRCH') { log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid); @@ -643,7 +649,7 @@ Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { } setTimeout(function() { stopProcess(); - }, 10000); + }, self.spawnStopTime); }); } diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 485b33dc..9d729845 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -131,6 +131,7 @@ describe('Bitcoind Functionality', function() { after(function(done) { this.timeout(20000); + bitcoind.node.stopping = true; bitcoind.stop(function(err, result) { done(); }); diff --git a/regtest/p2p.js b/regtest/p2p.js index acf718d1..4af160bd 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -163,6 +163,7 @@ describe('P2P Functionality', function() { this.timeout(20000); peer.on('disconnect', function() { log.info('Peer disconnected'); + bitcoind.node.stopping = true; bitcoind.stop(function(err, result) { done(); }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 7d3d227a..1699e554 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1001,6 +1001,66 @@ describe('Bitcoin Service', function() { }); }); + describe('#_stopSpawnedProcess', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'warn'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('it will kill process and resume', function(done) { + var readFile = sandbox.stub(); + readFile.onCall(0).callsArgWith(2, null, '4321'); + var error = new Error('Test error'); + error.code = 'ENOENT'; + readFile.onCall(1).callsArgWith(2, error); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFile: readFile + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind.spawnStopTime = 1; + bitcoind._process = {}; + bitcoind._process.kill = sinon.stub(); + bitcoind._stopSpawnedBitcoin(function(err) { + if (err) { + return done(err); + } + bitcoind._process.kill.callCount.should.equal(1); + log.warn.callCount.should.equal(1); + done(); + }); + }); + it('it will attempt to kill process and resume', function(done) { + var readFile = sandbox.stub(); + readFile.onCall(0).callsArgWith(2, null, '4321'); + var error = new Error('Test error'); + error.code = 'ENOENT'; + readFile.onCall(1).callsArgWith(2, error); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFile: readFile + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind.spawnStopTime = 1; + bitcoind._process = {}; + var error2 = new Error('Test error'); + error2.code = 'ESRCH'; + bitcoind._process.kill = sinon.stub().throws(error2); + bitcoind._stopSpawnedBitcoin(function(err) { + if (err) { + return done(err); + } + bitcoind._process.kill.callCount.should.equal(1); + log.warn.callCount.should.equal(2); + done(); + }); + }); + }); + describe('#_spawnChildProcess', function() { it('will give error from spawn config', function(done) { var bitcoind = new BitcoinService(baseConfig); From 92bae5f09a259399f755329263ba6970e403fcc9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 27 Apr 2016 10:57:37 -0400 Subject: [PATCH 151/299] general: code cleanup, refactoring and formatting --- lib/cli/bitcore.js | 2 +- lib/cli/bitcored.js | 2 +- lib/logger.js | 2 +- lib/scaffold/start.js | 219 ++++++++++++++++++--------------- lib/service.js | 4 +- lib/services/bitcoind.js | 85 +++++++------ test/bus.integration.js | 5 +- test/services/bitcoind.unit.js | 2 +- test/services/web.unit.js | 6 +- 9 files changed, 172 insertions(+), 155 deletions(-) diff --git a/lib/cli/bitcore.js b/lib/cli/bitcore.js index 3ea4771d..bcbaa5ee 100644 --- a/lib/cli/bitcore.js +++ b/lib/cli/bitcore.js @@ -9,7 +9,7 @@ function main(parentServicesPath, additionalServices) { moduleName: 'bitcore-node', configName: 'bitcore-node', processTitle: 'bitcore' - }).on('require', function (name, module) { + }).on('require', function (name) { console.log('Loading:', name); }).on('requireFail', function (name, err) { console.log('Unable to load:', name, err); diff --git a/lib/cli/bitcored.js b/lib/cli/bitcored.js index 158b2ee7..4462a8ea 100644 --- a/lib/cli/bitcored.js +++ b/lib/cli/bitcored.js @@ -9,7 +9,7 @@ function main(parentServicesPath, additionalServices) { moduleName: 'bitcore-node', configName: 'bitcore-node', processTitle: 'bitcored' - }).on('require', function (name, module) { + }).on('require', function (name) { console.log('Loading:', name); }).on('requireFail', function (name, err) { console.log('Unable to load:', name, err); diff --git a/lib/logger.js b/lib/logger.js index eb627650..8d9f48e0 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -45,7 +45,7 @@ Logger.prototype.warn = function() { * Proxies console.log with color and arg parsing magic * #_log */ -Logger.prototype._log = function(color, type) { +Logger.prototype._log = function(color) { if (process.env.NODE_ENV === 'test') { return; } diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index fa1365dc..bba6b5de 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -5,14 +5,116 @@ var BitcoreNode = require('../node'); var index = require('../'); var bitcore = require('bitcore-lib'); var _ = bitcore.deps._; -var $ = bitcore.util.preconditions; var log = index.log; -var child_process = require('child_process'); -var fs = require('fs'); var shuttingDown = false; log.debug = function() {}; +/** + * This function will instantiate and start a Node, requiring the necessary service + * modules, and registering event handlers. + * @param {Object} options + * @param {Object} options.servicesPath - The path to the location of service modules + * @param {String} options.path - The absolute path of the configuration file + * @param {Object} options.config - The parsed bitcore-node.json configuration file + * @param {Array} options.config.services - An array of services names. + * @param {Object} options.config.servicesConfig - Parameters to pass to each service + * @param {String} options.config.datadir - A relative (to options.path) or absolute path to the datadir + * @param {String} options.config.network - 'livenet', 'testnet' or 'regtest + * @param {Number} options.config.port - The port to use for the web service + */ +function start(options) { + /* jshint maxstatements: 20 */ + + var fullConfig = _.clone(options.config); + + var servicesPath; + if (options.servicesPath) { + servicesPath = options.servicesPath; // services are in a different directory than the config + } else { + servicesPath = options.path; // defaults to the same directory + } + + fullConfig.path = path.resolve(options.path, './bitcore-node.json'); + + if (fullConfig.datadir) { + throw new TypeError( + 'Configuration file (' + fullConfig.path + ') is not compatible with this version.' + + ' Please see https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md' + + ' for upgrade details.' + ); + } + + fullConfig.services = start.setupServices(require, servicesPath, options.config); + + var node = new BitcoreNode(fullConfig); + + // set up the event handlers for logging sync information + start.registerSyncHandlers(node); + + // setup handlers for uncaught exceptions and ctrl+c + start.registerExitHandlers(process, node); + + node.on('ready', function() { + log.info('Bitcore Node ready'); + }); + + node.on('error', function(err) { + log.error(err); + }); + + node.start(function(err) { + if(err) { + log.error('Failed to start services'); + if (err.stack) { + log.error(err.stack); + } + start.cleanShutdown(process, node); + } + }); + + return node; + +} + +/** + * Checks a service for the expected methods + * @param {Object} service + */ +function checkService(service) { + // check that the service supports expected methods + if (!service.module.prototype || + !service.module.dependencies || + !service.module.prototype.start || + !service.module.prototype.stop) { + throw new Error( + 'Could not load service "' + service.name + '" as it does not support necessary methods.' + ); + } +} + +/** + * Will require a module from local services directory first + * and then from available node_modules + * @param {Function} req + * @param {Object} service + */ +function loadModule(req, service) { + try { + // first try in the built-in bitcore-node services directory + service.module = req(path.resolve(__dirname, '../services/' + service.name)); + } catch(e) { + + // check if the package.json specifies a specific file to use + var servicePackage = req(service.name + '/package.json'); + var serviceModule = service.name; + if (servicePackage.bitcoreNode) { + serviceModule = service.name + '/' + servicePackage.bitcoreNode; + } + service.module = req(serviceModule); + } +} + /** * This function will loop over the configuration for services and require the * specified modules, and assemble an array in this format: @@ -42,29 +144,8 @@ function setupServices(req, servicesPath, config) { var hasConfig = config.servicesConfig && config.servicesConfig[service.name]; service.config = hasConfig ? config.servicesConfig[service.name] : {}; - try { - // first try in the built-in bitcore-node services directory - service.module = req(path.resolve(__dirname, '../services/' + service.name)); - } catch(e) { - - // check if the package.json specifies a specific file to use - var servicePackage = req(service.name + '/package.json'); - var serviceModule = service.name; - if (servicePackage.bitcoreNode) { - serviceModule = service.name + '/' + servicePackage.bitcoreNode; - } - service.module = req(serviceModule); - } - - // check that the service supports expected methods - if (!service.module.prototype || - !service.module.dependencies || - !service.module.prototype.start || - !service.module.prototype.stop) { - throw new Error( - 'Could not load service "' + service.name + '" as it does not support necessary methods.' - ); - } + loadModule(req, service); + checkService(service); services.push(service); } @@ -97,7 +178,7 @@ function registerSyncHandlers(node, delay) { clearInterval(interval); logSyncStatus(); }); - node.services.db.on('addblock', function(block) { + node.services.db.on('addblock', function() { count++; // Initialize logging if not already instantiated if (!interval) { @@ -132,20 +213,6 @@ function cleanShutdown(_process, node) { }); } -/** - * Will register event handlers to stop the node for `process` events - * `uncaughtException` and `SIGINT`. - * @param {Object} _process - The Node.js process - * @param {Node} node - */ -function registerExitHandlers(_process, node) { - //catches uncaught exceptions - _process.on('uncaughtException', exitHandler.bind(null, {exit:true}, _process, node)); - - //catches ctrl+c event - _process.on('SIGINT', exitHandler.bind(null, {sigint:true}, _process, node)); -} - /** * Will handle all the shutdown tasks that need to take place to ensure a safe exit * @param {Object} options @@ -177,69 +244,17 @@ function exitHandler(options, _process, node, err) { } /** - * This function will instantiate and start a Node, requiring the necessary service - * modules, and registering event handlers. - * @param {Object} options - * @param {Object} options.servicesPath - The path to the location of service modules - * @param {String} options.path - The absolute path of the configuration file - * @param {Object} options.config - The parsed bitcore-node.json configuration file - * @param {Array} options.config.services - An array of services names. - * @param {Object} options.config.servicesConfig - Parameters to pass to each service - * @param {String} options.config.datadir - A relative (to options.path) or absolute path to the datadir - * @param {String} options.config.network - 'livenet', 'testnet' or 'regtest - * @param {Number} options.config.port - The port to use for the web service + * Will register event handlers to stop the node for `process` events + * `uncaughtException` and `SIGINT`. + * @param {Object} _process - The Node.js process + * @param {Node} node */ -function start(options) { - - var fullConfig = _.clone(options.config); - - var servicesPath; - if (options.servicesPath) { - servicesPath = options.servicesPath; // services are in a different directory than the config - } else { - servicesPath = options.path; // defaults to the same directory - } - - fullConfig.path = path.resolve(options.path, './bitcore-node.json'); - - if (fullConfig.datadir) { - throw new TypeError( - 'Configuration file (' + fullConfig.path + ') is not compatible with this version.' + - ' Please see https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md' + - ' for upgrade details.' - ); - } - - fullConfig.services = start.setupServices(require, servicesPath, options.config); - - var node = new BitcoreNode(fullConfig); - - // set up the event handlers for logging sync information - start.registerSyncHandlers(node); - - // setup handlers for uncaught exceptions and ctrl+c - start.registerExitHandlers(process, node); - - node.on('ready', function() { - log.info('Bitcore Node ready'); - }); - - node.on('error', function(err) { - log.error(err); - }); - - node.start(function(err) { - if(err) { - log.error('Failed to start services'); - if (err.stack) { - log.error(err.stack); - } - start.cleanShutdown(process, node); - } - }); - - return node; +function registerExitHandlers(_process, node) { + //catches uncaught exceptions + _process.on('uncaughtException', exitHandler.bind(null, {exit:true}, _process, node)); + //catches ctrl+c event + _process.on('SIGINT', exitHandler.bind(null, {sigint:true}, _process, node)); } module.exports = start; diff --git a/lib/service.js b/lib/service.js index cc47ce1a..08d0ef5a 100644 --- a/lib/service.js +++ b/lib/service.js @@ -78,7 +78,7 @@ Service.prototype.stop = function(done) { * Setup express routes * @param {Express} app */ -Service.prototype.setupRoutes = function(app) { +Service.prototype.setupRoutes = function() { // Setup express routes here }; @@ -86,6 +86,4 @@ Service.prototype.getRoutePrefix = function() { return this.name; }; - - module.exports = Service; diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index da702678..b749c89e 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1,7 +1,6 @@ 'use strict'; var fs = require('fs'); -var path = require('path'); var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); @@ -16,7 +15,6 @@ var _ = bitcore.deps._; var index = require('../'); var errors = index.errors; var log = index.log; -var utils = require('../utils'); var Service = require('../service'); var Transaction = require('../transaction'); @@ -30,6 +28,7 @@ var Transaction = require('../transaction'); * @param {Node} options.node - A reference to the node */ function Bitcoin(options) { + /* jshint maxstatements: 20 */ if (!(this instanceof Bitcoin)) { return new Bitcoin(options); } @@ -1022,6 +1021,7 @@ Bitcoin.prototype._getHeightRangeQuery = function(options, clone) { * @param {Function} callback */ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { + /* jshint maxstatements: 16 */ var self = this; var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var rangeQuery = false; @@ -1091,6 +1091,48 @@ Bitcoin.prototype._getConfirmationsDetail = function(transaction) { return Math.max(0, confirmations); }; +Bitcoin.prototype._getAddressDetailsForInput = function(input, inputIndex, result, addressStrings) { + if (!input.script) { + return; + } + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress) { + var inputAddressString = inputAddress.toString(); + if (addressStrings.indexOf(inputAddressString) >= 0) { + if (!result.addresses[inputAddressString]) { + result.addresses[inputAddressString] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[inputAddressString].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; + } + } +}; + +Bitcoin.prototype._getAddressDetailsForOutput = function(output, outputIndex, result, addressStrings) { + if (!output.script) { + return; + } + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress) { + var outputAddressString = outputAddress.toString(); + if (addressStrings.indexOf(outputAddressString) >= 0) { + if (!result.addresses[outputAddressString]) { + result.addresses[outputAddressString] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[outputAddressString].outputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; + } + } +}; + Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addressStrings) { var result = { addresses: {}, @@ -1099,50 +1141,15 @@ Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addre for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { var input = transaction.inputs[inputIndex]; - if (!input.script) { - continue; - } - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { - var inputAddressString = inputAddress.toString(); - if (addressStrings.indexOf(inputAddressString) >= 0) { - if (!result.addresses[inputAddressString]) { - result.addresses[inputAddressString] = { - inputIndexes: [inputIndex], - outputIndexes: [] - }; - } else { - result.addresses[inputAddressString].inputIndexes.push(inputIndex); - } - result.satoshis -= input.output.satoshis; - } - } + this._getAddressDetailsForInput(input, inputIndex, result, addressStrings); } for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { var output = transaction.outputs[outputIndex]; - if (!output.script) { - continue; - } - var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress) { - var outputAddressString = outputAddress.toString(); - if (addressStrings.indexOf(outputAddressString) >= 0) { - if (!result.addresses[outputAddressString]) { - result.addresses[outputAddressString] = { - inputIndexes: [], - outputIndexes: [outputIndex] - }; - } else { - result.addresses[outputAddressString].outputIndexes.push(outputIndex); - } - result.satoshis += output.satoshis; - } - } + this._getAddressDetailsForOutput(output, outputIndex, result, addressStrings); } return result; - }; /** diff --git a/test/bus.integration.js b/test/bus.integration.js index 98bc8ab5..0573695d 100644 --- a/test/bus.integration.js +++ b/test/bus.integration.js @@ -1,12 +1,13 @@ +'use strict'; + var Service = require('../lib/service'); var BitcoreNode = require('../lib/node'); var util = require('util'); -var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); var TestService = function(options) { this.node = options.node; -} +}; util.inherits(TestService, Service); TestService.dependencies = []; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 1699e554..7f030668 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -24,7 +24,7 @@ var BitcoinService = proxyquire('../../lib/services/bitcoind', { var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.bitcoin.conf'), 'utf8'); describe('Bitcoin Service', function() { - var txhex = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000'; + var txhex = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000'; var baseConfig = { node: { diff --git a/test/services/web.unit.js b/test/services/web.unit.js index 5415af2d..5a582784 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -22,15 +22,11 @@ var fakeSocket = new EventEmitter(); fakeSocket.on('test/event1', function(data) { data.should.equal('testdata'); - done(); }); fakeSocketListener.emit('connection', fakeSocket); - fakeSocket.emit('subscribe', 'test/event1'); - - var WebService = proxyquire('../../lib/services/web', {http: httpStub, https: httpsStub, fs: fsStub}); describe('WebService', function() { @@ -323,7 +319,7 @@ describe('WebService', function() { var message = { method: 'two', params: [1, 2] - } + }; web.socketMessageHandler(message, function(response) { should.exist(response.error); response.error.message.should.equal('Method Not Found'); From 271dcd89021e046c83bf0fc62d75a9a1c9538a94 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 27 Apr 2016 11:13:55 -0400 Subject: [PATCH 152/299] build: add jshint to scripts and ci build --- .travis.yml | 1 + package.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c9912b43..9d896336 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,4 @@ node_js: script: - npm run regtest - npm run test + - npm run jshint diff --git a/package.json b/package.json index 94fbf0bf..a2c388ce 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "verify": "./scripts/download --skip-bitcoin-download --verify-bitcoin-download", "test": "NODE_ENV=test mocha -R spec --recursive", "regtest": "./scripts/regtest", + "jshint": "jshint --reporter=node_modules/jshint-stylish ./lib", "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" }, "tags": [ @@ -67,6 +68,8 @@ "benchmark": "1.0.0", "bitcore-p2p": "^1.1.0", "chai": "^3.5.0", + "jshint": "^2.9.2", + "jshint-stylish": "^2.1.0", "mocha": "^2.4.5", "proxyquire": "^1.3.1", "rimraf": "^2.4.2", From ea792b692f8b106fece67bccf60c3906fd2ac192 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 27 Apr 2016 12:00:47 -0400 Subject: [PATCH 153/299] scaffold: remove outdated logging of db service sync status --- lib/scaffold/start.js | 48 ------------------------------------- test/scaffold/start.unit.js | 28 ---------------------- 2 files changed, 76 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index bba6b5de..a01c0d16 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -49,9 +49,6 @@ function start(options) { var node = new BitcoreNode(fullConfig); - // set up the event handlers for logging sync information - start.registerSyncHandlers(node); - // setup handlers for uncaught exceptions and ctrl+c start.registerExitHandlers(process, node); @@ -153,50 +150,6 @@ function setupServices(req, servicesPath, config) { return services; } -/** - * Will register event handlers to log the current db sync status. - * @param {Node} node - */ -function registerSyncHandlers(node, delay) { - - delay = delay || 10000; - var interval = false; - var count = 0; - - function logSyncStatus() { - log.info( - 'Database Sync Status: Tip:', node.services.db.tip.hash, - 'Height:', node.services.db.tip.__height, - 'Rate:', count/10, 'blocks per second' - ); - } - - node.on('ready', function() { - - if (node.services.db) { - node.on('synced', function() { - clearInterval(interval); - logSyncStatus(); - }); - node.services.db.on('addblock', function() { - count++; - // Initialize logging if not already instantiated - if (!interval) { - interval = setInterval(function() { - logSyncStatus(); - count = 0; - }, delay); - } - }); - } - - }); - - node.on('stopping', function() { - clearInterval(interval); - }); -} - /** * Will shutdown a node and then the process * @param {Object} _process - The Node.js process object @@ -260,6 +213,5 @@ function registerExitHandlers(_process, node) { module.exports = start; module.exports.registerExitHandlers = registerExitHandlers; module.exports.exitHandler = exitHandler; -module.exports.registerSyncHandlers = registerSyncHandlers; module.exports.setupServices = setupServices; module.exports.cleanShutdown = cleanShutdown; diff --git a/test/scaffold/start.unit.js b/test/scaffold/start.unit.js index 85b2c507..38d14feb 100644 --- a/test/scaffold/start.unit.js +++ b/test/scaffold/start.unit.js @@ -96,34 +96,6 @@ describe('#start', function() { }).should.throw('Could not load service'); }); }); - describe('#registerSyncHandlers', function() { - it('will log the sync status at an interval', function(done) { - var log = { - info: sinon.stub() - }; - var registerSyncHandlers = proxyquire('../../lib/scaffold/start', { - '../': { - log: log - } - }).registerSyncHandlers; - var node = new EventEmitter(); - node.services = { - db: new EventEmitter() - }; - node.services.db.tip = { - hash: 'hash', - __height: 10 - }; - registerSyncHandlers(node, 10); - node.emit('ready'); - node.services.db.emit('addblock'); - setTimeout(function() { - node.emit('synced'); - log.info.callCount.should.be.within(3, 4); - done(); - }, 35); - }); - }); describe('#cleanShutdown', function() { it('will call node stop and process exit', function() { var log = { From 24d1bc82e9e3ff9ab49f5b292624fddd5773cf54 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 27 Apr 2016 13:46:50 -0400 Subject: [PATCH 154/299] logger: added option to disable formatting - systemd journalctl includes timestamps in log messages already - updated logger to use console.error, console.warn, console.info, and etc. --- lib/logger.js | 27 ++++++++++----- lib/node.js | 6 ++++ package.json | 4 +-- test/logger.unit.js | 83 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 test/logger.unit.js diff --git a/lib/logger.js b/lib/logger.js index 8d9f48e0..4084c2cc 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,14 +1,22 @@ 'use strict'; +var bitcore = require('bitcore-lib'); +var _ = bitcore.deps._; var colors = require('colors/safe'); /** * Wraps console.log with some special magic * @constructor */ -function Logger() { +function Logger(options) { + if (!options) { + options = {}; + } + this.formatting = _.isUndefined(options.formatting) ? Logger.DEFAULT_FORMATTING : options.formatting; } +Logger.DEFAULT_FORMATTING = true; + /** * Prints an info message * #info @@ -46,16 +54,17 @@ Logger.prototype.warn = function() { * #_log */ Logger.prototype._log = function(color) { - if (process.env.NODE_ENV === 'test') { - return; - } - var args = Array.prototype.slice.call(arguments); args = args.slice(1); - var date = new Date(); - var typeString = colors[color].italic(args.shift() + ':'); - args[0] = '[' + date.toISOString() + ']' + ' ' + typeString + ' ' + args[0]; - console.log.apply(console, args); + var level = args.shift(); + + if (this.formatting) { + var date = new Date(); + var typeString = colors[color].italic(level + ':'); + args[0] = '[' + date.toISOString() + ']' + ' ' + typeString + ' ' + args[0]; + } + var fn = console[level] || console.log; + fn.apply(console, args); }; module.exports = Logger; diff --git a/lib/node.js b/lib/node.js index 4d80fc48..75431598 100644 --- a/lib/node.js +++ b/lib/node.js @@ -27,6 +27,7 @@ var errors = require('./errors'); * ``` * * @param {Object} config - The configuration of the node + * @param {Array} config.formatLogs - Option to disable formatting of logs * @param {Array} config.services - The array of services * @param {Number} config.port - The HTTP port for services * @param {Boolean} config.https - Enable https @@ -43,6 +44,11 @@ function Node(config) { this.configPath = config.path; this.errors = errors; this.log = log; + + if (!_.isUndefined(config.formatLogs)) { + this.log.formatting = config.formatLogs ? true : false; + } + this.network = null; this.services = {}; this._unloadedServices = []; diff --git a/package.json b/package.json index a2c388ce..2c61bebc 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,10 @@ "scripts": { "preinstall": "./scripts/download", "verify": "./scripts/download --skip-bitcoin-download --verify-bitcoin-download", - "test": "NODE_ENV=test mocha -R spec --recursive", + "test": "mocha -R spec --recursive", "regtest": "./scripts/regtest", "jshint": "jshint --reporter=node_modules/jshint-stylish ./lib", - "coverage": "NODE_ENV=test istanbul cover _mocha -- --recursive" + "coverage": "istanbul cover _mocha -- --recursive" }, "tags": [ "bitcoin", diff --git a/test/logger.unit.js b/test/logger.unit.js new file mode 100644 index 00000000..75522814 --- /dev/null +++ b/test/logger.unit.js @@ -0,0 +1,83 @@ +'use strict'; + +var sinon = require('sinon'); +var chai = require('chai'); +var should = chai.should(); +var Logger = require('../lib/logger'); + +describe('Logger', function() { + var sandbox = sinon.sandbox.create(); + afterEach(function() { + sandbox.restore(); + }); + + it('will instatiate without options', function() { + var logger = new Logger(); + should.exist(logger); + logger.formatting.should.equal(true); + }); + + it('will instatiate with formatting option', function() { + var logger = new Logger({ + formatting: false + }); + logger.formatting.should.equal(false); + var logger2 = new Logger({ + formatting: true + }); + logger2.formatting.should.equal(true); + }); + + it('will log with formatting', function() { + var logger = new Logger({formatting: true}); + + sandbox.stub(console, 'info'); + logger.info('Test info log'); + console.info.callCount.should.equal(1); + console.info.restore(); + + sandbox.stub(console, 'error'); + logger.error(new Error('Test error log')); + console.error.callCount.should.equal(1); + console.error.restore(); + + sandbox.stub(console, 'log'); + logger.debug('Test debug log'); + console.log.callCount.should.equal(1); + console.log.restore(); + + sandbox.stub(console, 'warn'); + logger.warn('Test warn log'); + console.warn.callCount.should.equal(1); + console.warn.restore(); + }); + + it('will log without formatting', function() { + var logger = new Logger({formatting: false}); + + sandbox.stub(console, 'info'); + logger.info('Test info log'); + console.info.callCount.should.equal(1); + should.not.exist(console.info.args[0][0].match(/^\[/)); + console.info.restore(); + + sandbox.stub(console, 'error'); + logger.error(new Error('Test error log')); + console.error.callCount.should.equal(1); + console.error.args[0][0].should.be.instanceof(Error); + console.error.restore(); + + sandbox.stub(console, 'log'); + logger.debug('Test debug log'); + console.log.callCount.should.equal(1); + should.equal(console.log.args[0][0].match(/^\[/), null); + console.log.restore(); + + sandbox.stub(console, 'warn'); + logger.warn('Test warn log'); + console.warn.callCount.should.equal(1); + should.equal(console.warn.args[0][0].match(/^\[/), null); + console.warn.restore(); + }); + +}); From d969ad7fb673370c657a3ccaf939a48952ed7436 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 27 Apr 2016 14:38:58 -0400 Subject: [PATCH 155/299] build: include bitcoind in package.json bin --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c61bebc..671a30e5 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ } ], "bin": { - "bitcore-node": "./bin/bitcore-node" + "bitcore-node": "./bin/bitcore-node", + "bitcoind": "./bin/bitcoind" }, "scripts": { "preinstall": "./scripts/download", From c22f6505eb688a5634456cd5f1187a44a1ddd370 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Apr 2016 12:04:07 -0400 Subject: [PATCH 156/299] bitcoind: reduce duplicate tx messages remember a larger number of tx zmq messages to not emit a transaction twice once from the block and another from the mempool --- lib/services/bitcoind.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b749c89e..4077c848 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -111,7 +111,7 @@ Bitcoin.prototype._initCaches = function() { this.blockCache = LRU(144); this.rawBlockCache = LRU(72); this.blockHeaderCache = LRU(288); - this.zmqKnownTransactions = LRU(50); + this.zmqKnownTransactions = LRU(5000); this.zmqKnownBlocks = LRU(50); this.lastTip = 0; this.lastTipTimeout = false; @@ -467,7 +467,7 @@ Bitcoin.prototype._updateTip = function(node, message) { Bitcoin.prototype._zmqTransactionHandler = function(node, message) { var self = this; - var id = message.toString('binary'); + var id = bitcore.crypto.Hash.sha256sha256(message).toString('binary'); if (!self.zmqKnownTransactions.get(id)) { self.zmqKnownTransactions.set(id, true); self.emit('tx', message); From 2e912af9b4047c587226c5274bfabfe93579b567 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Apr 2016 12:34:26 -0400 Subject: [PATCH 157/299] bitcoind: subscribe to zmq event closer to 100% sync Instead of subscribing at >= 0.995 subscribe at >= 0.9999 progress --- lib/services/bitcoind.js | 38 ++++++++++++++++++++-------------- test/services/bitcoind.unit.js | 4 ++-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4077c848..4d24d1d7 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -28,7 +28,6 @@ var Transaction = require('../transaction'); * @param {Node} options.node - A reference to the node */ function Bitcoin(options) { - /* jshint maxstatements: 20 */ if (!(this instanceof Bitcoin)) { return new Bitcoin(options); } @@ -46,18 +45,8 @@ function Bitcoin(options) { this.subscriptions.rawtransaction = []; this.subscriptions.hashblock = []; - // limits - this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; - this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; - this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; - - // spawn restart setting - this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME; - this.spawnStopTime = options.spawnStopTime || Bitcoin.DEFAULT_SPAWN_STOP_TIME; - - // try all interval - this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; - this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; + // set initial settings + this._initDefaults(options); // available bitcoind nodes this._initClients(); @@ -75,6 +64,7 @@ Bitcoin.dependencies = []; Bitcoin.DEFAULT_MAX_HISTORY = 10; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; +Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS = 0.9999; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000; Bitcoin.DEFAULT_SPAWN_STOP_TIME = 10000; @@ -97,6 +87,24 @@ Bitcoin.DEFAULT_CONFIG_SETTINGS = { uacomment: 'bitcore' }; +Bitcoin.prototype._initDefaults = function(options) { + // limits + this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; + this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; + this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; + + // spawn restart setting + this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME; + this.spawnStopTime = options.spawnStopTime || Bitcoin.DEFAULT_SPAWN_STOP_TIME; + + // try all interval + this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; + this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; + + // sync progress level when zmq subscribes to events + this.zmqSubscribeProgress = options.zmqSubscribeProgress || Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS; +}; + Bitcoin.prototype._initCaches = function() { // caches valid until there is a new block this.utxosCache = LRU(50000); @@ -498,8 +506,8 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { if (err) { return callback(self._wrapRPCError(err)); } - var percentSynced = response.result.verificationprogress * 100; - if (Math.round(percentSynced) >= 99) { + var progress = response.result.verificationprogress; + if (progress >= self.zmqSubscribeProgress) { // subscribe to events for further updates self._subscribeZmqEvents(node); clearInterval(interval); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 7f030668..1e642461 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -810,8 +810,8 @@ describe('Bitcoin Service', function() { bitcoind._checkSyncedAndSubscribeZmqEvents(node); setTimeout(function() { log.error.callCount.should.equal(2); - blockEvents.should.equal(10); - bitcoind._updateTip.callCount.should.equal(10); + blockEvents.should.equal(11); + bitcoind._updateTip.callCount.should.equal(11); bitcoind._subscribeZmqEvents.callCount.should.equal(1); done(); }, 200); From b0290899ce39170ea7ce1c2fb03a93b9e6820efa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Apr 2016 16:19:33 -0400 Subject: [PATCH 158/299] bitcoind: handle empty input from pid file --- lib/services/bitcoind.js | 4 ++++ test/services/bitcoind.unit.js | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4d24d1d7..b86174ef 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -643,6 +643,10 @@ Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { return callback(err); } pid = parseInt(pid); + if (!Number.isFinite(pid)) { + // pid doesn't exist we can continue + return callback(null); + } try { log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); self._process.kill(pid, 'SIGINT'); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 1e642461..bbc88a8b 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1059,6 +1059,44 @@ describe('Bitcoin Service', function() { done(); }); }); + it('it will attempt to kill process with NaN', function(done) { + var readFile = sandbox.stub(); + readFile.onCall(0).callsArgWith(2, null, ' '); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFile: readFile + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind.spawnStopTime = 1; + bitcoind._process = {}; + bitcoind._process.kill = sinon.stub(); + bitcoind._stopSpawnedBitcoin(function(err) { + if (err) { + return done(err); + } + done(); + }); + }); + it('it will attempt to kill process without pid', function(done) { + var readFile = sandbox.stub(); + readFile.onCall(0).callsArgWith(2, null, ''); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFile: readFile + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind.spawnStopTime = 1; + bitcoind._process = {}; + bitcoind._process.kill = sinon.stub(); + bitcoind._stopSpawnedBitcoin(function(err) { + if (err) { + return done(err); + } + done(); + }); + }); }); describe('#_spawnChildProcess', function() { From c9154d4e0e74a4c874d3a6735493e3cf3e48fcbb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 14:23:53 -0400 Subject: [PATCH 159/299] docs: bump disk prereq to 200GB --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 388a9383..7820c5ff 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and ## Prerequisites - Node.js v0.12 or v4.2 -- ~150GB of disk storage +- ~200GB of disk storage - ~4GB of RAM ## Configuration From abfb07f5f820310091d1211fe0070ef2a4175251 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 14:27:13 -0400 Subject: [PATCH 160/299] build: update bitcoind-rpc commit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 671a30e5..2e46b93c 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "braydonf/bitcoind-rpc#4850733b9806bc5e8e1508fa90f3c45782e6ee80", + "bitcoind-rpc": "braydonf/bitcoind-rpc#381490bae5e3a8cefc0528b40666cd6980183499", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", From 7be7a7dce53d5b7e49d8ed47acfc90039ca8fd1d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 14:28:17 -0400 Subject: [PATCH 161/299] scaffold: update error message to be more accurate --- lib/scaffold/start.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index a01c0d16..f2ee95a8 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -85,7 +85,7 @@ function checkService(service) { !service.module.prototype.start || !service.module.prototype.stop) { throw new Error( - 'Could not load service "' + service.name + '" as it does not support necessary methods.' + 'Could not load service "' + service.name + '" as it does not support necessary methods and properties.' ); } } From 36f337afb31629ea035f68d906f227ecb268c61e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 14:30:07 -0400 Subject: [PATCH 162/299] web: update jsdoc with enableSocketRPC option --- lib/services/web.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/services/web.js b/lib/services/web.js index 5f809806..1e292676 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -27,6 +27,7 @@ var log = index.log; * @param {Object} options.httpsOptions - Options passed into https.createServer, defaults to node settings. * @param {String} options.httpsOptions.key - Path to key file * @param {String} options.httpsOptions.cert - Path to cert file + * @param {Boolean} options.enableSocketRPC - Option to enable/disable websocket RPC handling * @param {Number} options.port - The port for the service, defaults to node settings. */ var WebService = function(options) { From 27112fc1d754eb944b9b2a6965be5ebfcae9920f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 16:37:28 -0400 Subject: [PATCH 163/299] docs: make note about libzmq-dev --- docs/development.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/development.md b/docs/development.md index 8dfdc8b0..39683df6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -35,6 +35,8 @@ For Ubuntu: sudo apt-get install libzmq3-dev sudo apt-get install build-essential ``` +**Note**: Make sure that libzmq-dev is not installed, it should be removed when installing libzmq3-dev. + For Mac OS X: ```bash From d9d50c1f0c8ab9860f9df4b392a22aa0ec0913c0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 16:41:06 -0400 Subject: [PATCH 164/299] docs: update prereqs in readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7820c5ff..28063355 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and ## Prerequisites -- Node.js v0.12 or v4.2 +- GNU/Linux x86_32/x86_64, or OSX 64bit *(for bitcoind distributed binaries)* +- Node.js v0.10, v0.12 or v4 +- ZeroMQ *(libzmq3-dev for Ubuntu/Debian or zeromq on OSX)* - ~200GB of disk storage - ~4GB of RAM From e24a9c96aefef7685ada77ccee6f9997393757b3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 9 May 2016 16:41:54 -0400 Subject: [PATCH 165/299] build: update bitcoind links to bitpay/bitcoin bitcore-rc1 release --- scripts/download | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/download b/scripts/download index 93a04775..0e175eca 100755 --- a/scripts/download +++ b/scripts/download @@ -6,8 +6,8 @@ root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" -url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12.0-bitcore-beta6" +url="https://github.com/bitpay/bitcoin/releases/download" +tag="v0.12-bitcore-rc1" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From 0387c1a6e41a21d4cc9774846ddd415443c73de9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 11:47:31 -0400 Subject: [PATCH 166/299] test: increase timeout for cluster test and decrease keypool resolves issues when the keypool takes time to fill --- regtest/cluster.js | 2 +- regtest/data/node1/bitcoin.conf | 1 + regtest/data/node2/bitcoin.conf | 1 + regtest/data/node3/bitcoin.conf | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/regtest/cluster.js b/regtest/cluster.js index 51a6e573..ddeb7429 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -52,7 +52,7 @@ describe('Bitcoin Cluster', function() { before(function(done) { log.info('Starting 3 bitcoind daemons'); - this.timeout(20000); + this.timeout(60000); async.each(nodesConf, function(nodeConf, next) { var opts = [ '--regtest', diff --git a/regtest/data/node1/bitcoin.conf b/regtest/data/node1/bitcoin.conf index 19bbf70a..54ed2aec 100644 --- a/regtest/data/node1/bitcoin.conf +++ b/regtest/data/node1/bitcoin.conf @@ -13,3 +13,4 @@ zmqpubhashblock=tcp://127.0.0.1:30611 rpcallowip=127.0.0.1 rpcuser=bitcoin rpcpassword=local321 +keypool=3 diff --git a/regtest/data/node2/bitcoin.conf b/regtest/data/node2/bitcoin.conf index 9e08fe9a..bcd09fe6 100644 --- a/regtest/data/node2/bitcoin.conf +++ b/regtest/data/node2/bitcoin.conf @@ -13,3 +13,4 @@ zmqpubhashblock=tcp://127.0.0.1:30622 rpcallowip=127.0.0.1 rpcuser=bitcoin rpcpassword=local321 +keypool=3 diff --git a/regtest/data/node3/bitcoin.conf b/regtest/data/node3/bitcoin.conf index 954b0892..8be13ef3 100644 --- a/regtest/data/node3/bitcoin.conf +++ b/regtest/data/node3/bitcoin.conf @@ -13,3 +13,4 @@ zmqpubhashblock=tcp://127.0.0.1:30633 rpcallowip=127.0.0.1 rpcuser=bitcoin rpcpassword=local321 +keypool=3 From 75c43559d463f0bce4bee3c6f214d45f05d0f936 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 13:48:56 -0400 Subject: [PATCH 167/299] bitcoind: paginate txids in address summary so that one request doesn't yield a 80MB response --- lib/services/bitcoind.js | 49 ++++++++++++++++++++++-------- test/services/bitcoind.unit.js | 54 +++++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 20 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b86174ef..54d2cee4 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -62,6 +62,7 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; +Bitcoin.DEFAULT_MAX_TXIDS = 1000; Bitcoin.DEFAULT_MAX_HISTORY = 10; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS = 0.9999; @@ -89,6 +90,7 @@ Bitcoin.DEFAULT_CONFIG_SETTINGS = { Bitcoin.prototype._initDefaults = function(options) { // limits + this.maxTxids = options.maxTxids || Bitcoin.DEFAULT_MAX_TXIDS; this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; @@ -1223,10 +1225,6 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, fromArg, toArg) { var to = parseInt(toArg); if (from >= 0 && to >= 0) { $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); - $.checkState( - (to - from) <= this.maxTransactionHistory, - '"from" (' + from + ') and "to" (' + to + ') range should be less than or equal to ' + this.maxTransactionHistory - ); txids = fullTxids.slice(from, to); } else { txids = fullTxids; @@ -1250,6 +1248,13 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var addressStrings = this._getAddressStrings(addresses); + if ((options.to - options.from) > self.maxTransactionHistory) { + return callback(new Error( + '"from" (' + options.from + ') and "to" (' + options.to + ') range should be less than or equal to ' + + self.maxTransactionHistory + )); + } + self.getAddressTxids(addresses, options, function(err, txids) { if (err) { return callback(err); @@ -1298,6 +1303,33 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var addresses = self._normalizeAddressArg(addressArg); var cacheKey = addresses.join(''); + function finishWithTxids() { + if (!options.noTxList) { + var allTxids = mempoolTxids.reverse().concat(summaryTxids); + var fromArg = parseInt(options.from || 0); + var toArg = parseInt(options.to || self.maxTxids); + + if ((toArg - fromArg) > self.maxTxids) { + return callback(new Error( + '"from" (' + fromArg + ') and "to" (' + toArg + ') range should be less than or equal to ' + + self.maxTxids + )); + } + var paginatedTxids; + try { + paginatedTxids = self._paginateTxids(allTxids, fromArg, toArg); + } catch(e) { + return callback(e); + } + + var allSummary = _.clone(summary); + allSummary.txids = paginatedTxids; + callback(null, allSummary); + } else { + callback(null, summary); + } + } + function querySummary() { async.parallel([ function getTxList(done) { @@ -1340,14 +1372,7 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { return callback(err); } self.summaryCache.set(cacheKey, summary); - if (!options.noTxList) { - var allTxids = mempoolTxids.reverse().concat(summaryTxids); - var allSummary = _.clone(summary); - allSummary.txids = allTxids; - callback(null, allSummary); - } else { - callback(null, summary); - } + finishWithTxids(); }); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index bbc88a8b..2f8875e7 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2158,13 +2158,6 @@ describe('Bitcoin Service', function() { var paginated = bitcoind._paginateTxids(txids, 3, 13); paginated.should.deep.equal([3, 4, 5, 6, 7, 8, 9, 10]); }); - it('slice txids based on "from" and "to" (3 to 30)', function() { - var bitcoind = new BitcoinService(baseConfig); - var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - (function() { - bitcoind._paginateTxids(txids, 3, 30); - }).should.throw(Error); - }); it('slice txids based on "from" and "to" (0 to 3)', function() { var bitcoind = new BitcoinService(baseConfig); var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; @@ -2194,6 +2187,14 @@ describe('Bitcoin Service', function() { describe('#getAddressHistory', function() { var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + it('will give error with "from" and "to" range that exceeds max size', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getAddressHistory(address, {from: 0, to: 30}, function(err) { + should.exist(err); + err.message.match(/^\"from/); + done(); + }); + }); it('will give an error if length of addresses is too long', function(done) { var addresses = []; for (var i = 0; i < 101; i++) { @@ -2241,7 +2242,6 @@ describe('Bitcoin Service', function() { var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; var memtxid1 = 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92'; var memtxid2 = 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce'; - it('will handle error from getAddressTxids', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.nodes.push({ @@ -2326,6 +2326,7 @@ describe('Bitcoin Service', function() { }) } }); + sinon.spy(bitcoind, '_paginateTxids'); bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { received: 30 * 1e8, @@ -2334,6 +2335,9 @@ describe('Bitcoin Service', function() { var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; var options = {}; bitcoind.getAddressSummary(address, options, function(err, summary) { + bitcoind._paginateTxids.callCount.should.equal(1); + bitcoind._paginateTxids.args[0][1].should.equal(0); + bitcoind._paginateTxids.args[0][2].should.equal(1000); summary.appearances.should.equal(3); summary.totalReceived.should.equal(3000000000); summary.totalSpent.should.equal(1000000000); @@ -2350,6 +2354,40 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will give error with "from" and "to" range that exceeds max size', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: [ + { + txid: memtxid1, + satoshis: -1000000 + }, + { + txid: memtxid2, + satoshis: 99999 + } + ] + }) + } + }); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { + received: 30 * 1e8, + balance: 20 * 1e8 + }); + var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; + var options = { + from: 0, + to: 1001 + }; + bitcoind.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.match(/^\"from/); + done(); + }); + }); it('will get from cache with noTxList', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.nodes.push({ From 85a0c16eef58380ecb3ed2d1a00f3b23e0966604 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 18:17:53 -0400 Subject: [PATCH 168/299] test: fixes for bitcoind regtest --- regtest/bitcoind.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 9d729845..fecdd1fc 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -201,6 +201,7 @@ describe('Bitcoind Functionality', function() { it('will get error with number greater than tip', function(done) { bitcoind.getBlock(1000000000, function(err, response) { should.exist(err); + err.code.should.equal(-8); done(); }); }); @@ -242,6 +243,7 @@ describe('Bitcoind Functionality', function() { if (err) { throw err; } + response.should.be.instanceOf(Buffer); assert(response.toString('hex') === txhex, 'incorrect tx data result'); done(); }); @@ -423,7 +425,7 @@ describe('Bitcoind Functionality', function() { }); describe('get transaction with block info', function() { - it('should include tx buffer, height and timestamp', function(done) { + it('should include tx with height and timestamp', function(done) { bitcoind.getTransactionWithBlockInfo(utxos[0].txid, function(err, tx) { if (err) { return done(err); From 26c87ea32aa30b16c665a6754d546e5cbca3ab8c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 18:19:58 -0400 Subject: [PATCH 169/299] test: check height from tip event in cluster regtest --- regtest/cluster.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/regtest/cluster.js b/regtest/cluster.js index ddeb7429..c8ab8754 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -156,7 +156,8 @@ describe('Bitcoin Cluster', function() { it('step 2: receive block events', function(done) { this.timeout(10000); - node.services.bitcoind.once('tip', function() { + node.services.bitcoind.once('tip', function(height) { + height.should.equal(1); done(); }); node.generateBlock(1, function(err, hashes) { From f6bbe542932903102774dd28d2881f7b2c1a983c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 18:33:37 -0400 Subject: [PATCH 170/299] test: bitcoind chainwork test modified comparison to show how the values differ --- regtest/bitcoind.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index fecdd1fc..b8cf9bc5 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -270,7 +270,7 @@ describe('Bitcoind Functionality', function() { should.exist(blockIndex); should.exist(blockIndex.chainwork); var work = new BN(blockIndex.chainwork, 'hex'); - work.cmp(expectedWork).should.equal(0); + work.toString(16).should.equal(expectedWork.toString(16)); expectedWork = expectedWork.add(new BN(2)); should.exist(blockIndex.previousblockhash); blockIndex.hash.should.equal(blockHashes[i]); @@ -306,7 +306,7 @@ describe('Bitcoind Functionality', function() { should.exist(header); should.exist(header.chainwork); var work = new BN(header.chainwork, 'hex'); - work.cmp(expectedWork).should.equal(0); + work.toString(16).should.equal(expectedWork.toString(16)); expectedWork = expectedWork.add(new BN(2)); should.exist(header.previousblockhash); header.hash.should.equal(blockHashes[i - 1]); From 8b0d16d5a3d1b999ab07a1c93e4ad18d7acb4a22 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 18:54:02 -0400 Subject: [PATCH 171/299] test: check callcount for retry in bitcoind spawn child method --- test/services/bitcoind.unit.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 2f8875e7..f2db82a1 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1185,6 +1185,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn.config.zmqpubrawtx = 'tcp://127.0.0.1:30001'; bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, new Error('test')); bitcoind._spawnChildProcess(function(err) { + bitcoind._loadTipFromNode.callCount.should.equal(60); err.should.be.instanceof(Error); done(); }); From 791047c10dc519df42ff577cfbdd33bc04972c0f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 19:30:36 -0400 Subject: [PATCH 172/299] bitcoind: bump max tx history default to 50 --- lib/services/bitcoind.js | 2 +- test/services/bitcoind.unit.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 54d2cee4..7b4be309 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -63,7 +63,7 @@ util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; Bitcoin.DEFAULT_MAX_TXIDS = 1000; -Bitcoin.DEFAULT_MAX_HISTORY = 10; +Bitcoin.DEFAULT_MAX_HISTORY = 50; Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS = 0.9999; Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index f2db82a1..c8a159fc 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2190,7 +2190,7 @@ describe('Bitcoin Service', function() { var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; it('will give error with "from" and "to" range that exceeds max size', function(done) { var bitcoind = new BitcoinService(baseConfig); - bitcoind.getAddressHistory(address, {from: 0, to: 30}, function(err) { + bitcoind.getAddressHistory(address, {from: 0, to: 51}, function(err) { should.exist(err); err.message.match(/^\"from/); done(); From cceb4186d46e574589272d5334591d4d05625618 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 19:35:37 -0400 Subject: [PATCH 173/299] test: bump timeout in bitcoind after/before --- regtest/bitcoind.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index b8cf9bc5..b5caa24c 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -29,7 +29,7 @@ var destKey = bitcore.PrivateKey(); describe('Bitcoind Functionality', function() { before(function(done) { - this.timeout(30000); + this.timeout(60000); // Add the regtest network bitcore.Networks.enableRegtest(); @@ -130,7 +130,7 @@ describe('Bitcoind Functionality', function() { }); after(function(done) { - this.timeout(20000); + this.timeout(60000); bitcoind.node.stopping = true; bitcoind.stop(function(err, result) { done(); From 4757edc57056115773fdf8da329a1972c519f80d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 19:40:29 -0400 Subject: [PATCH 174/299] test: add missing property checks --- test/services/bitcoind.unit.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index c8a159fc..e2a6e9fc 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3065,7 +3065,9 @@ describe('Bitcoin Service', function() { }); var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; bitcoind.getTransactionWithBlockInfo(txid, function(err, tx) { - // TODO verify additional info + should.equal(tx.__blockHash, '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6'); + should.equal(tx.__height, 530482); + should.equal(tx.__timestamp, 1439559434000); should.exist(tx); done(); }); From d399e9acea86a65e7ed50ec26fff77895cee76c2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 11:16:04 -0400 Subject: [PATCH 175/299] bitcoind: camelCase getInfo results for consistency with other bitcoind api responses --- lib/services/bitcoind.js | 12 ++++++------ regtest/bitcoind.js | 4 ++-- test/services/bitcoind.unit.js | 24 ++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 7b4be309..0070127b 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1724,14 +1724,14 @@ Bitcoin.prototype.getSpentInfo = function(options, callback) { * This will return information about the database in the format: * { * version: 110000, - * protocolversion: 70002, + * protocolVersion: 70002, * blocks: 151, - * timeoffset: 0, + * timeOffset: 0, * connections: 0, * difficulty: 4.6565423739069247e-10, * testnet: false, * network: 'testnet' - * relayfee: 1000, + * relayFee: 1000, * errors: '' * } * @param {Function} callback @@ -1745,14 +1745,14 @@ Bitcoin.prototype.getInfo = function(callback) { var result = response.result; var info = { version: result.version, - protocolversion: result.protocolversion, + protocolVersion: result.protocolversion, blocks: result.blocks, - timeoffset: result.timeoffset, + timeOffset: result.timeoffset, connections: result.connections, proxy: result.proxy, difficulty: result.difficulty, testnet: result.testnet, - relayfee: result.relayfee, + relayFee: result.relayfee, errors: result.errors, network: self.node.getNetworkName() }; diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index b5caa24c..754823ed 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -449,11 +449,11 @@ describe('Bitcoind Functionality', function() { should.exist(info); should.exist(info.version); should.exist(info.blocks); - should.exist(info.timeoffset); + should.exist(info.timeOffset); should.exist(info.connections); should.exist(info.difficulty); should.exist(info.testnet); - should.exist(info.relayfee); + should.exist(info.relayFee); should.exist(info.errors); done(); }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index e2a6e9fc..f2ded2b9 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3184,7 +3184,18 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); bitcoind.node.getNetworkName = sinon.stub().returns('testnet'); var getInfo = sinon.stub().callsArgWith(0, null, { - result: {} + result: { + version: 1, + protocolversion: 1, + blocks: 1, + timeoffset: 1, + connections: 1, + proxy: '', + difficulty: 1, + testnet: true, + relayfee: 10, + errors: '' + } }); bitcoind.nodes.push({ client: { @@ -3196,7 +3207,16 @@ describe('Bitcoin Service', function() { return done(err); } should.exist(info); - should.exist(info.network); + should.equal(info.version, 1); + should.equal(info.protocolVersion, 1); + should.equal(info.blocks, 1); + should.equal(info.timeOffset, 1); + should.equal(info.connections, 1); + should.equal(info.proxy, ''); + should.equal(info.difficulty, 1); + should.equal(info.testnet, true); + should.equal(info.relayFee, 10); + should.equal(info.errors, ''); info.network.should.equal('testnet'); done(); }); From b597a05cb4e51508ebb3579414c7e859defb8d96 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 11:39:14 -0400 Subject: [PATCH 176/299] bitcoind: camelCase result from getBlockHeader for consistency with other methods --- lib/services/bitcoind.js | 19 +++++++++- regtest/bitcoind.js | 18 ++++----- test/services/bitcoind.unit.js | 68 +++++++++++++++++++++++++++++++--- 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 0070127b..1d3934c9 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1523,8 +1523,23 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { if (err) { return done(self._wrapRPCError(err)); } - // TODO format response prevHash instead of previousblockhash, etc. - done(null, response.result); + var result = response.result; + var header = { + hash: result.hash, + version: result.version, + confirmations: result.confirmations, + height: result.height, + chainWork: result.chainwork, + prevHash: result.previousblockhash, + nextHash: result.nextblockhash, + merkleRoot: result.merkleroot, + time: result.time, + medianTime: result.mediantime, + nonce: result.nonce, + bits: result.bits, + difficulty: result.difficulty + }; + done(null, header); }); }, callback); } diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 754823ed..aa413660 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -268,13 +268,13 @@ describe('Bitcoind Functionality', function() { return done(err); } should.exist(blockIndex); - should.exist(blockIndex.chainwork); - var work = new BN(blockIndex.chainwork, 'hex'); + should.exist(blockIndex.chainWork); + var work = new BN(blockIndex.chainWork, 'hex'); work.toString(16).should.equal(expectedWork.toString(16)); expectedWork = expectedWork.add(new BN(2)); - should.exist(blockIndex.previousblockhash); + should.exist(blockIndex.prevHash); blockIndex.hash.should.equal(blockHashes[i]); - blockIndex.previousblockhash.should.equal(blockHashes[i - 1]); + blockIndex.prevHash.should.equal(blockHashes[i - 1]); blockIndex.height.should.equal(i + 1); done(); }); @@ -286,7 +286,7 @@ describe('Bitcoind Functionality', function() { return done(err); } should.exist(header); - should.equal(header.previousblockhash, undefined); + should.equal(header.prevHash, undefined); done(); }); }); @@ -304,13 +304,13 @@ describe('Bitcoind Functionality', function() { it('generate block ' + i, function() { bitcoind.getBlockHeader(i, function(err, header) { should.exist(header); - should.exist(header.chainwork); - var work = new BN(header.chainwork, 'hex'); + should.exist(header.chainWork); + var work = new BN(header.chainWork, 'hex'); work.toString(16).should.equal(expectedWork.toString(16)); expectedWork = expectedWork.add(new BN(2)); - should.exist(header.previousblockhash); + should.exist(header.prevHash); header.hash.should.equal(blockHashes[i - 1]); - header.previousblockhash.should.equal(blockHashes[i - 2]); + header.prevHash.should.equal(blockHashes[i - 2]); header.height.should.equal(i); }); }); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index f2ded2b9..ce149c1a 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2746,9 +2746,37 @@ describe('Bitcoin Service', function() { }); it('will give result from client getblockheader (from height)', function() { var bitcoind = new BitcoinService(baseConfig); - var result = {}; + var result = { + hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', + version: 536870912, + confirmations: 5, + height: 828781, + chainWork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + prevHash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + nextHash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8', + merkleRoot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + time: 1462979126, + medianTime: 1462976771, + nonce: 2981820714, + bits: '1a13ca10', + difficulty: 847779.0710240941 + }; var getBlockHeader = sinon.stub().callsArgWith(1, null, { - result: result + result: { + hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', + version: 536870912, + confirmations: 5, + height: 828781, + chainwork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + previousblockhash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + nextblockhash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8', + merkleroot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + time: 1462979126, + mediantime: 1462976771, + nonce: 2981820714, + bits: '1a13ca10', + difficulty: 847779.0710240941 + } }); var getBlockHash = sinon.stub().callsArgWith(1, null, { result: blockhash @@ -2762,14 +2790,42 @@ describe('Bitcoin Service', function() { bitcoind.getBlockHeader(0, function(err, blockHeader) { should.not.exist(err); getBlockHeader.args[0][0].should.equal(blockhash); - blockHeader.should.equal(result); + blockHeader.should.deep.equal(result); }); }); it('will give result from client getblockheader (from hash)', function() { var bitcoind = new BitcoinService(baseConfig); - var result = {}; + var result = { + hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', + version: 536870912, + confirmations: 5, + height: 828781, + chainWork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + prevHash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + nextHash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8', + merkleRoot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + time: 1462979126, + medianTime: 1462976771, + nonce: 2981820714, + bits: '1a13ca10', + difficulty: 847779.0710240941 + }; var getBlockHeader = sinon.stub().callsArgWith(1, null, { - result: result + result: { + hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', + version: 536870912, + confirmations: 5, + height: 828781, + chainwork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + previousblockhash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + nextblockhash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8', + merkleroot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + time: 1462979126, + mediantime: 1462976771, + nonce: 2981820714, + bits: '1a13ca10', + difficulty: 847779.0710240941 + } }); var getBlockHash = sinon.stub(); bitcoind.nodes.push({ @@ -2781,7 +2837,7 @@ describe('Bitcoin Service', function() { bitcoind.getBlockHeader(blockhash, function(err, blockHeader) { should.not.exist(err); getBlockHash.callCount.should.equal(0); - blockHeader.should.equal(result); + blockHeader.should.deep.equal(result); }); }); }); From ae91ff242022e300159781d8ac4dadd92fa04891 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 12:15:33 -0400 Subject: [PATCH 177/299] bitcoind: update jsdocs for getBlockHeader --- lib/services/bitcoind.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 1d3934c9..971c6da1 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1505,11 +1505,19 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { /** * Will return the block index information, the output will have the format: * { - * prevHash: '000000004956cc2edd1a8caa05eacfa3c69f4c490bfc9ace820257834115ab35', - * nextHash: '0000000000629d100db387f37d0f37c51118f250fb0946310a8c37316cbc4028' - * hash: ' 00000000009e2958c15ff9290d571bf9459e93b19765c6801ddeccadbb160a1e', - * chainWork: '0000000000000000000000000000000000000000000000000000000000000016', - * height: 10 + * hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', + * confirmations: 5, + * height: 828781, + * chainWork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + * prevHash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + * nextHash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8' + * version: 536870912, + * merkleRoot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + * time: 1462979126, + * medianTime: 1462976771, + * nonce: 2981820714, + * bits: '1a13ca10', + * difficulty: 847779.0710240941, * } * @param {String|Number} block - A block hash or block height * @param {Function} callback From 98bfd358d3403c0f7cdb4cac31a184c3b0a6389e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 12:51:24 -0400 Subject: [PATCH 178/299] build: update bitcoind-rpc with work limit exceeded handling --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2e46b93c..81a2bec8 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "braydonf/bitcoind-rpc#381490bae5e3a8cefc0528b40666cd6980183499", + "bitcoind-rpc": "braydonf/bitcoind-rpc#95836dbc3a4498d7eee8a5b35dceee059ca67fda", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", From 17e7a7eedb2945b717884ceb5dc9655236cd12df Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 13:13:33 -0400 Subject: [PATCH 179/299] build: bitcoind-rpc with explicit work limit exceeded handling fixes an issues were 500 error codes are used for both block not found as well as work limit exceeded errors --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 81a2bec8..bbd99939 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "braydonf/bitcoind-rpc#95836dbc3a4498d7eee8a5b35dceee059ca67fda", + "bitcoind-rpc": "braydonf/bitcoind-rpc#594d9aa0bee54ab247578785bd2acd16a8c012e6", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", From 950a9d521cef93ac536c965923520c15229bd2e6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 13:33:21 -0400 Subject: [PATCH 180/299] docs: make note about sorting of blockhashes --- docs/services/bitcoind.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index 8c49624a..b28d2699 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -64,7 +64,7 @@ node.services.bitcoind. **Getting Latest Blocks** ```js -// gives the block hashes within a range of timestamps +// gives the block hashes sorted from low to high within a range of timestamps var high = 1460393372; // Mon Apr 11 2016 12:49:25 GMT-0400 (EDT) var low = 1460306965; // Mon Apr 10 2016 12:49:25 GMT-0400 (EDT) node.services.bitcoind.getBlockHashesByTimestamp(high, low, function(err, blockHashes) { From 8bddf4f0d6dd559d5bd10f65705375a77bc0d50a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 12 May 2016 18:07:39 -0400 Subject: [PATCH 181/299] bitcoind: add getDetailedTransaction method Adds a new method getDetailedTransaction with a standard JavaScript object with block information, address, amounts and fees. And removes the getTransactionWithBlockInfo method since this new method is equivalent, and will serialize over an API correctly. Also includes a new method getBlockOverview to get the txids for a block, that can be combined with getDetailedTransaction for viewing block transactions with additional information. --- lib/services/bitcoind.js | 298 ++++++++++++++++++++++++--------- regtest/bitcoind.js | 36 +++- regtest/node.js | 55 +++--- scripts/download | 4 +- test/services/bitcoind.unit.js | 263 ++++++++++++++--------------- 5 files changed, 410 insertions(+), 246 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 971c6da1..89b9cf58 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -113,7 +113,8 @@ Bitcoin.prototype._initCaches = function() { this.txidsCache = LRU(50000); this.balanceCache = LRU(50000); this.summaryCache = LRU(50000); - this.transactionInfoCache = LRU(100000); + this.blockOverviewCache = LRU(144); + this.transactionDetailedCache = LRU(100000); // caches valid indefinitely this.transactionCache = LRU(100000); @@ -150,6 +151,7 @@ Bitcoin.prototype.getAPIMethods = function() { ['getBlock', this, this.getBlock, 1], ['getRawBlock', this, this.getRawBlock, 1], ['getBlockHeader', this, this.getBlockHeader, 1], + ['getBlockOverview', this, this.getBlockOverview, 1], ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], ['getBestBlockHash', this, this.getBestBlockHash, 0], ['getSpentInfo', this, this.getSpentInfo, 1], @@ -157,8 +159,8 @@ Bitcoin.prototype.getAPIMethods = function() { ['syncPercentage', this, this.syncPercentage, 0], ['isSynced', this, this.isSynced, 0], ['getRawTransaction', this, this.getRawTransaction, 1], - ['getTransaction', this, this.getTransaction, 2], - ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], + ['getTransaction', this, this.getTransaction, 1], + ['getDetailedTransaction', this, this.getDetailedTransaction, 1], ['sendTransaction', this, this.sendTransaction, 1], ['estimateFee', this, this.estimateFee, 1], ['getAddressTxids', this, this.getAddressTxids, 2], @@ -320,11 +322,12 @@ Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { }; Bitcoin.prototype._resetCaches = function() { - this.transactionInfoCache.reset(); + this.transactionDetailedCache.reset(); this.utxosCache.reset(); this.txidsCache.reset(); this.balanceCache.reset(); this.summaryCache.reset(); + this.blockOverviewCache.reset(); }; Bitcoin.prototype._tryAll = function(func, callback) { @@ -1096,8 +1099,8 @@ Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { Bitcoin.prototype._getConfirmationsDetail = function(transaction) { $.checkState(this.height > 0, 'current height is unknown'); var confirmations = 0; - if (transaction.__height >= 0) { - confirmations = this.height - transaction.__height + 1; + if (transaction.height >= 0) { + confirmations = this.height - transaction.height + 1; } if (confirmations < 0) { log.warn('Negative confirmations calculated for transaction:', transaction.hash); @@ -1106,44 +1109,38 @@ Bitcoin.prototype._getConfirmationsDetail = function(transaction) { }; Bitcoin.prototype._getAddressDetailsForInput = function(input, inputIndex, result, addressStrings) { - if (!input.script) { + if (!input.address) { return; } - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { - var inputAddressString = inputAddress.toString(); - if (addressStrings.indexOf(inputAddressString) >= 0) { - if (!result.addresses[inputAddressString]) { - result.addresses[inputAddressString] = { - inputIndexes: [inputIndex], - outputIndexes: [] - }; - } else { - result.addresses[inputAddressString].inputIndexes.push(inputIndex); - } - result.satoshis -= input.output.satoshis; + var address = input.address; + if (addressStrings.indexOf(address) >= 0) { + if (!result.addresses[address]) { + result.addresses[address] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[address].inputIndexes.push(inputIndex); } + result.satoshis -= input.satoshis; } }; Bitcoin.prototype._getAddressDetailsForOutput = function(output, outputIndex, result, addressStrings) { - if (!output.script) { + if (!output.address) { return; } - var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress) { - var outputAddressString = outputAddress.toString(); - if (addressStrings.indexOf(outputAddressString) >= 0) { - if (!result.addresses[outputAddressString]) { - result.addresses[outputAddressString] = { - inputIndexes: [], - outputIndexes: [outputIndex] - }; - } else { - result.addresses[outputAddressString].outputIndexes.push(outputIndex); - } - result.satoshis += output.satoshis; + var address = output.address; + if (addressStrings.indexOf(address) >= 0) { + if (!result.addresses[address]) { + result.addresses[address] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[address].outputIndexes.push(outputIndex); } + result.satoshis += output.satoshis; } }; @@ -1163,6 +1160,8 @@ Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addre this._getAddressDetailsForOutput(output, outputIndex, result, addressStrings); } + $.checkState(Number.isFinite(result.satoshis)); + return result; }; @@ -1171,35 +1170,25 @@ Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addre * @param {Object} txid - A bitcoin transaction id * @param {Function} callback */ -Bitcoin.prototype._getDetailedTransaction = function(txid, options, next) { +Bitcoin.prototype._getAddressDetailedTransaction = function(txid, options, next) { var self = this; - self.getTransactionWithBlockInfo( + self.getDetailedTransaction( txid, function(err, transaction) { if (err) { return next(err); } - transaction.populateInputs(self, [], function(err) { - if (err) { - return next(err); - } + var addressDetails = self._getAddressDetailsForTransaction(transaction, options.addressStrings); - var addressDetails = self._getAddressDetailsForTransaction(transaction, options.addressStrings); - - var details = { - addresses: addressDetails.addresses, - satoshis: addressDetails.satoshis, - height: transaction.__height, - confirmations: self._getConfirmationsDetail(transaction), - timestamp: transaction.__timestamp, - // TODO bitcore-lib should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - tx: transaction - }; - next(null, details); - }); + var details = { + addresses: addressDetails.addresses, + satoshis: addressDetails.satoshis, + confirmations: self._getConfirmationsDetail(transaction), + tx: transaction + }; + next(null, details); } ); }; @@ -1270,7 +1259,7 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { async.mapSeries( txids, function(txid, next) { - self._getDetailedTransaction(txid, { + self._getAddressDetailedTransaction(txid, { queryMempool: queryMempool, addressStrings: addressStrings }, next); @@ -1437,6 +1426,69 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { } }; +/** + * Similar to getBlockHeader but will include a list of txids + * @param {String|Number} block - A block hash or block height number + * @param {Function} callback + */ +Bitcoin.prototype.getBlockOverview = function(blockArg, callback) { + var self = this; + + function queryBlock(blockhash) { + var cachedBlock = self.blockOverviewCache.get(blockhash); + if (cachedBlock) { + return setImmediate(function() { + callback(null, cachedBlock); + }); + } else { + self._tryAll(function(done) { + self.client.getBlock(blockhash, true, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var result = response.result; + var blockOverview = { + hash: result.hash, + version: result.version, + confirmations: result.confirmations, + height: result.height, + chainWork: result.chainwork, + prevHash: result.previousblockhash, + nextHash: result.nextblockhash, + merkleRoot: result.merkleroot, + time: result.time, + medianTime: result.mediantime, + nonce: result.nonce, + bits: result.bits, + difficulty: result.difficulty, + txids: result.tx + }; + self.blockOverviewCache.set(blockhash, blockOverview); + done(null, blockOverview); + }); + }, callback); + } + } + + if (_.isNumber(blockArg)) { + self._tryAll(function(done) { + self.client.getBlockHash(blockArg, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + done(null, response.result); + }); + }, function(err, blockhash) { + if (err) { + return callback(err); + } + queryBlock(blockhash); + }); + } else { + queryBlock(blockArg); + } +}; + /** * Will retrieve a block as a Bitcore object * @param {String|Number} block - A block hash or block height number @@ -1672,19 +1724,101 @@ Bitcoin.prototype.getTransaction = function(txid, callback) { }; /** - * Will get a transaction as Bitcore Transaction with additional fields: + * Will get a detailed view of a transaction including addresses, amounts and fees. + * + * Example result: * { - * __blockHash: '2725743288feae6bdaa976590af7cb12d7b535b5a242787de6d2789c73682ed1', - * __height: 48, - * __timestamp: 1442951110, // in seconds - * } - * @param {String} txid - The transaction hash + * blockHash: '000000000000000002cd0ba6e8fae058747d2344929ed857a18d3484156c9250', + * height: 411462, + * blockTimestamp: 1463070382, + * version: 1, + * hash: 'de184cc227f6d1dc0316c7484aa68b58186a18f89d853bb2428b02040c394479', + * locktime: 411451, + * coinbase: true, + * inputs: [ + * { + * prevTxId: '3d003413c13eec3fa8ea1fe8bbff6f40718c66facffe2544d7516c9e2900cac2', + * outputIndex: 0, + * sequence: 123456789, + * script: [hexString], + * scriptAsm: [asmString], + * satoshis: 771146 + * } + * ], + * outputs: [ + * { + * satoshis: 811146, + * script: '76a914d2955017f4e3d6510c57b427cf45ae29c372c99088ac', + * scriptAsm: 'OP_DUP OP_HASH160 d2955017f4e3d6510c57b427cf45ae29c372c990 OP_EQUALVERIFY OP_CHECKSIG', + * address: '1LCTmj15p7sSXv3jmrPfA6KGs6iuepBiiG', + * spentTxId: '4316b98e7504073acd19308b4b8c9f4eeb5e811455c54c0ebfe276c0b1eb6315', + * spentIndex: 1, + * spentHeight: 100 + * } + * ], + * inputSatoshis: 771146, + * outputSatoshis: 811146, + * feeSatoshis: 40000 + * }; + * + * @param {String} txid - The hex string of the transaction * @param {Function} callback */ -Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { - // TODO give response back as standard js object with bitcore tx +Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { var self = this; - var tx = self.transactionInfoCache.get(txid); + var tx = self.transactionDetailedCache.get(txid); + + function addInputsToTx(tx, result) { + tx.inputs = []; + tx.inputSatoshis = 0; + for(var inputIndex = 0; inputIndex < result.vin.length; inputIndex++) { + var input = result.vin[inputIndex]; + if (!tx.coinbase) { + tx.inputSatoshis += input.valueSat; + } + var script; + var scriptAsm; + if (input.scriptSig) { + script = input.scriptSig.hex; + scriptAsm = input.scriptSig.asm; + } else if (input.coinbase) { + script = input.coinbase; + scriptAsm = null; + } + tx.inputs.push({ + prevTxId: input.txid || null, + outputIndex: _.isUndefined(input.vout) ? null : input.vout, + script: script, + scriptAsm: scriptAsm || null, + sequence: input.sequence, + address: input.address || null, + satoshis: _.isUndefined(input.valueSat) ? null : input.valueSat + }); + } + } + + function addOutputsToTx(tx, result) { + tx.outputs = []; + tx.outputSatoshis = 0; + for(var outputIndex = 0; outputIndex < result.vout.length; outputIndex++) { + var out = result.vout[outputIndex]; + tx.outputSatoshis += out.valueSat; + var address = null; + if (out.scriptPubKey && out.scriptPubKey.addresses && out.scriptPubKey.addresses.length > 0) { + address = out.scriptPubKey.addresses[0]; + } + tx.outputs.push({ + satoshis: out.valueSat, + script: out.scriptPubKey.hex, + scriptAsm: out.scriptPubKey.asm, + spentTxId: out.spentTxId, + spentIndex: out.spentIndex, + spentHeight: out.spentHeight, + address: address + }); + } + } + if (tx) { return setImmediate(function() { callback(null, tx); @@ -1695,18 +1829,32 @@ Bitcoin.prototype.getTransactionWithBlockInfo = function(txid, callback) { if (err) { return done(self._wrapRPCError(err)); } - var tx = Transaction(); - tx.fromString(response.result.hex); - tx.__blockHash = response.result.blockhash; - tx.__height = response.result.height ? response.result.height : -1; - tx.__timestamp = response.result.time; - - for (var i = 0; i < response.result.vout.length; i++) { - tx.outputs[i].__spentTxId = response.result.vout[i].spentTxId; - tx.outputs[i].__spentIndex = response.result.vout[i].spentIndex; - tx.outputs[i].__spentHeight = response.result.vout[i].spentHeight; + var result = response.result; + var tx = { + hex: result.hex, + blockHash: result.blockhash, + height: result.height ? result.height : -1, + blockTimestamp: result.time, + version: result.version, + hash: txid, + locktime: result.locktime, + }; + + if (result.vin[0] && result.vin[0].coinbase) { + tx.coinbase = true; + } + + addInputsToTx(tx, result); + addOutputsToTx(tx, result); + + if (!tx.coinbase) { + tx.feeSatoshis = tx.inputSatoshis - tx.outputSatoshis; + } else { + tx.feeSatoshis = 0; } - self.transactionInfoCache.set(txid, tx); + + self.transactionDetailedCache.set(txid, tx); + done(null, tx); }); }, callback); diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index aa413660..13346416 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -424,16 +424,38 @@ describe('Bitcoind Functionality', function() { }); }); - describe('get transaction with block info', function() { - it('should include tx with height and timestamp', function(done) { - bitcoind.getTransactionWithBlockInfo(utxos[0].txid, function(err, tx) { + describe('get detailed transaction', function() { + it('should include details for coinbase tx', function(done) { + bitcoind.getDetailedTransaction(utxos[0].txid, function(err, tx) { if (err) { return done(err); } - should.exist(tx.__height); - tx.__height.should.be.a('number'); - should.exist(tx.__timestamp); - should.exist(tx.__blockHash); + should.exist(tx.height); + tx.height.should.be.a('number'); + should.exist(tx.blockTimestamp); + should.exist(tx.blockHash); + tx.coinbase.should.equal(true); + tx.version.should.equal(1); + tx.hex.should.be.a('string'); + tx.locktime.should.equal(0); + tx.feeSatoshis.should.equal(0); + tx.outputSatoshis.should.equal(50 * 1e8); + tx.inputSatoshis.should.equal(0); + tx.inputs.length.should.equal(1); + tx.outputs.length.should.equal(1); + should.equal(tx.inputs[0].prevTxId, null); + should.equal(tx.inputs[0].outputIndex, null); + tx.inputs[0].script.should.be.a('string'); + should.equal(tx.inputs[0].scriptAsm, null); + should.equal(tx.inputs[0].address, null); + should.equal(tx.inputs[0].satoshis, null); + tx.outputs[0].satoshis.should.equal(50 * 1e8); + tx.outputs[0].script.should.be.a('string'); + tx.outputs[0].scriptAsm.should.be.a('string'); + tx.outputs[0].spentTxId.should.be.a('string'); + tx.outputs[0].spentIndex.should.equal(0); + tx.outputs[0].spentHeight.should.be.a('number'); + tx.outputs[0].address.should.be.a('string'); done(); }); }); diff --git a/regtest/node.js b/regtest/node.js index 40de6de2..9593e4fd 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -205,9 +205,8 @@ describe('Node Functionality', function() { info.addresses[address].inputIndexes.should.deep.equal([]); info.satoshis.should.equal(10 * 1e8); info.confirmations.should.equal(3); - info.timestamp.should.be.a('number'); - info.fees.should.be.within(950, 4000); - info.tx.should.be.an.instanceof(Transaction); + info.tx.blockTimestamp.should.be.a('number'); + info.tx.feeSatoshis.should.be.within(950, 4000); done(); }); }); @@ -395,13 +394,13 @@ describe('Node Functionality', function() { results.totalCount.should.equal(4); var history = results.items; history.length.should.equal(4); - history[0].height.should.equal(159); + history[0].tx.height.should.equal(159); history[0].confirmations.should.equal(1); - history[1].height.should.equal(158); + history[1].tx.height.should.equal(158); should.exist(history[1].addresses[address4]); - history[2].height.should.equal(157); + history[2].tx.height.should.equal(157); should.exist(history[2].addresses[address3]); - history[3].height.should.equal(156); + history[3].tx.height.should.equal(156); should.exist(history[3].addresses[address2]); history[3].satoshis.should.equal(tx2Amount); history[3].tx.hash.should.equal(tx2Hash); @@ -429,9 +428,9 @@ describe('Node Functionality', function() { results.totalCount.should.equal(2); var history = results.items; history.length.should.equal(2); - history[0].height.should.equal(158); + history[0].tx.height.should.equal(158); history[0].confirmations.should.equal(2); - history[1].height.should.equal(157); + history[1].tx.height.should.equal(157); should.exist(history[1].addresses[address3]); done(); }); @@ -456,8 +455,8 @@ describe('Node Functionality', function() { results.totalCount.should.equal(2); var history = results.items; history.length.should.equal(2); - history[0].height.should.equal(157); - history[1].height.should.equal(156); + history[0].tx.height.should.equal(157); + history[1].tx.height.should.equal(156); done(); }); }); @@ -481,9 +480,9 @@ describe('Node Functionality', function() { results.totalCount.should.equal(4); var history = results.items; history.length.should.equal(3); - history[0].height.should.equal(159); + history[0].tx.height.should.equal(159); history[0].confirmations.should.equal(1); - history[1].height.should.equal(158); + history[1].tx.height.should.equal(158); should.exist(history[1].addresses[address4]); done(); }); @@ -501,18 +500,18 @@ describe('Node Functionality', function() { results.totalCount.should.equal(6); var history = results.items; history.length.should.equal(6); - history[0].height.should.equal(159); + history[0].tx.height.should.equal(159); history[0].addresses[address].inputIndexes.should.deep.equal([0, 1]); history[0].addresses[address].outputIndexes.should.deep.equal([2]); history[0].confirmations.should.equal(1); - history[1].height.should.equal(158); - history[2].height.should.equal(157); - history[3].height.should.equal(156); - history[4].height.should.equal(155); + history[1].tx.height.should.equal(158); + history[2].tx.height.should.equal(157); + history[3].tx.height.should.equal(156); + history[4].tx.height.should.equal(155); history[4].satoshis.should.equal(-10000); history[4].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); history[4].addresses[address].inputIndexes.should.deep.equal([0]); - history[5].height.should.equal(152); + history[5].tx.height.should.equal(152); history[5].satoshis.should.equal(10 * 1e8); done(); }); @@ -561,7 +560,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(159); + history[0].tx.height.should.equal(159); done(); }); }); @@ -576,7 +575,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(158); + history[0].tx.height.should.equal(158); done(); }); }); @@ -591,7 +590,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(157); + history[0].tx.height.should.equal(157); done(); }); }); @@ -606,7 +605,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(156); + history[0].tx.height.should.equal(156); done(); }); }); @@ -621,7 +620,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(155); + history[0].tx.height.should.equal(155); history[0].satoshis.should.equal(-10000); history[0].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); history[0].addresses[address].inputIndexes.should.deep.equal([0]); @@ -639,7 +638,7 @@ describe('Node Functionality', function() { } var history = results.items; history.length.should.equal(1); - history[0].height.should.equal(152); + history[0].tx.height.should.equal(152); history[0].satoshis.should.equal(10 * 1e8); done(); }); @@ -744,13 +743,13 @@ describe('Node Functionality', function() { it('will not show confirmation count for orphaned transaction', function(done) { // This test verifies that in the situation that the transaction is not in the mempool and // is included in an orphaned block transaction index that the confirmation count will be unconfirmed. - node.getTransactionWithBlockInfo(orphanedTransaction, function(err, data) { + node.getDetailedTransaction(orphanedTransaction, function(err, data) { if (err) { return done(err); } should.exist(data); - should.exist(data.__height); - data.__height.should.equal(-1); + should.exist(data.height); + data.height.should.equal(-1); done(); }); }); diff --git a/scripts/download b/scripts/download index 0e175eca..081888c9 100755 --- a/scripts/download +++ b/scripts/download @@ -6,8 +6,8 @@ root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" -url="https://github.com/bitpay/bitcoin/releases/download" -tag="v0.12-bitcore-rc1" +url="https://github.com/braydonf/bitcoin/releases/download" +tag="v0.12-bitcore-rc2-spent" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index ce149c1a..28d70494 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -5,6 +5,7 @@ var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); var crypto = require('crypto'); var bitcore = require('bitcore-lib'); +var _ = bitcore.deps._; var sinon = require('sinon'); var proxyquire = require('proxyquire'); var fs = require('fs'); @@ -51,7 +52,7 @@ describe('Bitcoin Service', function() { should.exist(bitcoind.txidsCache); should.exist(bitcoind.balanceCache); should.exist(bitcoind.summaryCache); - should.exist(bitcoind.transactionInfoCache); + should.exist(bitcoind.transactionDetailedCache); should.exist(bitcoind.transactionCache); should.exist(bitcoind.rawTransactionCache); @@ -90,7 +91,7 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var methods = bitcoind.getAPIMethods(); should.exist(methods); - methods.length.should.equal(20); + methods.length.should.equal(21); }); }); @@ -295,14 +296,14 @@ describe('Bitcoin Service', function() { var keys = []; for (var i = 0; i < 10; i++) { keys.push(crypto.randomBytes(32)); - bitcoind.transactionInfoCache.set(keys[i], {}); + bitcoind.transactionDetailedCache.set(keys[i], {}); bitcoind.utxosCache.set(keys[i], {}); bitcoind.txidsCache.set(keys[i], {}); bitcoind.balanceCache.set(keys[i], {}); bitcoind.summaryCache.set(keys[i], {}); } bitcoind._resetCaches(); - should.equal(bitcoind.transactionInfoCache.get(keys[0]), undefined); + should.equal(bitcoind.transactionDetailedCache.get(keys[0]), undefined); should.equal(bitcoind.utxosCache.get(keys[0]), undefined); should.equal(bitcoind.txidsCache.get(keys[0]), undefined); should.equal(bitcoind.balanceCache.get(keys[0]), undefined); @@ -1919,7 +1920,7 @@ describe('Bitcoin Service', function() { }); it('should get 1 confirmation', function() { var tx = new Transaction(txhex); - tx.__height = 10; + tx.height = 10; var bitcoind = new BitcoinService(baseConfig); bitcoind.height = 10; var confirmations = bitcoind._getConfirmationsDetail(tx); @@ -1929,7 +1930,7 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var tx = new Transaction(txhex); bitcoind.height = 11; - tx.__height = 10; + tx.height = 10; var confirmations = bitcoind._getConfirmationsDetail(tx); confirmations.should.equal(2); }); @@ -1937,7 +1938,7 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var tx = new Transaction(txhex); bitcoind.height = 3; - tx.__height = 10; + tx.height = 10; var confirmations = bitcoind._getConfirmationsDetail(tx); log.warn.callCount.should.equal(1); confirmations.should.equal(0); @@ -1946,7 +1947,7 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var tx = new Transaction(txhex); bitcoind.height = 1000; - tx.__height = 1; + tx.height = 1; var confirmations = bitcoind._getConfirmationsDetail(tx); confirmations.should.equal(1000); }); @@ -1955,46 +1956,37 @@ describe('Bitcoin Service', function() { describe('#_getAddressDetailsForTransaction', function() { it('will calculate details for the transaction', function(done) { /* jshint sub:true */ - var tx = bitcore.Transaction({ - 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', - 'version': 1, - 'inputs': [ + var tx = { + inputs: [ { - 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', - 'outputIndex': 0, - 'sequenceNumber': 4294967295, - 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', - 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', - 'output': { - 'satoshis': 1000000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' - } + satoshis: 1000000000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' } ], - 'outputs': [ + outputs: [ { - 'satoshis': 100000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + satoshis: 100000000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' }, { - 'satoshis': 200000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + satoshis: 200000000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' }, { - 'satoshis': 50000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + satoshis: 50000000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' }, { - 'satoshis': 300000000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + satoshis: 300000000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' }, { - 'satoshis': 349990000, - 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + satoshis: 349990000, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW' } ], - 'nLockTime': 0 - }); + locktime: 0 + }; var bitcoind = new BitcoinService(baseConfig); var addresses = ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']; var details = bitcoind._getAddressDetailsForTransaction(tx, addresses); @@ -2008,105 +2000,40 @@ describe('Bitcoin Service', function() { }); }); - describe('#_getDetailedTransaction', function() { + describe('#_getAddressDetailedTransaction', function() { it('will get detailed transaction info', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var tx = { - populateInputs: sinon.stub().callsArg(2), - __height: 20, - __timestamp: 1453134151, - isCoinbase: sinon.stub().returns(false), - getFee: sinon.stub().returns(1000) + height: 20, }; var bitcoind = new BitcoinService(baseConfig); - bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, tx); + bitcoind.getDetailedTransaction = sinon.stub().callsArgWith(1, null, tx); bitcoind.height = 300; + var addresses = {}; bitcoind._getAddressDetailsForTransaction = sinon.stub().returns({ - addresses: {}, + addresses: addresses, satoshis: 1000, }); - bitcoind._getDetailedTransaction(txid, {}, function(err) { + bitcoind._getAddressDetailedTransaction(txid, {}, function(err, details) { if (err) { return done(err); } + details.addresses.should.equal(addresses); + details.satoshis.should.equal(1000); + details.confirmations.should.equal(281); + details.tx.should.equal(tx); done(); }); }); - it('give error from getTransactionWithBlockInfo', function(done) { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - var bitcoind = new BitcoinService(baseConfig); - bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, new Error('test')); - bitcoind._getDetailedTransaction(txid, {}, function(err) { - err.should.be.instanceof(Error); - done(); - }); - }); - it('give error from populateInputs', function(done) { + it('give error from getDetailedTransaction', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - var tx = { - populateInputs: sinon.stub().callsArgWith(2, new Error('test')), - }; var bitcoind = new BitcoinService(baseConfig); - bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, tx); - bitcoind._getDetailedTransaction(txid, {}, function(err) { + bitcoind.getDetailedTransaction = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind._getAddressDetailedTransaction(txid, {}, function(err) { err.should.be.instanceof(Error); done(); }); }); - - it('will correct detailed info', function(done) { - // block #314159 - // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e - // outputIndex 1 - var txAddress = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; - var txString = '0100000001a08ee59fcd5d86fa170abb6d925d62d5c5c476359681b70877c04f270c4ef246000000008a47304402203fb9b476bb0c37c9b9ed5784ebd67ae589492be11d4ae1612be29887e3e4ce750220741ef83781d1b3a5df8c66fa1957ad0398c733005310d7d9b1d8c2310ef4f74c0141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02b0a75fac000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac20badc02000000001976a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac00000000'; - var previousTxString = '010000000155532fad2869bb951b0bd646a546887f6ee668c4c0ee13bf3f1c4bce6d6e3ed9000000008c4930460221008540795f4ef79b1d2549c400c61155ca5abbf3089c84ad280e1ba6db2a31abce022100d7d162175483d51174d40bba722e721542c924202a0c2970b07e680b51f3a0670141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02f0af3caf000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac80969800000000001976a91421277e65777760d1f3c7c982ba14ed8f934f005888ac00000000'; - var transaction = new Transaction(); - var previousTransaction = new Transaction(); - previousTransaction.fromString(previousTxString); - var previousTransactionTxid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; - transaction.fromString(txString); - var txid = transaction.hash; - transaction.__blockHash = '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45'; - transaction.__height = 314159; - transaction.__timestamp = 1407292005; - var bitcoind = new BitcoinService(baseConfig); - bitcoind.height = 314159; - bitcoind.getTransactionWithBlockInfo = sinon.stub().callsArgWith(1, null, transaction); - bitcoind.getTransaction = function(prevTxid, callback) { - prevTxid.should.equal(previousTransactionTxid); - setImmediate(function() { - callback(null, previousTransaction); - }); - }; - var transactionInfo = { - addresses: {}, - txid: txid, - timestamp: 1407292005, - satoshis: 48020000, - address: txAddress - }; - transactionInfo.addresses[txAddress] = {}; - transactionInfo.addresses[txAddress].outputIndexes = [1]; - transactionInfo.addresses[txAddress].inputIndexes = []; - bitcoind._getAddressDetailsForTransaction = sinon.stub().returns(transactionInfo); - bitcoind._getDetailedTransaction(txid, {}, function(err, info) { - if (err) { - return done(err); - } - info.addresses[txAddress].should.deep.equal({ - outputIndexes: [1], - inputIndexes: [] - }); - info.satoshis.should.equal(48020000); - info.height.should.equal(314159); - info.confirmations.should.equal(1); - info.timestamp.should.equal(1407292005); - info.fees.should.equal(20000); - info.tx.should.equal(transaction); - done(); - }); - }); }); describe('#_getAddressStrings', function() { @@ -2221,7 +2148,7 @@ describe('Bitcoin Service', function() { }); it('will paginate', function(done) { var bitcoind = new BitcoinService(baseConfig); - bitcoind._getDetailedTransaction = function(txid, options, callback) { + bitcoind._getAddressDetailedTransaction = function(txid, options, callback) { callback(null, txid); }; var txids = ['one', 'two', 'three', 'four']; @@ -3075,7 +3002,7 @@ describe('Bitcoin Service', function() { }); }); - describe('#getTransactionWithBlockInfo', function() { + describe('#getDetailedTransaction', function() { var txBuffer = new Buffer('01000000016f95980911e01c2c664b3e78299527a47933aac61a515930a8fe0213d1ac9abe01000000da0047304402200e71cda1f71e087c018759ba3427eb968a9ea0b1decd24147f91544629b17b4f0220555ee111ed0fc0f751ffebf097bdf40da0154466eb044e72b6b3dcd5f06807fa01483045022100c86d6c8b417bff6cc3bbf4854c16bba0aaca957e8f73e19f37216e2b06bb7bf802205a37be2f57a83a1b5a8cc511dc61466c11e9ba053c363302e7b99674be6a49fc0147522102632178d046673c9729d828cfee388e121f497707f810c131e0d3fc0fe0bd66d62103a0951ec7d3a9da9de171617026442fcd30f34d66100fab539853b43f508787d452aeffffffff0240420f000000000017a9148a31d53a448c18996e81ce67811e5fb7da21e4468738c9d6f90000000017a9148ce5408cfeaddb7ccb2545ded41ef478109454848700000000', 'hex'); var info = { blockHash: '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6', @@ -3083,7 +3010,40 @@ describe('Bitcoin Service', function() { timestamp: 1439559434000, buffer: txBuffer }; - + var rpcRawTransaction = { + hex: txBuffer.toString('hex'), + blockhash: info.blockHash, + height: info.height, + version: 1, + locktime: 411451, + time: info.timestamp, + vin: [ + { + valueSat: 110, + address: 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW', + txid: '3d003413c13eec3fa8ea1fe8bbff6f40718c66facffe2544d7516c9e2900cac2', + sequence: 0xFFFFFFFF, + vout: 0, + scriptSig: { + hex: 'scriptSigHex', + asm: 'scriptSigAsm' + } + } + ], + vout: [ + { + spentTxId: '4316b98e7504073acd19308b4b8c9f4eeb5e811455c54c0ebfe276c0b1eb6315', + spentIndex: 2, + spentHeight: 100, + valueSat: 100, + scriptPubKey: { + hex: '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', + asm: 'OP_DUP OP_HASH160 0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc OP_EQUALVERIFY OP_CHECKSIG', + addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] + } + } + ] + }; it('should give a transaction with height and timestamp', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.nodes.push({ @@ -3092,39 +3052,74 @@ describe('Bitcoin Service', function() { } }); var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; - bitcoind.getTransactionWithBlockInfo(txid, function(err) { + bitcoind.getDetailedTransaction(txid, function(err) { should.exist(err); err.should.be.instanceof(errors.RPCError); done(); }); }); - it('should give a transaction with height and timestamp', function(done) { + it('should give a transaction with all properties', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.nodes.push({ client: { getRawTransaction: sinon.stub().callsArgWith(2, null, { - result: { - hex: txBuffer.toString('hex'), - blockhash: info.blockHash, - height: info.height, - time: info.timestamp, - vout: [ - { - spentTxId: 'txid', - spentIndex: 2, - spentHeight: 100 - } - ] - } + result: rpcRawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.not.exist(tx.coinbase); + should.equal(tx.hex, txBuffer.toString('hex')); + should.equal(tx.blockHash, '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6'); + should.equal(tx.height, 530482); + should.equal(tx.blockTimestamp, 1439559434000); + should.equal(tx.version, 1); + should.equal(tx.locktime, 411451); + should.equal(tx.feeSatoshis, 10); + should.equal(tx.inputSatoshis, 110); + should.equal(tx.outputSatoshis, 100); + should.equal(tx.hash, txid); + var input = tx.inputs[0]; + should.equal(input.prevTxId, '3d003413c13eec3fa8ea1fe8bbff6f40718c66facffe2544d7516c9e2900cac2'); + should.equal(input.outputIndex, 0); + should.equal(input.satoshis, 110); + should.equal(input.sequence, 0xFFFFFFFF); + should.equal(input.script, 'scriptSigHex'); + should.equal(input.scriptAsm, 'scriptSigAsm'); + should.equal(input.address, 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'); + var output = tx.outputs[0]; + should.equal(output.satoshis, 100); + should.equal(output.script, '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'); + should.equal(output.scriptAsm, 'OP_DUP OP_HASH160 0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc OP_EQUALVERIFY OP_CHECKSIG'); + should.equal(output.address, 'mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'); + should.equal(output.spentTxId, '4316b98e7504073acd19308b4b8c9f4eeb5e811455c54c0ebfe276c0b1eb6315'); + should.equal(output.spentIndex, 2); + should.equal(output.spentHeight, 100); + done(); + }); + }); + it('should set coinbase to true', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = _.clone(rpcRawTransaction); + delete rawTransaction.vin[0]; + rawTransaction.vin = [ + { + coinbase: 'abcdef' + } + ]; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction }) } }); var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; - bitcoind.getTransactionWithBlockInfo(txid, function(err, tx) { - should.equal(tx.__blockHash, '00000000000ec715852ea2ecae4dc8563f62d603c820f81ac284cd5be0a944d6'); - should.equal(tx.__height, 530482); - should.equal(tx.__timestamp, 1439559434000); + bitcoind.getDetailedTransaction(txid, function(err, tx) { should.exist(tx); + should.equal(tx.coinbase, true); done(); }); }); From cd4432652d9e7fcb053e35cf51dbdc9e3e6f6ea9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 13 May 2016 18:51:01 -0400 Subject: [PATCH 182/299] main: remove transaction with populate methods The methods populateInputs and populateSpentInfo are nolonger necessary or used now that there is is getDetailedTransaction. --- index.js | 1 - lib/services/bitcoind.js | 2 +- lib/transaction.js | 74 ---------------- regtest/node.js | 2 +- test/services/bitcoind.unit.js | 2 +- test/transaction.unit.js | 152 --------------------------------- 6 files changed, 3 insertions(+), 230 deletions(-) delete mode 100644 lib/transaction.js delete mode 100644 test/transaction.unit.js diff --git a/index.js b/index.js index 0a210849..6626fe18 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,6 @@ module.exports = require('./lib'); module.exports.Node = require('./lib/node'); -module.exports.Transaction = require('./lib/transaction'); module.exports.Service = require('./lib/service'); module.exports.errors = require('./lib/errors'); diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 89b9cf58..7f4c68a9 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -11,12 +11,12 @@ var LRU = require('lru-cache'); var BitcoinRPC = require('bitcoind-rpc'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; +var Transaction = bitcore.Transaction; var index = require('../'); var errors = index.errors; var log = index.log; var Service = require('../service'); -var Transaction = require('../transaction'); /** * Provides a friendly event driven API to bitcoind in Node.js. Manages starting and diff --git a/lib/transaction.js b/lib/transaction.js deleted file mode 100644 index 66042a16..00000000 --- a/lib/transaction.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict'; - -var async = require('async'); -var bitcore = require('bitcore-lib'); -var Transaction = bitcore.Transaction; - -var MAX_TRANSACTION_LIMIT = 5; - -Transaction.prototype.populateSpentInfo = function(db, options, callback) { - var self = this; - var txid = self.hash; - - async.eachLimit( - Object.keys(self.outputs), - db.maxTransactionlimit || MAX_TRANSACTION_LIMIT, - function(outputIndex, next) { - db.getSpentInfo({ - txid: txid, - index: parseInt(outputIndex) - }, function(err, info) { - if (err) { - return next(err); - } - self.outputs[outputIndex].__spentTxId = info.txid; - self.outputs[outputIndex].__spentIndex = info.index; - self.outputs[outputIndex].__spentHeight = info.height; - next(); - }); - }, - callback - ); -}; - -Transaction.prototype.populateInputs = function(db, poolTransactions, callback) { - var self = this; - - if(this.isCoinbase()) { - return setImmediate(callback); - } - - async.eachLimit( - this.inputs, - db.maxTransactionLimit || MAX_TRANSACTION_LIMIT, - function(input, next) { - self._populateInput(db, input, poolTransactions, next); - }, - callback - ); -}; - -Transaction.prototype._populateInput = function(db, input, poolTransactions, callback) { - if (!input.prevTxId || !Buffer.isBuffer(input.prevTxId)) { - return callback(new TypeError('Input is expected to have prevTxId as a buffer')); - } - var txid = input.prevTxId.toString('hex'); - db.getTransaction(txid, function(err, prevTx) { - if(err) { - return callback(err); - } else if (!prevTx) { - // Check the pool for transaction - for(var i = 0; i < poolTransactions.length; i++) { - if(txid === poolTransactions[i].hash) { - input.output = poolTransactions[i].outputs[input.outputIndex]; - return callback(); - } - } - return callback(new Error('Previous tx ' + input.prevTxId.toString('hex') + ' not found')); - } - input.output = prevTx.outputs[input.outputIndex]; - callback(); - }); -}; - -module.exports = Transaction; diff --git a/regtest/node.js b/regtest/node.js index 9593e4fd..4f22e43f 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -17,7 +17,7 @@ var should = chai.should(); var BitcoinRPC = require('bitcoind-rpc'); var index = require('..'); -var Transaction = index.Transaction; +var Transaction = bitcore.Transaction; var BitcoreNode = index.Node; var BitcoinService = index.services.Bitcoin; var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG'; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 28d70494..49c0e679 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -15,7 +15,7 @@ var index = require('../../lib'); var log = index.log; var errors = index.errors; -var Transaction = require('../../lib/transaction'); +var Transaction = bitcore.Transaction; var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/bitcoin.conf'))); var BitcoinService = proxyquire('../../lib/services/bitcoind', { fs: { diff --git a/test/transaction.unit.js b/test/transaction.unit.js deleted file mode 100644 index 0a78c8c5..00000000 --- a/test/transaction.unit.js +++ /dev/null @@ -1,152 +0,0 @@ -'use strict'; - -var should = require('chai').should(); -var sinon = require('sinon'); -var bitcoinlib = require('../'); -var Transaction = bitcoinlib.Transaction; - -describe('Bitcoin Transaction', function() { - - describe('#populateSpentInfo', function() { - it('will call db.getSpentInfo with correct arguments', function(done) { - var tx = new Transaction(); - tx.to('1AGNa15ZQXAZUgFiqJ2i7Z2DPU2J6hW62i', 1000); - tx.to('3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', 2000); - var expectedHash = tx.hash; - var expectedIndex = 2; - var expectedHeight = 300000; - var db = { - getSpentInfo: sinon.stub().callsArgWith(1, null, { - txid: expectedHash, - index: expectedIndex, - height: expectedHeight - }) - }; - tx.populateSpentInfo(db, {}, function(err) { - if (err) { - return done(err); - } - db.getSpentInfo.args[0][0].txid.should.equal(tx.hash); - db.getSpentInfo.args[0][0].index.should.equal(0); - tx.outputs[0].__spentTxId.should.equal(expectedHash); - tx.outputs[0].__spentIndex.should.equal(expectedIndex); - tx.outputs[0].__spentHeight.should.equal(expectedHeight); - - db.getSpentInfo.args[1][0].txid.should.equal(tx.hash); - db.getSpentInfo.args[1][0].index.should.equal(1); - tx.outputs[1].__spentTxId.should.equal(expectedHash); - tx.outputs[1].__spentIndex.should.equal(expectedIndex); - tx.outputs[1].__spentHeight.should.equal(expectedHeight); - done(); - }); - }); - }); - - describe('#populateInputs', function() { - it('will call _populateInput with transactions', function() { - var tx = new Transaction(); - tx.isCoinbase = sinon.stub().returns(false); - tx._populateInput = sinon.stub().callsArg(3); - tx.inputs = ['input']; - var transactions = []; - var db = {}; - tx.populateInputs(db, transactions, function(err) { - tx._populateInput.callCount.should.equal(1); - tx._populateInput.args[0][0].should.equal(db); - tx._populateInput.args[0][1].should.equal('input'); - tx._populateInput.args[0][2].should.equal(transactions); - }); - }); - it('will skip coinbase transactions', function() { - var tx = new Transaction(); - tx.isCoinbase = sinon.stub().returns(true); - tx._populateInput = sinon.stub().callsArg(3); - tx.inputs = ['input']; - var transactions = []; - var db = {}; - tx.populateInputs(db, transactions, function(err) { - tx._populateInput.callCount.should.equal(0); - }); - }); - }); - - describe('#_populateInput', function() { - var input = { - prevTxId: new Buffer('d6cffbb343a6a41eeaa199478c985493843bfe6a59d674a5c188787416cbcda3', 'hex'), - outputIndex: 0 - }; - it('should give an error if the input does not have a prevTxId', function(done) { - var badInput = {}; - var tx = new Transaction(); - tx._populateInput({}, badInput, [], function(err) { - should.exist(err); - err.message.should.equal('Input is expected to have prevTxId as a buffer'); - done(); - }); - }); - it('should give an error if the input does not have a valid prevTxId', function(done) { - var badInput = { - prevTxId: 'bad' - }; - var tx = new Transaction(); - tx._populateInput({}, badInput, [], function(err) { - should.exist(err); - err.message.should.equal('Input is expected to have prevTxId as a buffer'); - done(); - }); - }); - it('if an error happened it should pass it along', function(done) { - var tx = new Transaction(); - var db = { - getTransaction: sinon.stub().callsArgWith(1, new Error('error')) - }; - tx._populateInput(db, input, [], function(err) { - should.exist(err); - err.message.should.equal('error'); - done(); - }); - }); - it('should return an error if the transaction for the input does not exist', function(done) { - var tx = new Transaction(); - var db = { - getTransaction: sinon.stub().callsArgWith(1, null, null) - }; - tx._populateInput(db, input, [], function(err) { - should.exist(err); - err.message.should.equal('Previous tx ' + input.prevTxId.toString('hex') + ' not found'); - done(); - }); - }); - it('should look through poolTransactions if database does not have transaction', function(done) { - var tx = new Transaction(); - var db = { - getTransaction: sinon.stub().callsArgWith(1, null, null) - }; - var transactions = [ - { - hash: 'd6cffbb343a6a41eeaa199478c985493843bfe6a59d674a5c188787416cbcda3', - outputs: ['output'] - } - ]; - tx._populateInput(db, input, transactions, function(err) { - should.not.exist(err); - input.output.should.equal('output'); - done(); - }); - }); - it('should set the output on the input', function(done) { - var prevTx = new Transaction(); - prevTx.outputs = ['output']; - var tx = new Transaction(); - var db = { - getTransaction: sinon.stub().callsArgWith(1, null, prevTx) - }; - tx._populateInput(db, input, [], function(err) { - should.not.exist(err); - input.output.should.equal('output'); - done(); - }); - }); - }); - -}); From 2cc06cc34b4ccdc804f6bbfb1efae874405ab125 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 15:39:29 -0400 Subject: [PATCH 183/299] build: update bitcoind release to include mempool spentindex --- scripts/download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download b/scripts/download index 081888c9..ad84d65d 100755 --- a/scripts/download +++ b/scripts/download @@ -7,7 +7,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12-bitcore-rc2-spent" +tag="v0.12-bitcore-rc2-spent2" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From 64ed44072967594d3c17ee1d664ea7a2a1de558f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 17:07:26 -0400 Subject: [PATCH 184/299] docs: update docs to reflect api changes --- docs/services/bitcoind.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index b28d2699..b07fe2d6 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -131,6 +131,11 @@ node.services.bitcoind.getBlock(blockHash, function(err, block) { node.services.bitcoind.getBlockHeader(blockHeight, function(err, blockHeader) { //... }); + +// get the block with a list of txids +node.services.bitcoind.getBlockOverview(blockHash, function(err, blockOverview) { + //... +}; ``` **Retrieving and Sending Transactions** @@ -151,11 +156,9 @@ node.services.bitcoind.getTransaction(txid, function(err, transaction) { //... }); -// also retrieve the block timestamp and height -node.services.bitcoind.getTransactionWithBlockInfo(txid, function(err, transaction) { - console.log(transaction.__blockHash); - console.log(transaction.__height); - console.log(transaction.__timestamp); // in seconds +// retrieve the transaction with input values, fees, spent and block info +node.services.bitcoind.getDetailedTransaction(txid, function(err, transaction) { + //... }); ``` @@ -240,11 +243,7 @@ The history format will be: } }, satoshis: 1000000000, - height: 150, // the block height of the transaction - confirmations: 3, - timestamp: 1442948127, // in seconds - fees: 191, - tx: // the populated transaction + tx: // the same format as getDetailedTransaction } ] } From 8f11a338344ccb9a6afde66e26a6de5459670060 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 17:34:40 -0400 Subject: [PATCH 185/299] test: add getBlockOverview unit tests and refactor --- lib/services/bitcoind.js | 86 ++++++++------------- test/services/bitcoind.unit.js | 134 +++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 55 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 7f4c68a9..6b199ec0 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1378,6 +1378,22 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { }; +Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { + var self = this; + if (_.isNumber(blockArg)) { + self._tryAll(function(done) { + self.client.getBlockHash(blockArg, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + done(null, response.result); + }); + }, callback); + } else { + callback(null, blockArg); + } +}; + /** * Will retrieve a block as a Node.js Buffer * @param {String|Number} block - A block hash or block height number @@ -1387,7 +1403,10 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { // TODO apply performance patch to the RPC method for raw data var self = this; - function queryBlock(blockhash) { + function queryBlock(err, blockhash) { + if (err) { + return callback(err); + } self._tryAll(function(done) { self.client.getBlock(blockhash, false, function(err, response) { if (err) { @@ -1406,23 +1425,7 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { callback(null, cachedBlock); }); } else { - if (_.isNumber(blockArg)) { - self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - done(null, response.result); - }); - }, function(err, blockhash) { - if (err) { - return callback(err); - } - queryBlock(blockhash); - }); - } else { - queryBlock(blockArg); - } + self._maybeGetBlockHash(blockArg, queryBlock); } }; @@ -1434,7 +1437,10 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { Bitcoin.prototype.getBlockOverview = function(blockArg, callback) { var self = this; - function queryBlock(blockhash) { + function queryBlock(err, blockhash) { + if (err) { + return callback(err); + } var cachedBlock = self.blockOverviewCache.get(blockhash); if (cachedBlock) { return setImmediate(function() { @@ -1470,23 +1476,7 @@ Bitcoin.prototype.getBlockOverview = function(blockArg, callback) { } } - if (_.isNumber(blockArg)) { - self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - done(null, response.result); - }); - }, function(err, blockhash) { - if (err) { - return callback(err); - } - queryBlock(blockhash); - }); - } else { - queryBlock(blockArg); - } + self._maybeGetBlockHash(blockArg, queryBlock); }; /** @@ -1498,7 +1488,10 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { // TODO apply performance patch to the RPC method for raw data var self = this; - function queryBlock(blockhash) { + function queryBlock(err, blockhash) { + if (err) { + return callback(err); + } var cachedBlock = self.blockCache.get(blockhash); if (cachedBlock) { return setImmediate(function() { @@ -1518,24 +1511,7 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { } } - if (_.isNumber(blockArg)) { - self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - done(null, response.result); - }); - }, function(err, blockhash) { - if (err) { - return callback(err); - } - queryBlock(blockhash); - }); - } else { - queryBlock(blockArg); - } - + self._maybeGetBlockHash(blockArg, queryBlock); }; /** diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 49c0e679..87ff5abb 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2769,6 +2769,140 @@ describe('Bitcoin Service', function() { }); }); + describe('#_maybeGetBlockHash', function() { + it('will get the block hash if argument is a number', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: 'blockhash' + }); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash(10, function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal('blockhash'); + getBlockHash.callCount.should.equal(1); + done(); + }); + }); + it('will try multiple nodes if one fails', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: 'blockhash' + }); + getBlockHash.onCall(0).callsArgWith(1, {code: -1, message: 'test'}); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash(10, function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal('blockhash'); + getBlockHash.callCount.should.equal(2); + done(); + }); + }); + it('will give error from getBlockHash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, {code: -1, message: 'test'}); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash(10, function(err, hash) { + getBlockHash.callCount.should.equal(2); + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + err.code.should.equal(-1); + done(); + }); + }); + }); + + describe('#getBlockOverview', function() { + var blockhash = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + it('will handle error from maybeGetBlockHash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._maybeGetBlockHash = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind.getBlockOverview(blockhash, function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); + it('will give expected result', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var blockResult = { + hash: blockhash, + version: 536870912, + confirmations: 5, + height: 828781, + chainwork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', + previousblockhash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', + nextblockhash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8', + merkleroot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', + time: 1462979126, + mediantime: 1462976771, + nonce: 2981820714, + bits: '1a13ca10', + difficulty: 847779.0710240941 + }; + var getBlock = sinon.stub().callsArgWith(2, null, { + result: blockResult + }); + bitcoind.nodes.push({ + client: { + getBlock: getBlock + } + }); + function checkBlock(blockOverview) { + blockOverview.hash.should.equal('00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'); + blockOverview.version.should.equal(536870912); + blockOverview.confirmations.should.equal(5); + blockOverview.height.should.equal(828781); + blockOverview.chainWork.should.equal('00000000000000000000000000000000000000000000000ad467352c93bc6a3b'); + blockOverview.prevHash.should.equal('0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9'); + blockOverview.nextHash.should.equal('00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8'); + blockOverview.merkleRoot.should.equal('124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9'); + blockOverview.time.should.equal(1462979126); + blockOverview.medianTime.should.equal(1462976771); + blockOverview.nonce.should.equal(2981820714); + blockOverview.bits.should.equal('1a13ca10'); + blockOverview.difficulty.should.equal(847779.0710240941); + } + bitcoind.getBlockOverview(blockhash, function(err, blockOverview) { + if (err) { + return done(err); + } + checkBlock(blockOverview); + bitcoind.getBlockOverview(blockhash, function(err, blockOverview) { + checkBlock(blockOverview); + getBlock.callCount.should.equal(1); + done(); + }); + }); + }); + }); + describe('#estimateFee', function() { it('will give rpc error', function(done) { var bitcoind = new BitcoinService(baseConfig); From 484b7075894cdb998eaf65e585a4686d5b6a7f09 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 17:39:54 -0400 Subject: [PATCH 186/299] bitcoind: update jsdocs for getDetailedTransaction --- lib/services/bitcoind.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 6b199ec0..921fa59d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1718,6 +1718,7 @@ Bitcoin.prototype.getTransaction = function(txid, callback) { * sequence: 123456789, * script: [hexString], * scriptAsm: [asmString], + * address: '1LCTmj15p7sSXv3jmrPfA6KGs6iuepBiiG', * satoshis: 771146 * } * ], From fa6474e85fa78458e5bd3f24242420e0990c2ee3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 18:01:12 -0400 Subject: [PATCH 187/299] bitcoind: handle block height number as string --- lib/services/bitcoind.js | 5 +++-- test/services/bitcoind.unit.js | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 921fa59d..632b7a12 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1380,9 +1380,10 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { var self = this; - if (_.isNumber(blockArg)) { + if (_.isNumber(blockArg) || blockArg.length < 64) { + var height = parseInt(blockArg, 10); self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { + self.client.getBlockHash(height, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 87ff5abb..c36c3756 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2789,6 +2789,25 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will get the block hash if argument is a number (as string)', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, null, { + result: 'blockhash' + }); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash('10', function(err, hash) { + if (err) { + return done(err); + } + hash.should.equal('blockhash'); + getBlockHash.callCount.should.equal(1); + done(); + }); + }); it('will try multiple nodes if one fails', function(done) { var bitcoind = new BitcoinService(baseConfig); var getBlockHash = sinon.stub().callsArgWith(1, null, { From a48bcaf900fc2f853c4a276f723a707cfdaaeace Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 17 May 2016 18:16:38 -0400 Subject: [PATCH 188/299] web: added logging for web socket events --- lib/bus.js | 1 + lib/node.js | 7 +++++-- lib/services/bitcoind.js | 2 ++ lib/services/web.js | 10 +++++++++- test/services/web.unit.js | 5 +++++ 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/bus.js b/lib/bus.js index 720261d0..d4f3bdd5 100644 --- a/lib/bus.js +++ b/lib/bus.js @@ -13,6 +13,7 @@ var util = require('util'); function Bus(params) { events.EventEmitter.call(this); this.node = params.node; + this.remoteAddress = params.remoteAddress; } util.inherits(Bus, events.EventEmitter); diff --git a/lib/node.js b/lib/node.js index 75431598..c2350b50 100644 --- a/lib/node.js +++ b/lib/node.js @@ -87,8 +87,11 @@ Node.prototype._setNetwork = function(config) { * Will instantiate a new Bus for this node. * @returns {Bus} */ -Node.prototype.openBus = function() { - return new Bus({node: this}); +Node.prototype.openBus = function(options) { + if (!options) { + options = {}; + } + return new Bus({node: this, remoteAddress: options.remoteAddress}); }; /** diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 632b7a12..04055a47 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -195,6 +195,7 @@ Bitcoin.prototype.getPublishEvents = function() { Bitcoin.prototype.subscribe = function(name, emitter) { this.subscriptions[name].push(emitter); + log.info(emitter.remoteAddress, 'subscribing:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); }; Bitcoin.prototype.unsubscribe = function(name, emitter) { @@ -202,6 +203,7 @@ Bitcoin.prototype.unsubscribe = function(name, emitter) { if (index > -1) { this.subscriptions[name].splice(index, 1); } + log.info(emitter.remoteAddress, 'unsubscribing:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); }; Bitcoin.prototype._getDefaultConfig = function() { diff --git a/lib/services/web.js b/lib/services/web.js index 1e292676..0008b745 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -158,23 +158,30 @@ WebService.prototype.getEventNames = function() { return eventNames; }; +WebService.prototype._getRemoteAddress = function(socket) { + return socket.conn.remoteAddress; +}; + /** * This function is responsible for managing a socket.io connection, including * instantiating a new Bus, subscribing/unsubscribing and handling RPC commands. * @param {Socket} socket - A socket.io socket instance */ WebService.prototype.socketHandler = function(socket) { - var bus = this.node.openBus(); + var self = this; + var bus = this.node.openBus({remoteAddress: self._getRemoteAddress(socket)}); if (this.enableSocketRPC) { socket.on('message', this.socketMessageHandler.bind(this)); } socket.on('subscribe', function(name, params) { + log.info(self._getRemoteAddress(socket), 'web socket subscribe:', name); bus.subscribe(name, params); }); socket.on('unsubscribe', function(name, params) { + log.info(self._getRemoteAddress(socket), 'web socket unsubscribe:', name); bus.unsubscribe(name, params); }); @@ -194,6 +201,7 @@ WebService.prototype.socketHandler = function(socket) { }); socket.on('disconnect', function() { + log.info(self._getRemoteAddress(socket), 'web socket disconnect'); bus.close(); }); }; diff --git a/test/services/web.unit.js b/test/services/web.unit.js index 5a582784..c0f720d6 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -212,6 +212,7 @@ describe('WebService', function() { describe('#socketHandler', function() { var bus = new EventEmitter(); + bus.remoteAddress = '127.0.0.1'; var Module1 = function() {}; Module1.prototype.getPublishEvents = function() { @@ -241,6 +242,8 @@ describe('WebService', function() { done(); }; socket = new EventEmitter(); + socket.conn = {}; + socket.conn.remoteAddress = '127.0.0.1'; web.socketHandler(socket); socket.emit('message', 'data'); }); @@ -250,6 +253,8 @@ describe('WebService', function() { web.eventNames = web.getEventNames(); web.socketMessageHandler = sinon.stub(); socket = new EventEmitter(); + socket.conn = {}; + socket.conn.remoteAddress = '127.0.0.1'; web.socketHandler(socket); socket.on('message', function() { web.socketMessageHandler.callCount.should.equal(0); From 4df9b5f6cfb86c00eb34044d4086cf9ca18d9d6d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 17 May 2016 20:20:11 -0400 Subject: [PATCH 189/299] bitcoind: add addresstxid event --- lib/bus.js | 2 + lib/services/bitcoind.js | 127 ++++++++++++++++++++++++++++++++- test/services/bitcoind.unit.js | 15 ++-- 3 files changed, 136 insertions(+), 8 deletions(-) diff --git a/lib/bus.js b/lib/bus.js index d4f3bdd5..a43e99c5 100644 --- a/lib/bus.js +++ b/lib/bus.js @@ -2,6 +2,8 @@ var events = require('events'); var util = require('util'); +var bitcore = require('bitcore-lib'); +var _ = bitcore.deps._; /** * The bus represents a connection to node, decoupled from the transport layer, that can diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 04055a47..24dd1ec6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -5,6 +5,7 @@ var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); var bitcore = require('bitcore-lib'); +var Address = bitcore.Address; var zmq = require('zmq'); var async = require('async'); var LRU = require('lru-cache'); @@ -44,6 +45,7 @@ function Bitcoin(options) { this.subscriptions = {}; this.subscriptions.rawtransaction = []; this.subscriptions.hashblock = []; + this.subscriptions.address = {}; // set initial settings this._initDefaults(options); @@ -189,13 +191,19 @@ Bitcoin.prototype.getPublishEvents = function() { scope: this, subscribe: this.subscribe.bind(this, 'hashblock'), unsubscribe: this.unsubscribe.bind(this, 'hashblock') + }, + { + name: 'bitcoind/addresstxid', + scope: this, + subscribe: this.subscribeAddress.bind(this), + unsubscribe: this.unsubscribeAddress.bind(this) } ]; }; Bitcoin.prototype.subscribe = function(name, emitter) { this.subscriptions[name].push(emitter); - log.info(emitter.remoteAddress, 'subscribing:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); + log.info(emitter.remoteAddress, 'subscribe:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); }; Bitcoin.prototype.unsubscribe = function(name, emitter) { @@ -203,7 +211,54 @@ Bitcoin.prototype.unsubscribe = function(name, emitter) { if (index > -1) { this.subscriptions[name].splice(index, 1); } - log.info(emitter.remoteAddress, 'unsubscribing:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); + log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); +}; + +Bitcoin.prototype.subscribeAddress = function(emitter, addresses) { + for(var i = 0; i < addresses.length; i++) { + var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); + if(!this.subscriptions.address[hashHex]) { + this.subscriptions.address[hashHex] = []; + } + this.subscriptions.address[hashHex].push(emitter); + } + log.info(emitter.remoteAddress, 'subscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); +}; + +Bitcoin.prototype.unsubscribeAddress = function(emitter, addresses) { + if(!addresses) { + return this.unsubscribeAddressAll(emitter); + } + for(var i = 0; i < addresses.length; i++) { + var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); + if(this.subscriptions.address[hashHex]) { + var emitters = this.subscriptions.address[hashHex]; + var index = emitters.indexOf(emitter); + if(index > -1) { + emitters.splice(index, 1); + } + } + } + log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); +}; + +/** + * A helper function for the `unsubscribe` method to unsubscribe from all addresses. + * @param {String} name - The name of the event + * @param {EventEmitter} emitter - An instance of an event emitter + */ +Bitcoin.prototype.unsubscribeAddressAll = function(emitter) { + for(var hashHex in this.subscriptions.address) { + var emitters = this.subscriptions.address[hashHex]; + var index = emitters.indexOf(emitter); + if(index > -1) { + emitters.splice(index, 1); + } + if (emitters.length === 0) { + delete this.subscriptions.address[hashHex]; + } + } + log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); }; Bitcoin.prototype._getDefaultConfig = function() { @@ -477,12 +532,72 @@ Bitcoin.prototype._updateTip = function(node, message) { }); } } +}; +Bitcoin.prototype._getAddressHashesFromInput = function(input, addressHashes) { + if (!input.script) { + return; + } + var hashBuffer; + var script = input.script; + if (script.isPublicKeyHashIn()) { + hashBuffer = bitcore.crypto.Hash.sha256ripemd160(input.script.chunks[1].buf); + } else if (script.isScriptHashIn()) { + hashBuffer = bitcore.crypto.Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); + } else { + return; + } + addressHashes.push(hashBuffer.toString('hex')); +}; + +Bitcoin.prototype._getAddressHashesFromOutput = function(output, addressHashes) { + if (!output.script) { + return; + } + var script = output.script; + var hashBuffer; + if (script.isPublicKeyHashOut()) { + hashBuffer = script.chunks[2].buf; + } else if (script.isScriptHashOut()) { + hashBuffer = script.chunks[1].buf; + } else { + return; + } + addressHashes.push(hashBuffer.toString('hex')); +}; + +Bitcoin.prototype._getAddressHashesFromTransaction = function(transaction) { + var addressHashes = []; + + for (var i = 0; i < transaction.inputs.length; i++) { + var input = transaction.inputs[i]; + this._getAddressHashesFromInput(input, addressHashes); + } + + for (var j = 0; j < transaction.outputs.length; j++) { + var output = transaction.outputs[j]; + this._getAddressHashesFromOutput(output, addressHashes); + } + + return addressHashes; +}; + +Bitcoin.prototype._notifyAddressTxidSubscribers = function(txid, transaction) { + var addressHashes = this._getAddressHashesFromTransaction(transaction); + for (var i = 0; i < addressHashes.length; i++) { + if(this.subscriptions.address[addressHashes[i]]) { + var emitters = this.subscriptions.address[addressHashes[i]]; + for(var j = 0; j < emitters.length; j++) { + emitters[j].emit('bitcoind/addresstxid', txid); + } + } + } }; Bitcoin.prototype._zmqTransactionHandler = function(node, message) { var self = this; - var id = bitcore.crypto.Hash.sha256sha256(message).toString('binary'); + var hash = bitcore.crypto.Hash.sha256sha256(message); + var id = hash.toString('binary'); if (!self.zmqKnownTransactions.get(id)) { self.zmqKnownTransactions.set(id, true); self.emit('tx', message); @@ -491,6 +606,12 @@ Bitcoin.prototype._zmqTransactionHandler = function(node, message) { for (var i = 0; i < this.subscriptions.rawtransaction.length; i++) { this.subscriptions.rawtransaction[i].emit('bitcoind/rawtransaction', message.toString('hex')); } + + var tx = bitcore.Transaction(); + tx.fromString(message); + var txid = bitcore.util.buffer.reverse(hash).toString('hex'); + self._notifyAddressTxidSubscribers(txid, tx); + } }; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index c36c3756..92353bcf 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -74,6 +74,7 @@ describe('Bitcoin Service', function() { it('will set subscriptions', function() { var bitcoind = new BitcoinService(baseConfig); bitcoind.subscriptions.should.deep.equal({ + address: {}, rawtransaction: [], hashblock: [] }); @@ -100,7 +101,7 @@ describe('Bitcoin Service', function() { var bitcoind = new BitcoinService(baseConfig); var events = bitcoind.getPublishEvents(); should.exist(events); - events.length.should.equal(2); + events.length.should.equal(3); events[0].name.should.equal('bitcoind/rawtransaction'); events[0].scope.should.equal(bitcoind); events[0].subscribe.should.be.a('function'); @@ -109,6 +110,10 @@ describe('Bitcoin Service', function() { events[1].scope.should.equal(bitcoind); events[1].subscribe.should.be.a('function'); events[1].unsubscribe.should.be.a('function'); + events[2].name.should.equal('bitcoind/addresstxid'); + events[2].scope.should.equal(bitcoind); + events[2].subscribe.should.be.a('function'); + events[2].unsubscribe.should.be.a('function'); }); it('will call subscribe/unsubscribe with correct args', function() { var bitcoind = new BitcoinService(baseConfig); @@ -718,7 +723,7 @@ describe('Bitcoin Service', function() { describe('#_zmqTransactionHandler', function() { it('will emit to subscribers', function(done) { var bitcoind = new BitcoinService(baseConfig); - var expectedBuffer = new Buffer('abcdef', 'hex'); + var expectedBuffer = new Buffer(txhex, 'hex'); var emitter = new EventEmitter(); bitcoind.subscriptions.rawtransaction.push(emitter); emitter.on('bitcoind/rawtransaction', function(hex) { @@ -731,7 +736,7 @@ describe('Bitcoin Service', function() { }); it('will NOT emit to subscribers more than once for the same tx', function(done) { var bitcoind = new BitcoinService(baseConfig); - var expectedBuffer = new Buffer('abcdef', 'hex'); + var expectedBuffer = new Buffer(txhex, 'hex'); var emitter = new EventEmitter(); bitcoind.subscriptions.rawtransaction.push(emitter); emitter.on('bitcoind/rawtransaction', function() { @@ -743,7 +748,7 @@ describe('Bitcoin Service', function() { }); it('will emit "tx" event', function(done) { var bitcoind = new BitcoinService(baseConfig); - var expectedBuffer = new Buffer('abcdef', 'hex'); + var expectedBuffer = new Buffer(txhex, 'hex'); bitcoind.on('tx', function(buffer) { buffer.should.be.instanceof(Buffer); buffer.toString('hex').should.equal(expectedBuffer.toString('hex')); @@ -754,7 +759,7 @@ describe('Bitcoin Service', function() { }); it('will NOT emit "tx" event more than once for the same tx', function(done) { var bitcoind = new BitcoinService(baseConfig); - var expectedBuffer = new Buffer('abcdef', 'hex'); + var expectedBuffer = new Buffer(txhex, 'hex'); bitcoind.on('tx', function() { done(); }); From 57cb146ce072e777f90f0e05a95f2bba86445012 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 17 May 2016 23:03:04 -0400 Subject: [PATCH 190/299] build: fix jshint unused variable warnings --- lib/bus.js | 2 -- lib/services/bitcoind.js | 1 - 2 files changed, 3 deletions(-) diff --git a/lib/bus.js b/lib/bus.js index a43e99c5..d4f3bdd5 100644 --- a/lib/bus.js +++ b/lib/bus.js @@ -2,8 +2,6 @@ var events = require('events'); var util = require('util'); -var bitcore = require('bitcore-lib'); -var _ = bitcore.deps._; /** * The bus represents a connection to node, decoupled from the transport layer, that can diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 24dd1ec6..4d638cef 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -5,7 +5,6 @@ var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); var bitcore = require('bitcore-lib'); -var Address = bitcore.Address; var zmq = require('zmq'); var async = require('async'); var LRU = require('lru-cache'); From bf080422ed4b4b2514aa47b1efa32534b6bdee21 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 00:17:45 -0400 Subject: [PATCH 191/299] web: get remoteAddress for socket with cloudflare header --- lib/services/web.js | 11 ++++++----- test/services/web.unit.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/services/web.js b/lib/services/web.js index 0008b745..b050b5d5 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -159,7 +159,7 @@ WebService.prototype.getEventNames = function() { }; WebService.prototype._getRemoteAddress = function(socket) { - return socket.conn.remoteAddress; + return socket.client.request.headers['cf-connecting-ip'] || socket.conn.remoteAddress; }; /** @@ -169,19 +169,20 @@ WebService.prototype._getRemoteAddress = function(socket) { */ WebService.prototype.socketHandler = function(socket) { var self = this; - var bus = this.node.openBus({remoteAddress: self._getRemoteAddress(socket)}); + var remoteAddress = self._getRemoteAddress(socket); + var bus = this.node.openBus({remoteAddress: remoteAddress}); if (this.enableSocketRPC) { socket.on('message', this.socketMessageHandler.bind(this)); } socket.on('subscribe', function(name, params) { - log.info(self._getRemoteAddress(socket), 'web socket subscribe:', name); + log.info(remoteAddress, 'web socket subscribe:', name); bus.subscribe(name, params); }); socket.on('unsubscribe', function(name, params) { - log.info(self._getRemoteAddress(socket), 'web socket unsubscribe:', name); + log.info(remoteAddress, 'web socket unsubscribe:', name); bus.unsubscribe(name, params); }); @@ -201,7 +202,7 @@ WebService.prototype.socketHandler = function(socket) { }); socket.on('disconnect', function() { - log.info(self._getRemoteAddress(socket), 'web socket disconnect'); + log.info(remoteAddress, 'web socket disconnect'); bus.close(); }); }; diff --git a/test/services/web.unit.js b/test/services/web.unit.js index c0f720d6..57490932 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -210,6 +210,32 @@ describe('WebService', function() { }); }); + describe('#_getRemoteAddress', function() { + it('will get remote address from cloudflare header', function() { + var web = new WebService({node: defaultNode}); + var socket = {}; + socket.conn = {}; + socket.client = {}; + socket.client.request = {}; + socket.client.request.headers = { + 'cf-connecting-ip': '127.0.0.1' + }; + var remoteAddress = web._getRemoteAddress(socket); + remoteAddress.should.equal('127.0.0.1'); + }); + it('will get remote address from connection', function() { + var web = new WebService({node: defaultNode}); + var socket = {}; + socket.conn = {}; + socket.conn.remoteAddress = '127.0.0.1'; + socket.client = {}; + socket.client.request = {}; + socket.client.request.headers = {}; + var remoteAddress = web._getRemoteAddress(socket); + remoteAddress.should.equal('127.0.0.1'); + }); + }); + describe('#socketHandler', function() { var bus = new EventEmitter(); bus.remoteAddress = '127.0.0.1'; @@ -244,6 +270,9 @@ describe('WebService', function() { socket = new EventEmitter(); socket.conn = {}; socket.conn.remoteAddress = '127.0.0.1'; + socket.client = {}; + socket.client.request = {}; + socket.client.request.headers = {}; web.socketHandler(socket); socket.emit('message', 'data'); }); @@ -255,6 +284,9 @@ describe('WebService', function() { socket = new EventEmitter(); socket.conn = {}; socket.conn.remoteAddress = '127.0.0.1'; + socket.client = {}; + socket.client.request = {}; + socket.client.request.headers = {}; web.socketHandler(socket); socket.on('message', function() { web.socketMessageHandler.callCount.should.equal(0); From 522c822304327f8ec7f9491b23787e3e3e2afd85 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 00:24:08 -0400 Subject: [PATCH 192/299] test: use callback instead of ready event --- regtest/node.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/regtest/node.js b/regtest/node.js index 4f22e43f..17d316f5 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -69,7 +69,10 @@ describe('Node Functionality', function() { log.error(err); }); - node.on('ready', function() { + node.start(function(err) { + if (err) { + return done(err); + } client = new BitcoinRPC({ protocol: 'http', @@ -94,12 +97,7 @@ describe('Node Functionality', function() { throw err; } }); - }); - node.start(function(err) { - if (err) { - throw err; - } }); From 6fbadb6c42530735134b785196669379a66df359 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 00:33:57 -0400 Subject: [PATCH 193/299] test: stub logging in unit tests --- test/bus.integration.js | 11 +++++++++++ test/node.unit.js | 23 +++++++++++++++++++++++ test/scaffold/start.integration.js | 10 ++++++++++ test/services/bitcoind.unit.js | 28 ++++++++++++++++++++++++++++ test/services/web.unit.js | 11 +++++++++++ 5 files changed, 83 insertions(+) diff --git a/test/bus.integration.js b/test/bus.integration.js index 0573695d..42a27d84 100644 --- a/test/bus.integration.js +++ b/test/bus.integration.js @@ -1,9 +1,12 @@ 'use strict'; +var sinon = require('sinon'); var Service = require('../lib/service'); var BitcoreNode = require('../lib/node'); var util = require('util'); var should = require('chai').should(); +var index = require('../lib'); +var log = index.log; var TestService = function(options) { this.node = options.node; @@ -41,6 +44,14 @@ TestService.prototype.unsubscribe = function(name, emitter) { describe('Bus Functionality', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('should subscribe to testEvent', function(done) { var node = new BitcoreNode({ datadir: './', diff --git a/test/node.unit.js b/test/node.unit.js index 1e843fe0..192cb992 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -7,6 +7,8 @@ var Networks = bitcore.Networks; var proxyquire = require('proxyquire'); var util = require('util'); var BaseService = require('../lib/service'); +var index = require('../lib'); +var log = index.log; describe('Bitcore Node', function() { @@ -171,6 +173,13 @@ describe('Bitcore Node', function() { }); describe('#_startService', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will instantiate an instance and load api methods', function() { var node = new Node(baseConfig); function TestService() {} @@ -213,6 +222,13 @@ describe('Bitcore Node', function() { }); describe('#start', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will call start for each service', function(done) { var node = new Node(baseConfig); @@ -302,6 +318,13 @@ describe('Bitcore Node', function() { }); describe('#stop', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will call stop for each service', function(done) { var node = new Node(baseConfig); function TestService() {} diff --git a/test/scaffold/start.integration.js b/test/scaffold/start.integration.js index 5523f29d..887572b6 100644 --- a/test/scaffold/start.integration.js +++ b/test/scaffold/start.integration.js @@ -4,9 +4,19 @@ var should = require('chai').should(); var sinon = require('sinon'); var proxyquire = require('proxyquire'); var BitcoinService = require('../../lib/services/bitcoind'); +var index = require('../../lib'); +var log = index.log; describe('#start', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'error'); + }); + afterEach(function() { + sandbox.restore(); + }); + describe('will dynamically create a node from a configuration', function() { it('require each bitcore-node service with default config', function(done) { diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 92353bcf..96ad398f 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -140,6 +140,13 @@ describe('Bitcoin Service', function() { }); describe('#subscribe', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will push to subscriptions', function() { var bitcoind = new BitcoinService(baseConfig); var emitter = {}; @@ -153,6 +160,13 @@ describe('Bitcoin Service', function() { }); describe('#unsubscribe', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will remove item from subscriptions', function() { var bitcoind = new BitcoinService(baseConfig); var emitter1 = {}; @@ -1106,6 +1120,13 @@ describe('Bitcoin Service', function() { }); describe('#_spawnChildProcess', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will give error from spawn config', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._loadSpawnConfiguration = sinon.stub().throws(new Error('test')); @@ -1263,6 +1284,13 @@ describe('Bitcoin Service', function() { }); describe('#start', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will give error if "spawn" and "connect" are both not configured', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.options = {}; diff --git a/test/services/web.unit.js b/test/services/web.unit.js index 57490932..3d0ff593 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -5,6 +5,9 @@ var sinon = require('sinon'); var EventEmitter = require('events').EventEmitter; var proxyquire = require('proxyquire'); +var index = require('../../lib'); +var log = index.log; + var httpStub = { createServer: sinon.spy() }; @@ -237,6 +240,14 @@ describe('WebService', function() { }); describe('#socketHandler', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); + var bus = new EventEmitter(); bus.remoteAddress = '127.0.0.1'; From 1800294dfe690749e2f6bdbc8c4c486b05716f1c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 10:20:18 -0400 Subject: [PATCH 194/299] bitcoind: change dataformat of addresstxid event Adds the address to the message to quickly determine the address associated with the event. --- lib/services/bitcoind.js | 110 ++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4d638cef..38d8fddb 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -214,30 +214,52 @@ Bitcoin.prototype.unsubscribe = function(name, emitter) { }; Bitcoin.prototype.subscribeAddress = function(emitter, addresses) { + var self = this; + + function addAddress(addressStr) { + if(self.subscriptions.address[addressStr]) { + var emitters = self.subscriptions.address[addressStr]; + var index = emitters.indexOf(emitter); + if (index === -1) { + self.subscriptions.address[addressStr].push(emitter); + } + } else { + self.subscriptions.address[addressStr] = [emitter]; + } + } + for(var i = 0; i < addresses.length; i++) { - var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(!this.subscriptions.address[hashHex]) { - this.subscriptions.address[hashHex] = []; + if (bitcore.Address.isValid(addresses[i], this.node.network)) { + addAddress(addresses[i]); } - this.subscriptions.address[hashHex].push(emitter); } + log.info(emitter.remoteAddress, 'subscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); }; Bitcoin.prototype.unsubscribeAddress = function(emitter, addresses) { + var self = this; if(!addresses) { return this.unsubscribeAddressAll(emitter); } - for(var i = 0; i < addresses.length; i++) { - var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); - if(this.subscriptions.address[hashHex]) { - var emitters = this.subscriptions.address[hashHex]; - var index = emitters.indexOf(emitter); - if(index > -1) { - emitters.splice(index, 1); + + function removeAddress(addressStr) { + var emitters = self.subscriptions.address[addressStr]; + var index = emitters.indexOf(emitter); + if(index > -1) { + emitters.splice(index, 1); + if (emitters.length === 0) { + delete self.subscriptions.address[addressStr]; } } } + + for(var i = 0; i < addresses.length; i++) { + if(this.subscriptions.address[addresses[i]]) { + removeAddress(addresses[i]); + } + } + log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); }; @@ -533,61 +555,43 @@ Bitcoin.prototype._updateTip = function(node, message) { } }; -Bitcoin.prototype._getAddressHashesFromInput = function(input, addressHashes) { - if (!input.script) { - return; - } - var hashBuffer; - var script = input.script; - if (script.isPublicKeyHashIn()) { - hashBuffer = bitcore.crypto.Hash.sha256ripemd160(input.script.chunks[1].buf); - } else if (script.isScriptHashIn()) { - hashBuffer = bitcore.crypto.Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - } else { - return; - } - addressHashes.push(hashBuffer.toString('hex')); -}; - -Bitcoin.prototype._getAddressHashesFromOutput = function(output, addressHashes) { - if (!output.script) { - return; - } - var script = output.script; - var hashBuffer; - if (script.isPublicKeyHashOut()) { - hashBuffer = script.chunks[2].buf; - } else if (script.isScriptHashOut()) { - hashBuffer = script.chunks[1].buf; - } else { - return; - } - addressHashes.push(hashBuffer.toString('hex')); -}; - -Bitcoin.prototype._getAddressHashesFromTransaction = function(transaction) { - var addressHashes = []; +Bitcoin.prototype._getAddressesFromTransaction = function(transaction) { + var addresses = []; for (var i = 0; i < transaction.inputs.length; i++) { var input = transaction.inputs[i]; - this._getAddressHashesFromInput(input, addressHashes); + if (input.script) { + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress) { + addresses.push(inputAddress.toString()); + } + } } for (var j = 0; j < transaction.outputs.length; j++) { var output = transaction.outputs[j]; - this._getAddressHashesFromOutput(output, addressHashes); + if (output.script) { + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress) { + addresses.push(outputAddress.toString()); + } + } } - return addressHashes; + return addresses; }; Bitcoin.prototype._notifyAddressTxidSubscribers = function(txid, transaction) { - var addressHashes = this._getAddressHashesFromTransaction(transaction); - for (var i = 0; i < addressHashes.length; i++) { - if(this.subscriptions.address[addressHashes[i]]) { - var emitters = this.subscriptions.address[addressHashes[i]]; + var addresses = this._getAddressesFromTransaction(transaction); + for (var i = 0; i < addresses.length; i++) { + var address = addresses[i]; + if(this.subscriptions.address[address]) { + var emitters = this.subscriptions.address[address]; for(var j = 0; j < emitters.length; j++) { - emitters[j].emit('bitcoind/addresstxid', txid); + emitters[j].emit('bitcoind/addresstxid', { + address: address, + txid: txid + }); } } } From 28ff52ece68833555131e06836ce1f39e018acec Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 11:20:23 -0400 Subject: [PATCH 195/299] tests: add tests for addresstxid event --- lib/services/bitcoind.js | 2 +- test/services/bitcoind.unit.js | 99 ++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 38d8fddb..ff10c390 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -578,7 +578,7 @@ Bitcoin.prototype._getAddressesFromTransaction = function(transaction) { } } - return addresses; + return _.uniq(addresses); }; Bitcoin.prototype._notifyAddressTxidSubscribers = function(txid, transaction) { diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 96ad398f..0ca30418 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -734,6 +734,105 @@ describe('Bitcoin Service', function() { }); }); + describe('#_getAddressesFromTransaction', function() { + it('will get results using bitcore.Transaction', function() { + var bitcoind = new BitcoinService(baseConfig); + var wif = 'L2Gkw3kKJ6N24QcDuH4XDqt9cTqsKTVNDGz1CRZhk9cq4auDUbJy'; + var privkey = bitcore.PrivateKey.fromWIF(wif); + var inputAddress = privkey.toAddress(bitcore.Networks.testnet); + var outputAddress = bitcore.Address('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'); + var tx = bitcore.Transaction(); + tx.from({ + txid: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + outputIndex: 0, + script: bitcore.Script(inputAddress), + address: inputAddress.toString(), + satoshis: 5000000000 + }); + tx.to(outputAddress, 5000000000); + tx.sign(privkey); + var addresses = bitcoind._getAddressesFromTransaction(tx); + addresses.length.should.equal(2); + addresses[0].should.equal(inputAddress.toString()); + addresses[1].should.equal(outputAddress.toString()); + }); + it('will handle non-standard script types', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = bitcore.Transaction(); + tx.addInput(bitcore.Transaction.Input({ + prevTxId: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + script: bitcore.Script('OP_TRUE'), + outputIndex: 1, + output: { + script: bitcore.Script('OP_TRUE'), + satoshis: 5000000000 + } + })); + tx.addOutput(bitcore.Transaction.Output({ + script: bitcore.Script('OP_TRUE'), + satoshis: 5000000000 + })); + var addresses = bitcoind._getAddressesFromTransaction(tx); + addresses.length.should.equal(0); + }); + it('will handle unparsable script types or missing input script', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = bitcore.Transaction(); + tx.addOutput(bitcore.Transaction.Output({ + script: new Buffer('4c', 'hex'), + satoshis: 5000000000 + })); + var addresses = bitcoind._getAddressesFromTransaction(tx); + addresses.length.should.equal(0); + }); + it('will return unique values', function() { + var bitcoind = new BitcoinService(baseConfig); + var tx = bitcore.Transaction(); + var address = bitcore.Address('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'); + tx.addOutput(bitcore.Transaction.Output({ + script: bitcore.Script(address), + satoshis: 5000000000 + })); + tx.addOutput(bitcore.Transaction.Output({ + script: bitcore.Script(address), + satoshis: 5000000000 + })); + var addresses = bitcoind._getAddressesFromTransaction(tx); + addresses.length.should.equal(1); + }); + }); + + describe('#_notifyAddressTxidSubscribers', function() { + it('will emit event if matching addresses', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind._getAddressesFromTransaction = sinon.stub().returns([address]); + var emitter = new EventEmitter(); + bitcoind.subscriptions.address[address] = [emitter]; + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var transaction = {}; + emitter.on('bitcoind/addresstxid', function(data) { + data.address.should.equal(address); + data.txid.should.equal(txid); + done(); + }); + sinon.spy(emitter, 'emit'); + bitcoind._notifyAddressTxidSubscribers(txid, transaction); + emitter.emit.callCount.should.equal(1); + }); + it('will NOT emit event without matching addresses', function() { + var bitcoind = new BitcoinService(baseConfig); + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind._getAddressesFromTransaction = sinon.stub().returns([address]); + var emitter = new EventEmitter(); + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var transaction = {}; + emitter.emit = sinon.stub(); + bitcoind._notifyAddressTxidSubscribers(txid, transaction); + emitter.emit.callCount.should.equal(0); + }); + }); + describe('#_zmqTransactionHandler', function() { it('will emit to subscribers', function(done) { var bitcoind = new BitcoinService(baseConfig); From 83880910dc3e9fc0ce8f6509be35e84216a0ded9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 11:22:36 -0400 Subject: [PATCH 196/299] docs: add documentation for addresstxid event --- docs/services/bitcoind.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index b07fe2d6..d38f303e 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -298,12 +298,14 @@ node.services.bitcoind.on('block', function(blockHash) { For details on instantiating a bus for a node, see the [Bus Documentation](../bus.md). - Name: `bitcoind/rawtransaction` - Name: `bitcoind/hashblock` +- Name: `bitcoind/addresstxid`, Arguments: [address, address...] **Examples:** ```js bus.subscribe('bitcoind/rawtransaction'); bus.subscribe('bitcoind/hashblock'); +bus.subscribe('bitcoind/addresstxid', ['13FMwCYz3hUhwPcaWuD2M1U2KzfTtvLM89']); bus.on('bitcoind/rawtransaction', function(transactionHex) { //... @@ -312,4 +314,9 @@ bus.on('bitcoind/rawtransaction', function(transactionHex) { bus.on('bitcoind/hashblock', function(blockhashHex) { //... }); + +bus.on('bitcoind/addresstxid', function(data) { + // data.address; + // data.txid; +}); ``` From 73197fdc755d689b2048781b8d508a3377e26033 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 18 May 2016 20:04:23 -0400 Subject: [PATCH 197/299] build: update url to download bitcoin-0.12-bitcore-rc3 --- scripts/download | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/download b/scripts/download index ad84d65d..2e8c6714 100755 --- a/scripts/download +++ b/scripts/download @@ -6,8 +6,8 @@ root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" -url="https://github.com/braydonf/bitcoin/releases/download" -tag="v0.12-bitcore-rc2-spent2" +url="https://github.com/bitpay/bitcoin/releases/download" +tag="v0.12-bitcore-rc3" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From cd9bbc8661765104c7f5b257a883afd39a4579ce Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 11:27:54 -0400 Subject: [PATCH 198/299] scaffold: expanded v2 config checks --- lib/scaffold/start.js | 51 ++++++++++++++++++++++++++++++++----- test/scaffold/start.unit.js | 29 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index f2ee95a8..e052ca11 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -10,6 +10,48 @@ var shuttingDown = false; log.debug = function() {}; +/** + * Checks for configuration options from version 2. This includes an "address" and + * "db" service, or having "datadir" at the root of the config. + */ +function checkConfigVersion2(fullConfig) { + var datadirUndefined = _.isUndefined(fullConfig.datadir); + var addressDefined = (fullConfig.services.indexOf('address') >= 0); + var dbDefined = (fullConfig.services.indexOf('db') >= 0); + + if (!datadirUndefined || addressDefined || dbDefined) { + + console.warn('\nConfiguration file is not compatible with this version. \n' + + 'A reindex for bitcoind is necessary for this upgrade with bitcoin.conf option "reindex=1". \n' + + 'There are changes necessary in both bitcoin.conf and bitcore-node.json.' + + 'To upgrade please see the details below and documentation at: \n' + + 'https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md \n'); + + if (!datadirUndefined) { + console.warn('Please remove "datadir" and add it to the config at ' + fullConfig.path + ' with:'); + var missingConfig = { + servicesConfig: { + bitcoind: { + spawn: { + datadir: fullConfig.datadir, + exec: path.resolve(__dirname, '../../bin/bitcoind') + } + } + } + }; + console.warn(JSON.stringify(missingConfig, null, 2) + '\n'); + } + + if (addressDefined || dbDefined) { + console.warn('Please remove "address" and/or "db" from "services" in: ' + fullConfig.path + '\n'); + } + + return true; + } + + return false; +} + /** * This function will instantiate and start a Node, requiring the necessary service * modules, and registering event handlers. @@ -37,12 +79,8 @@ function start(options) { fullConfig.path = path.resolve(options.path, './bitcore-node.json'); - if (fullConfig.datadir) { - throw new TypeError( - 'Configuration file (' + fullConfig.path + ') is not compatible with this version.' + - ' Please see https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md' + - ' for upgrade details.' - ); + if (checkConfigVersion2(fullConfig)) { + process.exit(1); } fullConfig.services = start.setupServices(require, servicesPath, options.config); @@ -215,3 +253,4 @@ module.exports.registerExitHandlers = registerExitHandlers; module.exports.exitHandler = exitHandler; module.exports.setupServices = setupServices; module.exports.cleanShutdown = cleanShutdown; +module.exports.checkConfigVersion2 = checkConfigVersion2; diff --git a/test/scaffold/start.unit.js b/test/scaffold/start.unit.js index 38d14feb..efc06299 100644 --- a/test/scaffold/start.unit.js +++ b/test/scaffold/start.unit.js @@ -8,6 +8,35 @@ var proxyquire = require('proxyquire'); var start = require('../../lib/scaffold/start'); describe('#start', function() { + describe('#checkConfigVersion2', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(console, 'warn'); + }); + afterEach(function() { + sandbox.restore(); + }); + it('will give true with "datadir" at root', function() { + var checkConfigVersion2 = proxyquire('../../lib/scaffold/start', {}).checkConfigVersion2; + var v2 = checkConfigVersion2({datadir: '/home/user/.bitcore/data', services: []}); + v2.should.equal(true); + }); + it('will give true with "address" service enabled', function() { + var checkConfigVersion2 = proxyquire('../../lib/scaffold/start', {}).checkConfigVersion2; + var v2 = checkConfigVersion2({services: ['address']}); + v2.should.equal(true); + }); + it('will give true with "db" service enabled', function() { + var checkConfigVersion2 = proxyquire('../../lib/scaffold/start', {}).checkConfigVersion2; + var v2 = checkConfigVersion2({services: ['db']}); + v2.should.equal(true); + }); + it('will give false without "datadir" at root and "address", "db" services disabled', function() { + var checkConfigVersion2 = proxyquire('../../lib/scaffold/start', {}).checkConfigVersion2; + var v2 = checkConfigVersion2({services: []}); + v2.should.equal(false); + }); + }); describe('#setupServices', function() { var cwd = process.cwd(); var setupServices = proxyquire('../../lib/scaffold/start', {}).setupServices; From 4001a41d2daa5b8098a34730c3280ae4c87e073c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 12:53:17 -0400 Subject: [PATCH 199/299] docs: add additional node about reindexing --- docs/upgrade.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 19d88a53..c568554a 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -13,6 +13,8 @@ Indexes include *more information* and are now also *faster*. Because of this a - `-timestampindex` - `-spentindex` +To start reindexing add `reindex=1` during the **first startup only**. + ### Configuration Options - The `bitcoin.conf` file in will need to be updated to include additional indexes *(see below)*. From bce64d86e34b806719532cb8250d3d14c353a878 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 13:02:52 -0400 Subject: [PATCH 200/299] scaffold: upgrade message formatting --- lib/scaffold/start.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index e052ca11..378e96c3 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -22,8 +22,8 @@ function checkConfigVersion2(fullConfig) { if (!datadirUndefined || addressDefined || dbDefined) { console.warn('\nConfiguration file is not compatible with this version. \n' + - 'A reindex for bitcoind is necessary for this upgrade with bitcoin.conf option "reindex=1". \n' + - 'There are changes necessary in both bitcoin.conf and bitcore-node.json.' + + 'A reindex for bitcoind is necessary for this upgrade with the "reindex=1" bitcoin.conf option. \n' + + 'There are changes necessary in both bitcoin.conf and bitcore-node.json. \n\n' + 'To upgrade please see the details below and documentation at: \n' + 'https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md \n'); From 584dd2cb98230991539de2484bf779429ecf5d8c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 13:46:03 -0400 Subject: [PATCH 201/299] test: add unit test for node getNetworkName --- test/node.unit.js | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/test/node.unit.js b/test/node.unit.js index 192cb992..94d22c3c 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -12,9 +12,7 @@ var log = index.log; describe('Bitcore Node', function() { - var baseConfig = { - datadir: 'testdir' - }; + var baseConfig = {}; var Node; @@ -317,6 +315,30 @@ describe('Bitcore Node', function() { }); }); + describe('#getNetworkName', function() { + afterEach(function() { + bitcore.Networks.disableRegtest(); + }); + it('it will return the network name for livenet', function() { + var node = new Node(baseConfig); + node.getNetworkName().should.equal('livenet'); + }); + it('it will return the network name for testnet', function() { + var baseConfig = { + network: 'testnet' + }; + var node = new Node(baseConfig); + node.getNetworkName().should.equal('testnet'); + }); + it('it will return the network for regtest', function() { + var baseConfig = { + network: 'regtest' + }; + var node = new Node(baseConfig); + node.getNetworkName().should.equal('regtest'); + }); + }); + describe('#stop', function() { var sandbox = sinon.sandbox.create(); beforeEach(function() { From 202971ec0cdd5a6b1ba545251bb89219250aadd7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 15:44:33 -0400 Subject: [PATCH 202/299] test: increase test coverage for bitcoind adds tests for subscribing with addresses --- test/services/bitcoind.unit.js | 111 +++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 0ca30418..6f29edb5 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1,5 +1,7 @@ 'use strict'; +/* jshint sub: true */ + var path = require('path'); var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); @@ -190,6 +192,115 @@ describe('Bitcoin Service', function() { }); }); + describe('#subscribeAddress', function() { + it('will not an invalid address', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter = new EventEmitter(); + bitcoind.subscribeAddress(emitter, ['invalidaddress']); + should.not.exist(bitcoind.subscriptions.address['invalidaddress']); + }); + it('will add a valid address', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter = new EventEmitter(); + bitcoind.subscribeAddress(emitter, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + should.exist(bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + }); + it('will handle multiple address subscribers', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscribeAddress(emitter2, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + should.exist(bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(2); + }); + it('will not add the same emitter twice', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + bitcoind.subscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + should.exist(bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + }); + }); + + describe('#unsubscribeAddress', function() { + it('it will remove a subscription', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscribeAddress(emitter2, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + should.exist(bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(2); + bitcoind.unsubscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + }); + it('will unsubscribe subscriptions for an emitter', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter1, emitter2]; + bitcoind.unsubscribeAddress(emitter1); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + }); + it('will NOT unsubscribe subscription with missing address', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter1, emitter2]; + bitcoind.unsubscribeAddress(emitter1, ['1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(2); + }); + it('will NOT unsubscribe subscription with missing emitter', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter2]; + bitcoind.unsubscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'][0].should.equal(emitter2); + }); + it('will remove empty addresses', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter1, emitter2]; + bitcoind.unsubscribeAddress(emitter1, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + bitcoind.unsubscribeAddress(emitter2, ['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + should.not.exist(bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br']); + }); + it('will unsubscribe emitter for all addresses', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter1, emitter2]; + bitcoind.subscriptions.address['1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'] = [emitter1, emitter2]; + sinon.spy(bitcoind, 'unsubscribeAddressAll'); + bitcoind.unsubscribeAddress(emitter1); + bitcoind.unsubscribeAddressAll.callCount.should.equal(1); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + bitcoind.subscriptions.address['1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'].length.should.equal(1); + }); + }); + + describe('#unsubscribeAddressAll', function() { + it('will unsubscribe emitter for all addresses', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = new EventEmitter(); + var emitter2 = new EventEmitter(); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'] = [emitter1, emitter2]; + bitcoind.subscriptions.address['1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'] = [emitter1, emitter2]; + bitcoind.subscriptions.address['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] = [emitter2]; + bitcoind.subscriptions.address['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou'] = [emitter1]; + bitcoind.unsubscribeAddress(emitter1); + bitcoind.subscriptions.address['2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'].length.should.equal(1); + bitcoind.subscriptions.address['1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'].length.should.equal(1); + bitcoind.subscriptions.address['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].length.should.equal(1); + should.not.exist(bitcoind.subscriptions.address['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou']); + }); + }); + describe('#_getDefaultConfig', function() { it('will generate config file from defaults', function() { var bitcoind = new BitcoinService(baseConfig); From a4888e535471f35672e8feb50d4b7c7c8daacdd2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 15:45:13 -0400 Subject: [PATCH 203/299] test: increase test coverage for lib/node.js --- test/node.unit.js | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/test/node.unit.js b/test/node.unit.js index 94d22c3c..dc7b4a3e 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -34,7 +34,6 @@ describe('Bitcore Node', function() { }); it('will set properties', function() { var config = { - datadir: 'testdir', services: [ { name: 'test1', @@ -49,11 +48,15 @@ describe('Bitcore Node', function() { node._unloadedServices[0].name.should.equal('test1'); node._unloadedServices[0].module.should.equal(TestService); node.network.should.equal(Networks.defaultNetwork); + var node2 = TestNode(config); + node2._unloadedServices.length.should.equal(1); + node2._unloadedServices[0].name.should.equal('test1'); + node2._unloadedServices[0].module.should.equal(TestService); + node2.network.should.equal(Networks.defaultNetwork); }); it('will set network to testnet', function() { var config = { network: 'testnet', - datadir: 'testdir', services: [ { name: 'test1', @@ -69,7 +72,6 @@ describe('Bitcore Node', function() { it('will set network to regtest', function() { var config = { network: 'regtest', - datadir: 'testdir', services: [ { name: 'test1', @@ -84,6 +86,26 @@ describe('Bitcore Node', function() { should.exist(regtest); node.network.should.equal(regtest); }); + it('will be able to disable log formatting', function() { + var config = { + network: 'regtest', + services: [ + { + name: 'test1', + module: TestService + } + ], + formatLogs: false + }; + var TestNode = proxyquire('../lib/node', {}); + var node = new TestNode(config); + node.log.formatting.should.equal(false); + + var TestNode = proxyquire('../lib/node', {}); + config.formatLogs = true; + var node2 = new TestNode(config); + node2.log.formatting.should.equal(true); + }); }); describe('#openBus', function() { @@ -92,6 +114,11 @@ describe('Bitcore Node', function() { var bus = node.openBus(); bus.node.should.equal(node); }); + it('will use remoteAddress config option', function() { + var node = new Node(baseConfig); + var bus = node.openBus({remoteAddress: '127.0.0.1'}); + bus.remoteAddress.should.equal('127.0.0.1'); + }); }); describe('#getAllAPIMethods', function() { @@ -183,7 +210,8 @@ describe('Bitcore Node', function() { function TestService() {} util.inherits(TestService, BaseService); TestService.prototype.start = sinon.stub().callsArg(0); - TestService.prototype.getData = function() {}; + var getData = sinon.stub(); + TestService.prototype.getData = getData; TestService.prototype.getAPIMethods = function() { return [ ['getData', this, this.getData, 1] @@ -201,6 +229,8 @@ describe('Bitcore Node', function() { TestService.prototype.start.callCount.should.equal(1); should.exist(node.services.testservice); should.exist(node.getData); + node.getData(); + getData.callCount.should.equal(1); }); }); it('will give an error from start', function() { From 85a302ee9d811ca0fb9844bd08b4b900a6a383ef Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 15:57:38 -0400 Subject: [PATCH 204/299] test: unit tests for zmq socket setup --- test/services/bitcoind.unit.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 6f29edb5..68306bcc 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1097,6 +1097,28 @@ describe('Bitcoin Service', function() { }); describe('#_initZmqSubSocket', function() { + it('will setup zmq socket', function() { + var socket = new EventEmitter(); + socket.monitor = sinon.stub(); + socket.connect = sinon.stub(); + var socketFunc = function() { + return socket; + }; + var BitcoinService = proxyquire('../../lib/services/bitcoind', { + zmq: { + socket: socketFunc + } + }); + var bitcoind = new BitcoinService(baseConfig); + var node = {}; + bitcoind._initZmqSubSocket(node, 'url'); + node.zmqSubSocket.should.equal(socket); + socket.connect.callCount.should.equal(1); + socket.connect.args[0][0].should.equal('url'); + socket.monitor.callCount.should.equal(1); + socket.monitor.args[0][0].should.equal(500); + socket.monitor.args[0][1].should.equal(0); + }); }); describe('#_checkReindex', function() { From f1a9f6d06685464ee533d6b5ea0530ee310d5cd3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 15:58:54 -0400 Subject: [PATCH 205/299] test: stub logging in bitcoind tests --- test/services/bitcoind.unit.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 68306bcc..59964ed8 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -193,6 +193,13 @@ describe('Bitcoin Service', function() { }); describe('#subscribeAddress', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will not an invalid address', function() { var bitcoind = new BitcoinService(baseConfig); var emitter = new EventEmitter(); @@ -225,6 +232,13 @@ describe('Bitcoin Service', function() { }); describe('#unsubscribeAddress', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('it will remove a subscription', function() { var bitcoind = new BitcoinService(baseConfig); var emitter1 = new EventEmitter(); @@ -285,6 +299,13 @@ describe('Bitcoin Service', function() { }); describe('#unsubscribeAddressAll', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will unsubscribe emitter for all addresses', function() { var bitcoind = new BitcoinService(baseConfig); var emitter1 = new EventEmitter(); From 2a53aad34a54ed6dbb5c4a5407620200c9f1fc0c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 16:19:18 -0400 Subject: [PATCH 206/299] test: add test for respawn bitcoind --- test/services/bitcoind.unit.js | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 59964ed8..77060357 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1376,6 +1376,7 @@ describe('Bitcoin Service', function() { var sandbox = sinon.sandbox.create(); beforeEach(function() { sandbox.stub(log, 'info'); + sandbox.stub(log, 'warn'); }); afterEach(function() { sandbox.restore(); @@ -1440,6 +1441,44 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will respawn bitcoind spawned process', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'bitcoind'; + bitcoind.spawn.datadir = '/tmp/bitcoin'; + bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.config = {}; + bitcoind.spawnRestartTime = 1; + bitcoind._loadTipFromNode = sinon.stub().callsArg(1); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._checkReindex = sinon.stub().callsArg(1); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); + bitcoind._stopSpawnedBitcoin = sinon.stub().callsArg(0); + sinon.spy(bitcoind, '_spawnChildProcess'); + bitcoind._spawnChildProcess(function(err) { + if (err) { + return done(err); + } + process.once('exit', function() { + setTimeout(function() { + bitcoind._spawnChildProcess.callCount.should.equal(2); + done(); + }, 5); + }); + process.emit('exit', 1); + }); + }); it('will give error after 60 retries', function(done) { var process = new EventEmitter(); var spawn = sinon.stub().returns(process); From ea3c813d51b110ede7cba2daae69029dac2b8056 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 16:47:57 -0400 Subject: [PATCH 207/299] test: check that caching is working --- test/services/bitcoind.unit.js | 48 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 77060357..d761fdcb 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1775,14 +1775,15 @@ describe('Bitcoin Service', function() { }); it('will give balance', function(done) { var bitcoind = new BitcoinService(baseConfig); + var getAddressBalance = sinon.stub().callsArgWith(1, null, { + result: { + received: 100000, + balance: 10000 + } + }); bitcoind.nodes.push({ client: { - getAddressBalance: sinon.stub().callsArgWith(1, null, { - result: { - received: 100000, - balance: 10000 - } - }) + getAddressBalance: getAddressBalance } }); var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; @@ -1793,7 +1794,15 @@ describe('Bitcoin Service', function() { } data.balance.should.equal(10000); data.received.should.equal(100000); - done(); + bitcoind.getAddressBalance(address, options, function(err, data2) { + if (err) { + return done(err); + } + data2.balance.should.equal(10000); + data2.received.should.equal(100000); + getAddressBalance.callCount.should.equal(1); + done(); + }); }); }); }); @@ -3538,15 +3547,17 @@ describe('Bitcoin Service', function() { }); it('should give a transaction with all properties', function(done) { var bitcoind = new BitcoinService(baseConfig); + var getRawTransaction = sinon.stub().callsArgWith(2, null, { + result: rpcRawTransaction + }); bitcoind.nodes.push({ client: { - getRawTransaction: sinon.stub().callsArgWith(2, null, { - result: rpcRawTransaction - }) + getRawTransaction: getRawTransaction } }); var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; - bitcoind.getDetailedTransaction(txid, function(err, tx) { + function checkTx(tx) { + /* jshint maxstatements: 30 */ should.exist(tx); should.not.exist(tx.coinbase); should.equal(tx.hex, txBuffer.toString('hex')); @@ -3575,7 +3586,20 @@ describe('Bitcoin Service', function() { should.equal(output.spentTxId, '4316b98e7504073acd19308b4b8c9f4eeb5e811455c54c0ebfe276c0b1eb6315'); should.equal(output.spentIndex, 2); should.equal(output.spentHeight, 100); - done(); + } + bitcoind.getDetailedTransaction(txid, function(err, tx) { + if (err) { + return done(err); + } + checkTx(tx); + bitcoind.getDetailedTransaction(txid, function(err, tx) { + if (err) { + return done(err); + } + checkTx(tx); + getRawTransaction.callCount.should.equal(1); + done(); + }); }); }); it('should set coinbase to true', function(done) { From 0a95765e51a73b378ca55588fd78acfe97f2bd4f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 May 2016 16:48:17 -0400 Subject: [PATCH 208/299] bitcoind: fix indentation --- lib/services/bitcoind.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index ff10c390..e882839a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1433,9 +1433,9 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var paginatedTxids; try { paginatedTxids = self._paginateTxids(allTxids, fromArg, toArg); - } catch(e) { - return callback(e); - } + } catch(e) { + return callback(e); + } var allSummary = _.clone(summary); allSummary.txids = paginatedTxids; From 35a1b6dd04e3e55597c24ed35e65307ca1f6a1ba Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 10:21:28 -0400 Subject: [PATCH 209/299] test: more coverage for bitcoind service tests for catching errors in #_initChain --- test/services/bitcoind.unit.js | 82 +++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index d761fdcb..bc8980ad 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -190,6 +190,15 @@ describe('Bitcoin Service', function() { bitcoind.subscriptions.hashblock[2].should.equal(emitter4); bitcoind.subscriptions.hashblock[3].should.equal(emitter5); }); + it('will not remove item an already unsubscribed item', function() { + var bitcoind = new BitcoinService(baseConfig); + var emitter1 = {}; + var emitter3 = {}; + bitcoind.subscriptions.hashblock= [emitter1]; + bitcoind.unsubscribe('hashblock', emitter3); + bitcoind.subscriptions.hashblock.length.should.equal(1); + bitcoind.subscriptions.hashblock[0].should.equal(emitter1); + }); }); describe('#subscribeAddress', function() { @@ -561,6 +570,77 @@ describe('Bitcoin Service', function() { done(); }); }); + it('it will handle error from getBestBlockHash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, {code: -1, message: 'error'}); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash + } + }); + bitcoind._initChain(function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); + it('it will handle error from getBlock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, {}); + var getBlock = sinon.stub().callsArgWith(1, {code: -1, message: 'error'}); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash, + getBlock: getBlock + } + }); + bitcoind._initChain(function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); + it('it will handle error from getBlockHash', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, {}); + var getBlock = sinon.stub().callsArgWith(1, null, { + result: { + height: 10 + } + }); + var getBlockHash = sinon.stub().callsArgWith(1, {code: -1, message: 'error'}); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash, + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind._initChain(function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); + it('it will handle error from getRawBlock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, {}); + var getBlock = sinon.stub().callsArgWith(1, null, { + result: { + height: 10 + } + }); + var getBlockHash = sinon.stub().callsArgWith(1, null, {}); + bitcoind.nodes.push({ + client: { + getBestBlockHash: getBestBlockHash, + getBlock: getBlock, + getBlockHash: getBlockHash + } + }); + bitcoind.getRawBlock = sinon.stub().callsArgWith(1, new Error('test')); + bitcoind._initChain(function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); }); describe('#_getDefaultConf', function() { @@ -1780,7 +1860,7 @@ describe('Bitcoin Service', function() { received: 100000, balance: 10000 } - }); + }); bitcoind.nodes.push({ client: { getAddressBalance: getAddressBalance From 3fef6f5ffc071c91f59575802b4a225b9d3ac9d7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 13:07:51 -0400 Subject: [PATCH 210/299] test: increase test coverage of bitcoind service --- lib/services/bitcoind.js | 42 ++++----- test/services/bitcoind.unit.js | 157 ++++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 27 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index e882839a..7dcd8dc8 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1338,12 +1338,8 @@ Bitcoin.prototype._paginateTxids = function(fullTxids, fromArg, toArg) { var txids; var from = parseInt(fromArg); var to = parseInt(toArg); - if (from >= 0 && to >= 0) { - $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); - txids = fullTxids.slice(from, to); - } else { - txids = fullTxids; - } + $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); + txids = fullTxids.slice(from, to); return txids; }; @@ -1363,7 +1359,10 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; var addressStrings = this._getAddressStrings(addresses); - if ((options.to - options.from) > self.maxTransactionHistory) { + var fromArg = parseInt(options.from || 0); + var toArg = parseInt(options.to || self.maxTransactionHistory); + + if ((toArg - fromArg) > self.maxTransactionHistory) { return callback(new Error( '"from" (' + options.from + ') and "to" (' + options.to + ') range should be less than or equal to ' + self.maxTransactionHistory @@ -1377,7 +1376,7 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { var totalCount = txids.length; try { - txids = self._paginateTxids(txids, options.from, options.to); + txids = self._paginateTxids(txids, fromArg, toArg); } catch(e) { return callback(e); } @@ -1424,12 +1423,12 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { var fromArg = parseInt(options.from || 0); var toArg = parseInt(options.to || self.maxTxids); - if ((toArg - fromArg) > self.maxTxids) { - return callback(new Error( - '"from" (' + fromArg + ') and "to" (' + toArg + ') range should be less than or equal to ' + - self.maxTxids - )); - } + if ((toArg - fromArg) > self.maxTxids) { + return callback(new Error( + '"from" (' + fromArg + ') and "to" (' + toArg + ') range should be less than or equal to ' + + self.maxTxids + )); + } var paginatedTxids; try { paginatedTxids = self._paginateTxids(allTxids, fromArg, toArg); @@ -1751,19 +1750,13 @@ Bitcoin.prototype.estimateFee = function(blocks, callback) { Bitcoin.prototype.sendTransaction = function(tx, options, callback) { var self = this; var allowAbsurdFees = false; - var txString; - if (tx instanceof Transaction) { - txString = tx.serialize(); - } else { - txString = tx; - } if (_.isFunction(options) && _.isUndefined(callback)) { callback = options; } else if (_.isObject(options)) { allowAbsurdFees = options.allowAbsurdFees; } - this.client.sendRawTransaction(txString, allowAbsurdFees, function(err, response) { + this.client.sendRawTransaction(tx, allowAbsurdFees, function(err, response) { if (err) { return callback(self._wrapRPCError(err)); } @@ -1880,14 +1873,13 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { if (!tx.coinbase) { tx.inputSatoshis += input.valueSat; } - var script; - var scriptAsm; + var script = null; + var scriptAsm = null; if (input.scriptSig) { script = input.scriptSig.hex; scriptAsm = input.scriptSig.asm; } else if (input.coinbase) { script = input.coinbase; - scriptAsm = null; } tx.inputs.push({ prevTxId: input.txid || null, @@ -1908,7 +1900,7 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { var out = result.vout[outputIndex]; tx.outputSatoshis += out.valueSat; var address = null; - if (out.scriptPubKey && out.scriptPubKey.addresses && out.scriptPubKey.addresses.length > 0) { + if (out.scriptPubKey.addresses.length === 1) { address = out.scriptPubKey.addresses[0]; } tx.outputs.push({ diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index bc8980ad..cde71c63 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -944,6 +944,35 @@ describe('Bitcoin Service', function() { bitcoind._updateTip(node, message); bitcoind._updateTip(node, message); }); + it('will not call syncPercentage if node is stopping', function(done) { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind.syncPercentage = sinon.stub(); + bitcoind._resetCaches = sinon.stub(); + bitcoind.node.stopping = true; + var node = { + client: { + getBlock: sinon.stub().callsArgWith(1, null, { + result: { + height: 10 + } + }) + } + }; + bitcoind.on('tip', function() { + bitcoind.syncPercentage.callCount.should.equal(0); + done(); + }); + bitcoind._updateTip(node, message); + }); }); describe('#_getAddressesFromTransaction', function() { @@ -1268,6 +1297,16 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will call callback if reindex is not enabled', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var node = { + _reindex: false + }; + bitcoind._checkReindex(node, function() { + node._reindex.should.equal(false); + done(); + }); + }); }); describe('#_loadTipFromNode', function() { @@ -2078,7 +2117,7 @@ describe('Bitcoin Service', function() { }); }); - describe('#_getTxidsMempool', function() { + describe('#_getTxidsFromMempool', function() { it('will filter to txids', function() { var bitcoind = new BitcoinService(baseConfig); var deltas = [ @@ -2098,6 +2137,24 @@ describe('Bitcoin Service', function() { txids[1].should.equal('txid1'); txids[2].should.equal('txid2'); }); + it('will not include duplicates', function() { + var bitcoind = new BitcoinService(baseConfig); + var deltas = [ + { + txid: 'txid0', + }, + { + txid: 'txid0', + }, + { + txid: 'txid1', + } + ]; + var txids = bitcoind._getTxidsFromMempool(deltas); + txids.length.should.equal(2); + txids[0].should.equal('txid0'); + txids[1].should.equal('txid1'); + }); }); describe('#_getHeightRangeQuery', function() { @@ -2332,6 +2389,14 @@ describe('Bitcoin Service', function() { afterEach(function() { sandbox.restore(); }); + it('should get 0 confirmation', function() { + var tx = new Transaction(txhex); + tx.height = -1; + var bitcoind = new BitcoinService(baseConfig); + bitcoind.height = 10; + var confirmations = bitcoind._getConfirmationsDetail(tx); + confirmations.should.equal(0); + }); it('should get 1 confirmation', function() { var tx = new Transaction(txhex); tx.height = 10; @@ -3420,6 +3485,21 @@ describe('Bitcoin Service', function() { hash.should.equal(tx.hash); }); }); + it('missing callback will throw error', function() { + var bitcoind = new BitcoinService(baseConfig); + var sendRawTransaction = sinon.stub().callsArgWith(2, null, { + result: tx.hash + }); + bitcoind.nodes.push({ + client: { + sendRawTransaction: sendRawTransaction + } + }); + var transaction = bitcore.Transaction(); + (function() { + bitcoind.sendTransaction(transaction); + }).should.throw(Error); + }); }); describe('#getRawTransaction', function() { @@ -3684,7 +3764,7 @@ describe('Bitcoin Service', function() { }); it('should set coinbase to true', function(done) { var bitcoind = new BitcoinService(baseConfig); - var rawTransaction = _.clone(rpcRawTransaction); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); delete rawTransaction.vin[0]; rawTransaction.vin = [ { @@ -3705,6 +3785,79 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will not include address if address length is zero', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); + rawTransaction.vout[0].scriptPubKey.addresses = []; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.equal(tx.outputs[0].address, null); + done(); + }); + }); + it('will not include address if address length is greater than 1', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); + rawTransaction.vout[0].scriptPubKey.addresses = ['one', 'two']; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.equal(tx.outputs[0].address, null); + done(); + }); + }); + it('will not include script if input missing scriptSig or coinbase', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); + delete rawTransaction.vin[0].scriptSig; + delete rawTransaction.vin[0].coinbase; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.equal(tx.inputs[0].script, null); + done(); + }); + }); + it('will set height to -1 if missing height', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); + delete rawTransaction.height; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.equal(tx.height, -1); + done(); + }); + }); }); describe('#getBestBlockHash', function() { From 9c90f05c736dd3c9921b082b2353d3cbe0cdaa34 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 13:35:08 -0400 Subject: [PATCH 211/299] test: more coverage for bitcoind --- test/services/bitcoind.unit.js | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index cde71c63..39cf7e97 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1176,6 +1176,61 @@ describe('Bitcoin Service', function() { done(); }, 200); }); + it('it will clear interval if node is stopping', function(done) { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + var getBestBlockHash = sinon.stub().callsArgWith(0, {code: -1, message: 'error'}); + var node = { + _tipUpdateInterval: 1, + client: { + getBestBlockHash: getBestBlockHash + } + }; + bitcoind._checkSyncedAndSubscribeZmqEvents(node); + setTimeout(function() { + bitcoind.node.stopping = true; + var count = getBestBlockHash.callCount; + setTimeout(function() { + getBestBlockHash.callCount.should.equal(count); + done(); + }, 100); + }, 100); + }); + it('will not set interval if synced is true', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._updateTip = sinon.stub(); + bitcoind._subscribeZmqEvents = sinon.stub(); + var getBestBlockHash = sinon.stub().callsArgWith(0, null, { + result: '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45' + }); + var info = { + result: { + verificationprogress: 1.00 + } + }; + var getBlockchainInfo = sinon.stub().callsArgWith(0, null, info); + var node = { + _tipUpdateInterval: 1, + client: { + getBestBlockHash: getBestBlockHash, + getBlockchainInfo: getBlockchainInfo + } + }; + bitcoind._checkSyncedAndSubscribeZmqEvents(node); + setTimeout(function() { + getBestBlockHash.callCount.should.equal(1); + getBlockchainInfo.callCount.should.equal(1); + done(); + }, 200); + }); }); describe('#_subscribeZmqEvents', function() { @@ -1224,6 +1279,24 @@ describe('Bitcoin Service', function() { var message = new Buffer('abcdef', 'hex'); node.zmqSubSocket.emit('message', topic, message); }); + it('will ignore unknown topic types', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._zmqBlockHandler = sinon.stub(); + bitcoind._zmqTransactionHandler = sinon.stub(); + var node = { + zmqSubSocket: new EventEmitter() + }; + node.zmqSubSocket.subscribe = sinon.stub(); + bitcoind._subscribeZmqEvents(node); + node.zmqSubSocket.on('message', function() { + bitcoind._zmqBlockHandler.callCount.should.equal(0); + bitcoind._zmqTransactionHandler.callCount.should.equal(0); + done(); + }); + var topic = new Buffer('unknown', 'utf8'); + var message = new Buffer('abcdef', 'hex'); + node.zmqSubSocket.emit('message', topic, message); + }); }); describe('#_initZmqSubSocket', function() { From 4d1b853fd45d5ce6ea1441cb982a68ee74182012 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 13:36:04 -0400 Subject: [PATCH 212/299] test: increase timeout for before all in node address regtest --- regtest/node.js | 1 + 1 file changed, 1 insertion(+) diff --git a/regtest/node.js b/regtest/node.js index 17d316f5..2e8539ab 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -147,6 +147,7 @@ describe('Node Functionality', function() { var address; var unspentOutput; before(function(done) { + this.timeout(10000); address = testKey.toAddress(regtest).toString(); var startHeight = node.services.bitcoind.height; node.services.bitcoind.on('tip', function(height) { From 86b1acd0bed8bb21aad573204e653dd8b3abc6ed Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 14:06:03 -0400 Subject: [PATCH 213/299] test: coverage for bitcoind getAddressUnspentOutputs --- lib/services/bitcoind.js | 6 +- test/services/bitcoind.unit.js | 181 +++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 7dcd8dc8..69063d3f 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1064,15 +1064,15 @@ Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callb for (var i = 0; i < mempoolDeltas.length; i++) { var delta = mempoolDeltas[i]; - if (delta.satoshis > 0) { - mempoolUnspentOutputs.push(transformUnspentOutput(delta)); - } else if (delta.satoshis < 0) { + if (delta.prevtxid && delta.satoshis <= 0) { if (!spentOutputs[delta.prevtxid]) { spentOutputs[delta.prevtxid] = [delta.prevout]; } else { spentOutputs[delta.prevtxid].push(delta.prevout); } isSpentOutputs = true; + } else { + mempoolUnspentOutputs.push(transformUnspentOutput(delta)); } } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 39cf7e97..2bc8fa4e 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2169,6 +2169,187 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will update with mempool results with multiple outputs', function(done) { + var deltas = [ + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 1 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 2 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + }, + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 2, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(0); + done(); + }); + }); + it('will update with mempool results spending zero value output (likely never to happen)', function(done) { + var deltas = [ + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: 0, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 1 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 0, + height: 207111 + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(0); + done(); + }); + }); + it('will not filter results if mempool is not spending', function(done) { + var deltas = [ + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: 10000, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 0, + height: 207111 + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(2); + done(); + }); + }); + it('it will handle error from getAddressMempool', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.nodes.push({ + client: { + getAddressMempool: sinon.stub().callsArgWith(1, {code: -1, message: 'test'}) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err) { + err.should.be.instanceOf(Error); + done(); + }); + }); + it('should set query mempool if undefined', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getAddressMempool = sinon.stub().callsArgWith(1, {code: -1, message: 'test'}); + bitcoind.nodes.push({ + client: { + getAddressMempool: getAddressMempool + } + }); + var options = {}; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err) { + getAddressMempool.callCount.should.equal(1); + done(); + }); + }); }); describe('#_getBalanceFromMempool', function() { From 0c820c5987d92d0399e2661063f99035d1223b90 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 14:30:54 -0400 Subject: [PATCH 214/299] test: unit tests for bitcoind address details --- test/services/bitcoind.unit.js | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 2bc8fa4e..2706cd0f 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2445,6 +2445,15 @@ describe('Bitcoin Service', function() { }); describe('#getAddressTxids', function() { + it('will give error from _getHeightRangeQuery', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._getHeightRangeQuery = sinon.stub().throws(new Error('test')); + bitcoind.getAddressTxids('address', {}, function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + done(); + }); + }); it('will give rpc error from mempool query', function() { var bitcoind = new BitcoinService(baseConfig); bitcoind.nodes.push({ @@ -2686,6 +2695,98 @@ describe('Bitcoin Service', function() { }); }); + describe('#_getAddressDetailsForInput', function() { + it('will return if missing an address', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + bitcoind._getAddressDetailsForInput({}, 0, result, []); + should.not.exist(result.addresses); + should.not.exist(result.satoshis); + }); + it('will only add address if it matches', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + bitcoind._getAddressDetailsForInput({ + address: 'address1' + }, 0, result, ['address2']); + should.not.exist(result.addresses); + should.not.exist(result.satoshis); + }); + it('will instantiate if outputIndexes not defined', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = { + addresses: {} + }; + bitcoind._getAddressDetailsForInput({ + address: 'address1' + }, 0, result, ['address1']); + should.exist(result.addresses); + result.addresses['address1'].inputIndexes.should.deep.equal([0]); + result.addresses['address1'].outputIndexes.should.deep.equal([]); + }); + it('will push to inputIndexes', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = { + addresses: { + 'address1': { + inputIndexes: [1] + } + } + }; + bitcoind._getAddressDetailsForInput({ + address: 'address1' + }, 2, result, ['address1']); + should.exist(result.addresses); + result.addresses['address1'].inputIndexes.should.deep.equal([1, 2]); + }); + }); + + describe('#_getAddressDetailsForOutput', function() { + it('will return if missing an address', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + bitcoind._getAddressDetailsForOutput({}, 0, result, []); + should.not.exist(result.addresses); + should.not.exist(result.satoshis); + }); + it('will only add address if it matches', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = {}; + bitcoind._getAddressDetailsForOutput({ + address: 'address1' + }, 0, result, ['address2']); + should.not.exist(result.addresses); + should.not.exist(result.satoshis); + }); + it('will instantiate if outputIndexes not defined', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = { + addresses: {} + }; + bitcoind._getAddressDetailsForOutput({ + address: 'address1' + }, 0, result, ['address1']); + should.exist(result.addresses); + result.addresses['address1'].inputIndexes.should.deep.equal([]); + result.addresses['address1'].outputIndexes.should.deep.equal([0]); + }); + it('will push if outputIndexes defined', function() { + var bitcoind = new BitcoinService(baseConfig); + var result = { + addresses: { + 'address1': { + outputIndexes: [0] + } + } + }; + bitcoind._getAddressDetailsForOutput({ + address: 'address1' + }, 1, result, ['address1']); + should.exist(result.addresses); + result.addresses['address1'].outputIndexes.should.deep.equal([0, 1]); + }); + }); + describe('#_getAddressDetailsForTransaction', function() { it('will calculate details for the transaction', function(done) { /* jshint sub:true */ From 8d7d78a89eb88e8c1558af0fef429ae1332e5c5c Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 14:41:57 -0400 Subject: [PATCH 215/299] build: run coveralls for unit test coverage --- .coveralls.yml | 1 + .travis.yml | 2 ++ package.json | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 00000000..07c49339 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: DvrDb09a8vhPlVf6DT4cGBjcFOi6DfZN1 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9d896336..fbc8be0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,3 +18,5 @@ script: - npm run regtest - npm run test - npm run jshint +after_success: + - npm run coveralls \ No newline at end of file diff --git a/package.json b/package.json index bbd99939..d84633a2 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "test": "mocha -R spec --recursive", "regtest": "./scripts/regtest", "jshint": "jshint --reporter=node_modules/jshint-stylish ./lib", - "coverage": "istanbul cover _mocha -- --recursive" + "coverage": "istanbul cover _mocha -- --recursive", + "coveralls": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- --recursive -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js" }, "tags": [ "bitcoin", @@ -69,6 +70,8 @@ "benchmark": "1.0.0", "bitcore-p2p": "^1.1.0", "chai": "^3.5.0", + "coveralls": "^2.11.9", + "istanbul": "^0.4.3", "jshint": "^2.9.2", "jshint-stylish": "^2.1.0", "mocha": "^2.4.5", From 52cf30085834c1bfdc81309c6c9a68ccecd4e85f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 15:13:41 -0400 Subject: [PATCH 216/299] test: coverage for bitcoind getAddressSummary --- test/services/bitcoind.unit.js | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 2706cd0f..39ef2296 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3198,6 +3198,54 @@ describe('Bitcoin Service', function() { }); }); }); + it('will skip querying the mempool with queryMempool set to false', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getAddressMempool = sinon.stub(); + bitcoind.nodes.push({ + client: { + getAddressMempool: getAddressMempool + } + }); + sinon.spy(bitcoind, '_paginateTxids'); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { + received: 30 * 1e8, + balance: 20 * 1e8 + }); + var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; + var options = { + queryMempool: false + }; + bitcoind.getAddressSummary(address, options, function() { + getAddressMempool.callCount.should.equal(0); + done(); + }); + }); + it('will give error from _paginateTxids', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getAddressMempool = sinon.stub(); + bitcoind.nodes.push({ + client: { + getAddressMempool: getAddressMempool + } + }); + sinon.spy(bitcoind, '_paginateTxids'); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, [txid1, txid2, txid3]); + bitcoind.getAddressBalance = sinon.stub().callsArgWith(2, null, { + received: 30 * 1e8, + balance: 20 * 1e8 + }); + bitcoind._paginateTxids = sinon.stub().throws(new Error('test')); + var address = '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj'; + var options = { + queryMempool: false + }; + bitcoind.getAddressSummary(address, options, function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + done(); + }); + }); }); describe('#getRawBlock', function() { From 0cb795d9802b8d240baf64dc85b8a9b7f7e17f38 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 16:22:41 -0400 Subject: [PATCH 217/299] test: add bitcoind test for early shutdown while connecting --- lib/services/bitcoind.js | 7 ++++++- test/services/bitcoind.unit.js | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 69063d3f..788066bb 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -886,10 +886,12 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { Bitcoin.prototype._connectProcess = function(config, callback) { var self = this; var node = {}; + var exitShutdown = false; async.retry({times: 60, interval: self.startRetryInterval}, function(done) { if (self.node.stopping) { - return done(new Error('Stopping while trying to connect to bitcoind.')); + exitShutdown = true; + return done(); } node.client = new BitcoinRPC({ @@ -906,6 +908,9 @@ Bitcoin.prototype._connectProcess = function(config, callback) { if (err) { return callback(err); } + if (exitShutdown) { + return callback(new Error('Stopping while trying to connect to bitcoind.')); + } self._initZmqSubSocket(node, config.zmqpubrawtx); self._subscribeZmqEvents(node); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 39ef2296..285a91f1 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1738,6 +1738,27 @@ describe('Bitcoin Service', function() { }); describe('#_connectProcess', function() { + it('will give error if connecting while shutting down', function(done) { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new BitcoinService(config); + bitcoind.node.stopping = true; + bitcoind.startRetryInterval = 100; + bitcoind._loadTipFromNode = sinon.stub(); + bitcoind._connectProcess({}, function(err) { + err.should.be.instanceof(Error); + err.message.should.match(/Stopping while trying to connect/); + bitcoind._loadTipFromNode.callCount.should.equal(0); + done(); + }); + }); it('will give error from loadTipFromNode after 60 retries', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind._loadTipFromNode = sinon.stub().callsArgWith(1, new Error('test')); From 1d9b89f18779decbaff3945f672fd497694db272 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 16:29:36 -0400 Subject: [PATCH 218/299] test: coverage for getAddressHistory --- test/services/bitcoind.unit.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 285a91f1..8dd46391 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2978,6 +2978,25 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will give error with "from" and "to" order is reversed', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, []); + bitcoind.getAddressHistory(address, {from: 51, to: 0}, function(err) { + should.exist(err); + err.message.match(/^\"from/); + done(); + }); + }); + it('will give error from _getAddressDetailedTransaction', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.getAddressTxids = sinon.stub().callsArgWith(2, null, ['txid']); + bitcoind._getAddressDetailedTransaction = sinon.stub().callsArgWith(2, new Error('test')); + bitcoind.getAddressHistory(address, {}, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); it('will give an error if length of addresses is too long', function(done) { var addresses = []; for (var i = 0; i < 101; i++) { From f76b20617831042c7bae68779d30803bc6a94e7a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 16:39:22 -0400 Subject: [PATCH 219/299] test: coverage for getBlockHeader --- lib/services/bitcoind.js | 25 ++++++------------------- test/services/bitcoind.unit.js | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 788066bb..c459fd2d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1681,10 +1681,13 @@ Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, callback) { * @param {String|Number} block - A block hash or block height * @param {Function} callback */ -Bitcoin.prototype.getBlockHeader = function(block, callback) { +Bitcoin.prototype.getBlockHeader = function(blockArg, callback) { var self = this; - function queryHeader(blockhash) { + function queryHeader(err, blockhash) { + if (err) { + return callback(err); + } self._tryAll(function(done) { self.client.getBlockHeader(blockhash, function(err, response) { if (err) { @@ -1711,23 +1714,7 @@ Bitcoin.prototype.getBlockHeader = function(block, callback) { }, callback); } - if (_.isNumber(block)) { - self._tryAll(function(done) { - self.client.getBlockHash(block, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - done(null, response.result); - }); - }, function(err, blockhash) { - if (err) { - return callback(err); - } - queryHeader(blockhash); - }); - } else { - queryHeader(block); - } + self._maybeGetBlockHash(blockArg, queryHeader); }; /** diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 8dd46391..d6b8f0cf 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3567,6 +3567,18 @@ describe('Bitcoin Service', function() { describe('#getBlockHeader', function() { var blockhash = '00000000050a6d07f583beba2d803296eb1e9d4980c4a20f206c584e89a4f02b'; + it('will give error from getBlockHash', function() { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind.getBlockHeader(10, function(err) { + err.should.be.instanceof(Error); + }); + }); it('it will give rpc error from client getblockheader', function() { var bitcoind = new BitcoinService(baseConfig); var getBlockHeader = sinon.stub().callsArgWith(1, {code: -1, message: 'Test error'}); @@ -3790,6 +3802,20 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will give error from client.getBlock', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlock = sinon.stub().callsArgWith(2, {code: -1, message: 'test'}); + bitcoind.nodes.push({ + client: { + getBlock: getBlock + } + }); + bitcoind.getBlockOverview(blockhash, function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + done(); + }); + }); it('will give expected result', function(done) { var bitcoind = new BitcoinService(baseConfig); var blockResult = { From 2dddf01bb08615c2dc4bdd5d08b57cf219a28750 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 17:05:27 -0400 Subject: [PATCH 220/299] test: coverage for spawnChildProcess --- lib/services/bitcoind.js | 8 ++- test/services/bitcoind.unit.js | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index c459fd2d..60010eb0 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -847,9 +847,12 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { } }); + var exitShutdown = false; + async.retry({times: 60, interval: self.startRetryInterval}, function(done) { if (self.node.stopping) { - return done(new Error('Stopping while trying to connect to bitcoind.')); + exitShutdown = true; + return done(); } node.client = new BitcoinRPC({ @@ -866,6 +869,9 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { if (err) { return callback(err); } + if (exitShutdown) { + return callback(new Error('Stopping while trying to spawn bitcoind.')); + } self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index d6b8f0cf..bd312b2a 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -1569,12 +1569,14 @@ describe('Bitcoin Service', function() { beforeEach(function() { sandbox.stub(log, 'info'); sandbox.stub(log, 'warn'); + sandbox.stub(log, 'error'); }); afterEach(function() { sandbox.restore(); }); it('will give error from spawn config', function(done) { var bitcoind = new BitcoinService(baseConfig); + bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind._loadSpawnConfiguration = sinon.stub().throws(new Error('test')); bitcoind._spawnChildProcess(function(err) { err.should.be.instanceof(Error); @@ -1582,6 +1584,45 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will give error from stopSpawnedBitcoin', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind._stopSpawnedBitcoin = sinon.stub().callsArgWith(0, new Error('test')); + bitcoind._spawnChildProcess(function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + }); + }); + it('will exit spawn if shutdown', function() { + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(config); + bitcoind.spawn = {}; + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind._stopSpawnedBitcoin = sinon.stub().callsArgWith(0, null); + bitcoind.node.stopping = true; + bitcoind._spawnChildProcess(function(err) { + err.should.be.instanceOf(Error); + err.message.should.match(/Stopping while trying to spawn/); + }); + }); it('will include network with spawn command and init zmq/rpc on node', function(done) { var process = new EventEmitter(); var spawn = sinon.stub().returns(process); @@ -1671,6 +1712,92 @@ describe('Bitcoin Service', function() { process.emit('exit', 1); }); }); + it('will emit error during respawn', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var bitcoind = new TestBitcoinService(baseConfig); + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'bitcoind'; + bitcoind.spawn.datadir = '/tmp/bitcoin'; + bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.config = {}; + bitcoind.spawnRestartTime = 1; + bitcoind._loadTipFromNode = sinon.stub().callsArg(1); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._checkReindex = sinon.stub().callsArg(1); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); + bitcoind._stopSpawnedBitcoin = sinon.stub().callsArg(0); + sinon.spy(bitcoind, '_spawnChildProcess'); + bitcoind._spawnChildProcess(function(err) { + if (err) { + return done(err); + } + bitcoind._spawnChildProcess = sinon.stub().callsArgWith(0, new Error('test')); + bitcoind.on('error', function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('test'); + done(); + }); + process.emit('exit', 1); + }); + }); + it('will NOT respawn bitcoind spawned process if shutting down', function(done) { + var process = new EventEmitter(); + var spawn = sinon.stub().returns(process); + var TestBitcoinService = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync + }, + child_process: { + spawn: spawn + } + }); + var config = { + node: { + network: bitcore.Networks.testnet + }, + spawn: { + datadir: 'testdir', + exec: 'testpath' + } + }; + var bitcoind = new TestBitcoinService(config); + bitcoind._loadSpawnConfiguration = sinon.stub(); + bitcoind.spawn = {}; + bitcoind.spawn.exec = 'bitcoind'; + bitcoind.spawn.datadir = '/tmp/bitcoin'; + bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.config = {}; + bitcoind.spawnRestartTime = 1; + bitcoind._loadTipFromNode = sinon.stub().callsArg(1); + bitcoind._initZmqSubSocket = sinon.stub(); + bitcoind._checkReindex = sinon.stub().callsArg(1); + bitcoind._checkSyncedAndSubscribeZmqEvents = sinon.stub(); + bitcoind._stopSpawnedBitcoin = sinon.stub().callsArg(0); + sinon.spy(bitcoind, '_spawnChildProcess'); + bitcoind._spawnChildProcess(function(err) { + if (err) { + return done(err); + } + bitcoind.node.stopping = true; + process.once('exit', function() { + setTimeout(function() { + bitcoind._spawnChildProcess.callCount.should.equal(1); + done(); + }, 5); + }); + process.emit('exit', 1); + }); + }); it('will give error after 60 retries', function(done) { var process = new EventEmitter(); var spawn = sinon.stub().returns(process); From 88c15f6844d7e7f8f97b2da43e78c92c4fc508fb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 May 2016 17:15:07 -0400 Subject: [PATCH 221/299] scaffold: remove no longer needed '-dev' version handling --- lib/scaffold/create.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index aa9a3084..c521602e 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -11,12 +11,7 @@ var mkdirp = require('mkdirp'); var fs = require('fs'); var defaultBaseConfig = require('./default-base-config'); -var version; -if (packageFile.version.match('-dev')) { - version = '^' + packageFile.lastBuild; -} else { - version = '^' + packageFile.version; -} +var version = '^' + packageFile.version; var BASE_PACKAGE = { description: 'A full Bitcoin node build with Bitcore', From f38fa1324fd480d23992d84fd90f4cb9ab8f4850 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 25 May 2016 17:14:22 -0400 Subject: [PATCH 222/299] bitcoind: account for scriptPubKey.addresses not always being set --- lib/services/bitcoind.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 60010eb0..10fccc47 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1898,7 +1898,7 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { var out = result.vout[outputIndex]; tx.outputSatoshis += out.valueSat; var address = null; - if (out.scriptPubKey.addresses.length === 1) { + if (out.scriptPubKey && out.scriptPubKey.addresses && out.scriptPubKey.addresses.length === 1) { address = out.scriptPubKey.addresses[0]; } tx.outputs.push({ From c7ec2dcc89339816bc1600db441729991c05871d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 26 May 2016 09:16:08 -0400 Subject: [PATCH 223/299] test: bitcoind test for undefined scriptPubKey.addresses --- test/services/bitcoind.unit.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index bd312b2a..40a0392d 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -4417,6 +4417,24 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will handle scriptPubKey.addresses not being set', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); + delete rawTransaction.vout[0].scriptPubKey['addresses']; + bitcoind.nodes.push({ + client: { + getRawTransaction: sinon.stub().callsArgWith(2, null, { + result: rawTransaction + }) + } + }); + var txid = '2d950d00494caf6bfc5fff2a3f839f0eb50f663ae85ce092bc5f9d45296ae91f'; + bitcoind.getDetailedTransaction(txid, function(err, tx) { + should.exist(tx); + should.equal(tx.outputs[0].address, null); + done(); + }); + }); it('will not include script if input missing scriptSig or coinbase', function(done) { var bitcoind = new BitcoinService(baseConfig); var rawTransaction = JSON.parse((JSON.stringify(rpcRawTransaction))); From e8a35e2bb51fed2a9f9c91a625cc14fd0ac69a43 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 26 May 2016 10:15:42 -0400 Subject: [PATCH 224/299] bitcoind: bug with getting block hash from address Fixes an issue where passing an address as the blockArg would get the blockhash for the parsed integer of the address. `parseInt` would parse the address as an integer and then get the block hash for 1. A regular expression now checks that the string is numeric with only 0-9 and the length is less than 40, the size of a ripemd160, and also less than the length of a sha256 hash. --- lib/services/bitcoind.js | 5 ++-- test/services/bitcoind.unit.js | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 10fccc47..4e79fcd6 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1516,10 +1516,9 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { var self = this; - if (_.isNumber(blockArg) || blockArg.length < 64) { - var height = parseInt(blockArg, 10); + if (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) { self._tryAll(function(done) { - self.client.getBlockHash(height, function(err, response) { + self.client.getBlockHash(blockArg, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 40a0392d..29a29a23 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3831,6 +3831,57 @@ describe('Bitcoin Service', function() { }); describe('#_maybeGetBlockHash', function() { + it('will not get block hash with an address', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br', function(err, hash) { + if (err) { + return done(err); + } + getBlockHash.callCount.should.equal(0); + hash.should.equal('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'); + done(); + }); + }); + it('will not get block hash with non zero-nine numeric string', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash('109a', function(err, hash) { + if (err) { + return done(err); + } + getBlockHash.callCount.should.equal(0); + hash.should.equal('109a'); + done(); + }); + }); + it('will not get block hash with an address', function(done) { + var bitcoind = new BitcoinService(baseConfig); + var getBlockHash = sinon.stub(); + bitcoind.nodes.push({ + client: { + getBlockHash: getBlockHash + } + }); + bitcoind._maybeGetBlockHash('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br', function(err, hash) { + if (err) { + return done(err); + } + getBlockHash.callCount.should.equal(0); + hash.should.equal('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'); + done(); + }); + }); it('will get the block hash if argument is a number', function(done) { var bitcoind = new BitcoinService(baseConfig); var getBlockHash = sinon.stub().callsArgWith(1, null, { From aa7f0d7c605212eb8415e1de10721df43ae179c0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 26 May 2016 10:23:42 -0400 Subject: [PATCH 225/299] test: remove duplicated test --- test/services/bitcoind.unit.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 29a29a23..8719f7f5 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -3865,23 +3865,6 @@ describe('Bitcoin Service', function() { done(); }); }); - it('will not get block hash with an address', function(done) { - var bitcoind = new BitcoinService(baseConfig); - var getBlockHash = sinon.stub(); - bitcoind.nodes.push({ - client: { - getBlockHash: getBlockHash - } - }); - bitcoind._maybeGetBlockHash('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br', function(err, hash) { - if (err) { - return done(err); - } - getBlockHash.callCount.should.equal(0); - hash.should.equal('2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br'); - done(); - }); - }); it('will get the block hash if argument is a number', function(done) { var bitcoind = new BitcoinService(baseConfig); var getBlockHash = sinon.stub().callsArgWith(1, null, { From 47e3cf7fc8c9122385c41a649b17bfd9ba192f3e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 27 May 2016 11:43:50 -0400 Subject: [PATCH 226/299] build: update download of bitcoind to tag v0.12-bitcore --- scripts/download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download b/scripts/download index 2e8c6714..f142b986 100755 --- a/scripts/download +++ b/scripts/download @@ -7,7 +7,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.0" url="https://github.com/bitpay/bitcoin/releases/download" -tag="v0.12-bitcore-rc3" +tag="v0.12-bitcore" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From 29b59c6f7da1269d13340e9b35c79b3f665ac0b6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 27 May 2016 11:56:41 -0400 Subject: [PATCH 227/299] build: update bitcoind-rpc to version 0.6.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d84633a2..5258a230 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ ], "dependencies": { "async": "^1.3.0", - "bitcoind-rpc": "braydonf/bitcoind-rpc#594d9aa0bee54ab247578785bd2acd16a8c012e6", + "bitcoind-rpc": "^0.6.0", "bitcore-lib": "^0.13.13", "body-parser": "^1.13.3", "colors": "^1.1.2", From 6df93877150c81b24195d5fd3e7037574d96d2b8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 27 May 2016 13:21:15 -0400 Subject: [PATCH 228/299] docs: update release process --- docs/release.md | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/docs/release.md b/docs/release.md index e2ce5313..217d29fa 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,30 +1,8 @@ # Release Process -Binaries for bitcoind are distributed for convenience and built deterministically with Gitian. -## How to Verify Signatures - -``` -cd bin -gpg --verify bitcoin-0.12.0-linux64.tar.gz.sig bitcoin-0.12.0-linux64.tar.gz -``` - -To verify signatures, use the following PGP keys: -- @braydonf: [https://pgp.mit.edu/pks/lookup?op=get&search=0x9BBF07CAC07A276D](https://pgp.mit.edu/pks/lookup?op=get&search=0x9BBF07CAC07A276D) -- @kleetus: [https://pgp.mit.edu/pks/lookup?op=get&search=0x33195D27EF6BDB7F](https://pgp.mit.edu/pks/lookup?op=get&search=0x33195D27EF6BDB7F) -- @pnagurny: [https://pgp.mit.edu/pks/lookup?op=get&search=0x0909B33F0AA53013](https://pgp.mit.edu/pks/lookup?op=get&search=0x0909B33F0AA53013) +Binaries for bitcoind are distributed for convenience and built deterministically with Gitian, signatures for bitcoind are located at the [gitian.sigs](https://github.com/bitpay/gitian.sigs) respository. ## How to Release -Ensure you've followed the instructions in the README.md for building the project from source. When building for any platform, be sure to keep in mind the minimum supported C and C++ system libraries and build from source using this library. Example, Ubuntu 12.04 has the earliest system library for Linux that we support, so it would be easiest to build the Linux artifact using this version. A script will then upload the binaries to S3 for later use. You will also need credentials for BitPay's bitcore-node S3 bucket and be listed as an author for the bitcore-node's npm module. -- Create a file `.bitcore-node-upload.json` in your home directory -- The format of this file should be: - -```json -{ - "region": "us-west-2", - "accessKeyId": "xxx", - "secretAccessKey": "yyy" -} -``` When publishing to npm, the .gitignore file is used to exclude files from the npm publishing process. Be sure that the bitcore-node directory has only the directories and files that you would like to publish to npm. You might need to run the commands below on each platform that you intend to publish (e.g. Mac and Linux). @@ -33,16 +11,19 @@ To make a release, bump the `version` of the `package.json`: ```bash git checkout master git pull upstream master -git commit -a -m "Bump package version to " npm install -npm run package -npm run upload +npm run test +npm run regtest +npm run jshint +git commit -a -m "Bump package version to " +git push upstream master npm publish ``` Create a release tag and push it to the BitPay Github repo: ```bash +git tag -s v -m 'v' git tag git push upstream ``` From fd00be7e8c340bb2955dd154c5e946a28b517107 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 27 May 2016 14:09:51 -0400 Subject: [PATCH 229/299] Bump package version to 3.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5258a230..c5d1a83c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "2.1.1-dev", + "version": "3.0.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From c897f62d028e5a24a75bb846c67fa39ef4759dc4 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 27 May 2016 14:21:33 -0400 Subject: [PATCH 230/299] Update release.md --- docs/release.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/release.md b/docs/release.md index 217d29fa..b2aa0a30 100644 --- a/docs/release.md +++ b/docs/release.md @@ -24,6 +24,5 @@ Create a release tag and push it to the BitPay Github repo: ```bash git tag -s v -m 'v' -git tag -git push upstream +git push upstream v ``` From b7560933badced164737ed97017b7ec35ad68721 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 31 May 2016 12:58:31 -0400 Subject: [PATCH 231/299] docs: bump recommended memory --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28063355..b71ec76a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and - Node.js v0.10, v0.12 or v4 - ZeroMQ *(libzmq3-dev for Ubuntu/Debian or zeromq on OSX)* - ~200GB of disk storage -- ~4GB of RAM +- ~8GB of RAM ## Configuration From 814576953c45be7d671b0312442bad3a961a2d62 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 1 Jun 2016 11:33:06 -0400 Subject: [PATCH 232/299] bitcoind: relative spawn.datadir handling Will expand the datadir into an absolute path based on the location of the configuration file. This is to avoid unexpected behavior in regards to the location of configuration files. --- lib/node.js | 1 + lib/scaffold/start.js | 1 - lib/services/bitcoind.js | 13 ++++++++++- test/services/bitcoind.unit.js | 42 ++++++++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/lib/node.js b/lib/node.js index c2350b50..1c97c361 100644 --- a/lib/node.js +++ b/lib/node.js @@ -228,6 +228,7 @@ Node.prototype._startService = function(serviceInfo, callback) { Node.prototype._logTitle = function() { if (this.configPath) { log.info('Using config:', this.configPath); + log.info('Using network:', this.getNetworkName()); } }; diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 378e96c3..e8654575 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -61,7 +61,6 @@ function checkConfigVersion2(fullConfig) { * @param {Object} options.config - The parsed bitcore-node.json configuration file * @param {Array} options.config.services - An array of services names. * @param {Object} options.config.servicesConfig - Parameters to pass to each service - * @param {String} options.config.datadir - A relative (to options.path) or absolute path to the datadir * @param {String} options.config.network - 'livenet', 'testnet' or 'regtest * @param {Number} options.config.port - The port to use for the web service */ diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4e79fcd6..7e0e04f9 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1,6 +1,7 @@ 'use strict'; var fs = require('fs'); +var path = require('path'); var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); @@ -16,6 +17,7 @@ var Transaction = bitcore.Transaction; var index = require('../'); var errors = index.errors; var log = index.log; +var utils = require('../utils'); var Service = require('../service'); /** @@ -318,8 +320,17 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in bitcoind config options'); $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in bitcoind config options'); + if (!utils.isAbsolutePath(this.options.spawn.datadir)) { + $.checkState(this.node.configPath); + $.checkState(utils.isAbsolutePath(this.node.configPath)); + var baseConfigPath = path.dirname(this.node.configPath); + this.options.spawn.datadir = path.resolve(baseConfigPath, this.options.spawn.datadir); + } + var spawnOptions = this.options.spawn; - var configPath = spawnOptions.datadir + '/bitcoin.conf'; + var configPath = path.resolve(spawnOptions.datadir, './bitcoin.conf'); + + log.info('Using bitcoin config file:', configPath); this.spawn = {}; this.spawn.datadir = this.options.spawn.datadir; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 8719f7f5..f486113a 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -340,6 +340,13 @@ describe('Bitcoin Service', function() { }); describe('#_loadSpawnConfiguration', function() { + var sandbox = sinon.sandbox.create(); + beforeEach(function() { + sandbox.stub(log, 'info'); + }); + afterEach(function() { + sandbox.restore(); + }); it('will parse a bitcoin.conf file', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { @@ -352,7 +359,9 @@ describe('Bitcoin Service', function() { } }); var bitcoind = new TestBitcoin(baseConfig); - bitcoind._loadSpawnConfiguration({datadir: process.env.HOME + '/.bitcoin'}); + bitcoind.options.spawn.datadir = '/tmp/.bitcoin'; + var node = {}; + bitcoind._loadSpawnConfiguration(node); should.exist(bitcoind.spawn.config); bitcoind.spawn.config.should.deep.equal({ addressindex: 1, @@ -374,6 +383,33 @@ describe('Bitcoin Service', function() { zmqpubrawtx: 'tcp://127.0.0.1:28332' }); }); + it('will expand relative datadir to absolute path', function() { + var TestBitcoin = proxyquire('../../lib/services/bitcoind', { + fs: { + readFileSync: readFileSync, + existsSync: sinon.stub().returns(true), + writeFileSync: sinon.stub() + }, + mkdirp: { + sync: sinon.stub() + } + }); + var config = { + node: { + network: bitcore.Networks.testnet, + configPath: '/tmp/.bitcore/bitcore-node.json' + }, + spawn: { + datadir: './data', + exec: 'testpath' + } + }; + var bitcoind = new TestBitcoin(config); + bitcoind.options.spawn.datadir = './data'; + var node = {}; + bitcoind._loadSpawnConfiguration(node); + bitcoind.options.spawn.datadir.should.equal('/tmp/.bitcore/data'); + }); it('should throw an exception if txindex isn\'t enabled in the configuration', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { @@ -420,7 +456,9 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new TestBitcoin(config); - bitcoind._loadSpawnConfiguration({datadir: process.env.HOME + '/.bitcoin'}); + bitcoind.options.spawn.datadir = '/tmp/.bitcoin'; + var node = {}; + bitcoind._loadSpawnConfiguration(node); }); }); From 4d780a9d2dafddbe6dfdfdfe12512d22c65c9457 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 1 Jun 2016 11:41:41 -0400 Subject: [PATCH 233/299] bitcoind: separate function for relative datadir expanding --- lib/services/bitcoind.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 7e0e04f9..f1e21464 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -313,6 +313,15 @@ Bitcoin.prototype._parseBitcoinConf = function(configPath) { return options; }; +Bitcoin.prototype._expandRelativeDatadir = function() { + if (!utils.isAbsolutePath(this.options.spawn.datadir)) { + $.checkState(this.node.configPath); + $.checkState(utils.isAbsolutePath(this.node.configPath)); + var baseConfigPath = path.dirname(this.node.configPath); + this.options.spawn.datadir = path.resolve(baseConfigPath, this.options.spawn.datadir); + } +}; + Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ @@ -320,12 +329,7 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in bitcoind config options'); $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in bitcoind config options'); - if (!utils.isAbsolutePath(this.options.spawn.datadir)) { - $.checkState(this.node.configPath); - $.checkState(utils.isAbsolutePath(this.node.configPath)); - var baseConfigPath = path.dirname(this.node.configPath); - this.options.spawn.datadir = path.resolve(baseConfigPath, this.options.spawn.datadir); - } + this._expandRelativeDatadir(); var spawnOptions = this.options.spawn; var configPath = path.resolve(spawnOptions.datadir, './bitcoin.conf'); From 32a6b25a9151a495f7641c9522b0f221f6162c6b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 1 Jun 2016 13:07:27 -0400 Subject: [PATCH 234/299] docs: clarify getAddressSummary results --- docs/services/bitcoind.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index d38f303e..238a12bd 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -270,13 +270,20 @@ The `summary` will have the format (values are in satoshis): totalSpent: 0, balance: 1000000000, unconfirmedBalance: 1000000000, - appearances: 1, // number of transactions + appearances: 1, unconfirmedAppearances: 0, txids: [ '3f7d13efe12e82f873f4d41f7e63bb64708fc4c942eb8c6822fa5bd7606adb00' ] } ``` +**Notes**: +- `totalReceived` does not exclude change *(the amount of satoshis originating from the same address)* +- `unconfirmedBalance` is the delta that the unconfirmed transactions have on the total balance *(can be both positive and negative)* +- `unconfirmedAppearances` is the total number of unconfirmed transactions +- `appearances` is the total confirmed transactions +- `txids` Are sorted in block order with the most recent at the beginning. A maximum of 1000 *(default)* will be returned, the `from` and `to` options can be used to get further values. + ## Events The Bitcoin Service exposes two events via the Bus, and there are a few events that can be directly registered: From cf16a23408fcecbe7717d7d4f75ca256222af41a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 1 Jun 2016 19:43:00 -0400 Subject: [PATCH 235/299] bitcoind: added zmq precondition Adds a state check that transaction and block events are over the same host and port. This is to make sure that block events can be subscribed to and that the tip of the chain stays up to date for correct confirmation counts. --- lib/services/bitcoind.js | 5 +++++ test/services/bitcoind.unit.js | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 4e79fcd6..7707303c 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -390,6 +390,11 @@ Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' ); + $.checkState( + (spawnConfig.zmqpubhashblock === spawnConfig.zmqpubrawtx), + '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in bitcoin.conf' + ); + if (spawnConfig.reindex && spawnConfig.reindex === 1) { log.warn('Reindex option is currently enabled. This means that bitcoind is undergoing a reindex. ' + 'The reindex flag will start the index from beginning every time the node is started, so it ' + diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 8719f7f5..00f616ef 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -429,7 +429,7 @@ describe('Bitcoin Service', function() { beforeEach(function() { sandbox.stub(log, 'warn'); }); - after(function() { + afterEach(function() { sandbox.restore(); }); it('should warn the user if reindex is set to 1 in the bitcoin.conf file', function() { @@ -448,6 +448,22 @@ describe('Bitcoin Service', function() { log.warn.callCount.should.equal(1); node._reindex.should.equal(true); }); + it('should warn if zmq port and hosts do not match', function() { + var bitcoind = new BitcoinService(baseConfig); + var config = { + txindex: 1, + addressindex: 1, + spentindex: 1, + server: 1, + zmqpubrawtx: 'tcp://127.0.0.1:28332', + zmqpubhashblock: 'tcp://127.0.0.1:28331', + reindex: 1 + }; + var node = {}; + (function() { + bitcoind._checkConfigIndexes(config, node); + }).should.throw('"zmqpubrawtx" and "zmqpubhashblock"'); + }); }); describe('#_resetCaches', function() { From 61caf6974a4c7b7453a1b3b5fe2e4995f72d0f53 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 3 Jun 2016 15:41:14 -0400 Subject: [PATCH 236/299] cli: parse json params --- lib/cli/main.js | 4 +++- lib/utils.js | 13 +++++++++++++ test/utils.unit.js | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/cli/main.js b/lib/cli/main.js index 86ef3053..5e0e13cc 100644 --- a/lib/cli/main.js +++ b/lib/cli/main.js @@ -3,6 +3,7 @@ var program = require('commander'); var path = require('path'); var bitcorenode = require('..'); +var utils = require('../utils'); function main(servicesPath, additionalServices) { /* jshint maxstatements: 100 */ @@ -124,7 +125,8 @@ function main(servicesPath, additionalServices) { program .command('call [params...]') .description('Call an API method') - .action(function(method, params) { + .action(function(method, paramsArg) { + var params = utils.parseParamsWithJSON(paramsArg); var configInfo = findConfig(process.cwd()); if (!configInfo) { configInfo = defaultConfig(); diff --git a/lib/utils.js b/lib/utils.js index cae2a5fe..b4b3af7a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -26,4 +26,17 @@ if (!utils.isAbsolutePath) { utils.isAbsolutePath = require('path-is-absolute'); } +utils.parseParamsWithJSON = function parseParamsWithJSON(paramsArg) { + var params = paramsArg.map(function(paramArg) { + var param; + try { + param = JSON.parse(paramArg); + } catch(err) { + param = paramArg; + } + return param; + }); + return params; +}; + module.exports = utils; diff --git a/test/utils.unit.js b/test/utils.unit.js index 96bc7b6c..ff166b83 100644 --- a/test/utils.unit.js +++ b/test/utils.unit.js @@ -130,4 +130,22 @@ describe('Utils', function() { }); + describe('#parseParamsWithJSON', function() { + it('will parse object', function() { + var paramsArg = ['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', '{"start": 100, "end": 1}']; + var params = utils.parseParamsWithJSON(paramsArg); + params.should.deep.equal(['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', {start: 100, end: 1}]); + }); + it('will parse array', function() { + var paramsArg = ['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', '[0, 1]']; + var params = utils.parseParamsWithJSON(paramsArg); + params.should.deep.equal(['3CMNFxN1oHBc4R1EpboAL5yzHGgE611Xou', [0, 1]]); + }); + it('will parse numbers', function() { + var paramsArg = ['3', 0, 'b', '0', 0x12, '0.0001']; + var params = utils.parseParamsWithJSON(paramsArg); + params.should.deep.equal([3, 0, 'b', 0, 0x12, 0.0001]); + }); + }); + }); From 3043263e3baa5a35bb9f1c8f3666b65133804e6d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 3 Jun 2016 15:54:01 -0400 Subject: [PATCH 237/299] node: handle undefined service config --- lib/node.js | 15 ++++++++++----- test/node.unit.js | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/lib/node.js b/lib/node.js index 1c97c361..23d6b04a 100644 --- a/lib/node.js +++ b/lib/node.js @@ -179,13 +179,18 @@ Node.prototype.getServiceOrder = function() { Node.prototype._startService = function(serviceInfo, callback) { var self = this; - $.checkState(_.isObject(serviceInfo.config)); - $.checkState(!serviceInfo.config.node); - $.checkState(!serviceInfo.config.name); - log.info('Starting ' + serviceInfo.name); - var config = serviceInfo.config; + var config; + if (serviceInfo.config) { + $.checkState(_.isObject(serviceInfo.config)); + $.checkState(!serviceInfo.config.node); + $.checkState(!serviceInfo.config.name); + config = serviceInfo.config; + } else { + config = {}; + } + config.node = this; config.name = serviceInfo.name; var service = new serviceInfo.module(config); diff --git a/test/node.unit.js b/test/node.unit.js index dc7b4a3e..85e5be67 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -233,6 +233,33 @@ describe('Bitcore Node', function() { getData.callCount.should.equal(1); }); }); + it('will handle config not being set', function() { + var node = new Node(baseConfig); + function TestService() {} + util.inherits(TestService, BaseService); + TestService.prototype.start = sinon.stub().callsArg(0); + var getData = sinon.stub(); + TestService.prototype.getData = getData; + TestService.prototype.getAPIMethods = function() { + return [ + ['getData', this, this.getData, 1] + ]; + }; + var service = { + name: 'testservice', + module: TestService, + }; + node._startService(service, function(err) { + if (err) { + throw err; + } + TestService.prototype.start.callCount.should.equal(1); + should.exist(node.services.testservice); + should.exist(node.getData); + node.getData(); + getData.callCount.should.equal(1); + }); + }); it('will give an error from start', function() { var node = new Node(baseConfig); function TestService() {} From d31438b22f8a41bf30bc4c421b1a588585c291a5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 3 Jun 2016 16:16:11 -0400 Subject: [PATCH 238/299] index: export bitcore-lib as lib --- index.js | 2 ++ test/index.unit.js | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 test/index.unit.js diff --git a/index.js b/index.js index 6626fe18..1b37813f 100644 --- a/index.js +++ b/index.js @@ -23,3 +23,5 @@ module.exports.cli.main = require('./lib/cli/main'); module.exports.cli.daemon = require('./lib/cli/daemon'); module.exports.cli.bitcore = require('./lib/cli/bitcore'); module.exports.cli.bitcored = require('./lib/cli/bitcored'); + +module.exports.lib = require('bitcore-lib'); diff --git a/test/index.unit.js b/test/index.unit.js new file mode 100644 index 00000000..8c72f47d --- /dev/null +++ b/test/index.unit.js @@ -0,0 +1,12 @@ +'use strict'; + +var should = require('chai').should(); + +describe('Index Exports', function() { + it('will export bitcore-lib', function() { + var bitcore = require('../'); + should.exist(bitcore.lib); + should.exist(bitcore.lib.Transaction); + should.exist(bitcore.lib.Block); + }); +}); From 70fae5335cd1c9188a0e3b9c1db0d77323ff2928 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 3 Jun 2016 16:31:54 -0400 Subject: [PATCH 239/299] node: optional getAPIMethods and getPublishEvents --- lib/node.js | 44 ++++++++++++++++++++++------------------ test/node.unit.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/lib/node.js b/lib/node.js index 1c97c361..7bd0e1a2 100644 --- a/lib/node.js +++ b/lib/node.js @@ -102,7 +102,9 @@ Node.prototype.getAllAPIMethods = function() { var methods = []; for(var i in this.services) { var mod = this.services[i]; - methods = methods.concat(mod.getAPIMethods()); + if (mod.getAPIMethods) { + methods = methods.concat(mod.getAPIMethods()); + } } return methods; }; @@ -115,7 +117,9 @@ Node.prototype.getAllPublishEvents = function() { var events = []; for (var i in this.services) { var mod = this.services[i]; - events = events.concat(mod.getPublishEvents()); + if (mod.getPublishEvents) { + events = events.concat(mod.getPublishEvents()); + } } return events; }; @@ -199,24 +203,26 @@ Node.prototype._startService = function(serviceInfo, callback) { } // add API methods - var methodData = service.getAPIMethods(); - var methodNameConflicts = []; - methodData.forEach(function(data) { - var name = data[0]; - var instance = data[1]; - var method = data[2]; - - if (self[name]) { - methodNameConflicts.push(name); - } else { - self[name] = function() { - return method.apply(instance, arguments); - }; + if (service.getAPIMethods) { + var methodData = service.getAPIMethods(); + var methodNameConflicts = []; + methodData.forEach(function(data) { + var name = data[0]; + var instance = data[1]; + var method = data[2]; + + if (self[name]) { + methodNameConflicts.push(name); + } else { + self[name] = function() { + return method.apply(instance, arguments); + }; + } + }); + + if (methodNameConflicts.length > 0) { + return callback(new Error('Existing API method(s) exists: ' + methodNameConflicts.join(', '))); } - }); - - if (methodNameConflicts.length > 0) { - return callback(new Error('Existing API method(s) exists: ' + methodNameConflicts.join(', '))); } callback(); diff --git a/test/node.unit.js b/test/node.unit.js index dc7b4a3e..de6ef4a5 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -139,6 +139,21 @@ describe('Bitcore Node', function() { var methods = node.getAllAPIMethods(); methods.should.deep.equal(['db1', 'db2', 'mda1', 'mda2', 'mdb1', 'mdb2']); }); + it('will handle service without getAPIMethods defined', function() { + var node = new Node(baseConfig); + node.services = { + db: { + getAPIMethods: sinon.stub().returns(['db1', 'db2']), + }, + service1: {}, + service2: { + getAPIMethods: sinon.stub().returns(['mdb1', 'mdb2']) + } + }; + + var methods = node.getAllAPIMethods(); + methods.should.deep.equal(['db1', 'db2', 'mdb1', 'mdb2']); + }); }); describe('#getAllPublishEvents', function() { @@ -158,6 +173,20 @@ describe('Bitcore Node', function() { var events = node.getAllPublishEvents(); events.should.deep.equal(['db1', 'db2', 'mda1', 'mda2', 'mdb1', 'mdb2']); }); + it('will handle service without getPublishEvents defined', function() { + var node = new Node(baseConfig); + node.services = { + db: { + getPublishEvents: sinon.stub().returns(['db1', 'db2']), + }, + service1: {}, + service2: { + getPublishEvents: sinon.stub().returns(['mdb1', 'mdb2']) + } + }; + var events = node.getAllPublishEvents(); + events.should.deep.equal(['db1', 'db2', 'mdb1', 'mdb2']); + }); }); describe('#getServiceOrder', function() { @@ -343,6 +372,28 @@ describe('Bitcore Node', function() { }); }); + it('will handle service with getAPIMethods undefined', function(done) { + var node = new Node(baseConfig); + + function TestService() {} + util.inherits(TestService, BaseService); + TestService.prototype.start = sinon.stub().callsArg(0); + TestService.prototype.getData = function() {}; + + node.getServiceOrder = sinon.stub().returns([ + { + name: 'test', + module: TestService, + config: {} + }, + ]); + + node.start(function() { + TestService.prototype.start.callCount.should.equal(1); + done(); + }); + + }); }); describe('#getNetworkName', function() { From 3715f07c84b9a7a02ab21522ad1d49eb693aedcd Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 7 Jun 2016 08:48:31 -0400 Subject: [PATCH 240/299] bitcoind: get detailed transactions with concurrency increase performance of querying address history by executing multiple rpc calls concurrently with a configurable limit --- lib/services/bitcoind.js | 9 ++++++++- test/services/bitcoind.unit.js | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fd0ee0b4..ea5340bd 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -76,6 +76,7 @@ Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; +Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY = 5; Bitcoin.DEFAULT_CONFIG_SETTINGS = { server: 1, whitelist: '127.0.0.1', @@ -92,6 +93,8 @@ Bitcoin.DEFAULT_CONFIG_SETTINGS = { }; Bitcoin.prototype._initDefaults = function(options) { + /* jshint maxcomplexity: 15 */ + // limits this.maxTxids = options.maxTxids || Bitcoin.DEFAULT_MAX_TXIDS; this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; @@ -106,6 +109,9 @@ Bitcoin.prototype._initDefaults = function(options) { this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; + // rpc limits + this.transactionConcurrency = options.transactionConcurrency || Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY; + // sync progress level when zmq subscribes to events this.zmqSubscribeProgress = options.zmqSubscribeProgress || Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS; }; @@ -1412,8 +1418,9 @@ Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { return callback(e); } - async.mapSeries( + async.mapLimit( txids, + self.transactionConcurrency, function(txid, next) { self._getAddressDetailedTransaction(txid, { queryMempool: queryMempool, diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 4f602649..9cb7096f 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -83,6 +83,16 @@ describe('Bitcoin Service', function() { }); }); + describe('#_initDefaults', function() { + it('will set transaction concurrency', function() { + var bitcoind = new BitcoinService(baseConfig); + bitcoind._initDefaults({transactionConcurrency: 10}); + bitcoind.transactionConcurrency.should.equal(10); + bitcoind._initDefaults({}); + bitcoind.transactionConcurrency.should.equal(5); + }); + }); + describe('@dependencies', function() { it('will have no dependencies', function() { BitcoinService.dependencies.should.deep.equal([]); From 6ac912545bb3badef5cecc1dec1acb7c6cc671bb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 9 Jun 2016 11:12:52 -0400 Subject: [PATCH 241/299] bitcoind: _tryAll -> _tryAllClients Fixes a timing bug with not all clients being tried --- lib/services/bitcoind.js | 41 +++++++++++-------- test/services/bitcoind.unit.js | 73 ++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 46 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fd0ee0b4..04dd812c 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -428,8 +428,15 @@ Bitcoin.prototype._resetCaches = function() { this.blockOverviewCache.reset(); }; -Bitcoin.prototype._tryAll = function(func, callback) { - async.retry({times: this.nodes.length, interval: this.tryAllInterval || 1000}, func, callback); +Bitcoin.prototype._tryAllClients = function(func, callback) { + var self = this; + var nodesIndex = 0; + var retry = function(done) { + var client = self.nodes[nodesIndex].client; + nodesIndex++; + func(client, done); + }; + async.retry({times: this.nodes.length, interval: this.tryAllInterval || 1000}, retry, callback); }; Bitcoin.prototype._wrapRPCError = function(errObj) { @@ -1537,8 +1544,8 @@ Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { var self = this; if (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) { - self._tryAll(function(done) { - self.client.getBlockHash(blockArg, function(err, response) { + self._tryAllClients(function(client, done) { + client.getBlockHash(blockArg, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1563,7 +1570,7 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { if (err) { return callback(err); } - self._tryAll(function(done) { + self._tryAllClients(function(client, done) { self.client.getBlock(blockhash, false, function(err, response) { if (err) { return done(self._wrapRPCError(err)); @@ -1603,8 +1610,8 @@ Bitcoin.prototype.getBlockOverview = function(blockArg, callback) { callback(null, cachedBlock); }); } else { - self._tryAll(function(done) { - self.client.getBlock(blockhash, true, function(err, response) { + self._tryAllClients(function(client, done) { + client.getBlock(blockhash, true, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1654,8 +1661,8 @@ Bitcoin.prototype.getBlock = function(blockArg, callback) { callback(null, cachedBlock); }); } else { - self._tryAll(function(done) { - self.client.getBlock(blockhash, false, function(err, response) { + self._tryAllClients(function(client, done) { + client.getBlock(blockhash, false, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1713,8 +1720,8 @@ Bitcoin.prototype.getBlockHeader = function(blockArg, callback) { if (err) { return callback(err); } - self._tryAll(function(done) { - self.client.getBlockHeader(blockhash, function(err, response) { + self._tryAllClients(function(client, done) { + client.getBlockHeader(blockhash, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1795,8 +1802,8 @@ Bitcoin.prototype.getRawTransaction = function(txid, callback) { callback(null, tx); }); } else { - self._tryAll(function(done) { - self.client.getRawTransaction(txid, function(err, response) { + self._tryAllClients(function(client, done) { + client.getRawTransaction(txid, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1822,8 +1829,8 @@ Bitcoin.prototype.getTransaction = function(txid, callback) { callback(null, tx); }); } else { - self._tryAll(function(done) { - self.client.getRawTransaction(txid, function(err, response) { + self._tryAllClients(function(client, done) { + client.getRawTransaction(txid, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } @@ -1937,8 +1944,8 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { callback(null, tx); }); } else { - self._tryAll(function(done) { - self.client.getRawTransaction(txid, 1, function(err, response) { + self._tryAllClients(function(client, done) { + client.getRawTransaction(txid, 1, function(err, response) { if (err) { return done(self._wrapRPCError(err)); } diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 4f602649..4eca90ae 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -525,46 +525,61 @@ describe('Bitcoin Service', function() { }); }); - describe('#_tryAll', function() { - it('will retry the number of bitcoind nodes', function(done) { + describe('#_tryAllClients', function() { + it('will retry for each node client', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.tryAllInterval = 1; - bitcoind.nodes.push({}); - bitcoind.nodes.push({}); - bitcoind.nodes.push({}); - var count = 0; - var func = function(callback) { - count++; - if (count <= 2) { - callback(new Error('test')); - } else { - callback(); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('test')) } - }; - bitcoind._tryAll(function(next) { - func(next); - }, function() { - count.should.equal(3); + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('test')) + } + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArg(0) + } + }); + bitcoind._tryAllClients(function(client, next) { + client.getInfo(next); + }, function(err) { + if (err) { + return done(err); + } + bitcoind.nodes[0].client.getInfo.callCount.should.equal(1); + bitcoind.nodes[1].client.getInfo.callCount.should.equal(1); + bitcoind.nodes[2].client.getInfo.callCount.should.equal(1); done(); }); }); - it('will get error if all fail', function(done) { + it('will get error if all clients fail', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.tryAllInterval = 1; - bitcoind.nodes.push({}); - bitcoind.nodes.push({}); - bitcoind.nodes.push({}); - var count = 0; - var func = function(callback) { - count++; - callback(new Error('test')); - }; - bitcoind._tryAll(function(next) { - func(next); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('test')) + } + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('test')) + } + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('test')) + } + }); + bitcoind._tryAllClients(function(client, next) { + client.getInfo(next); }, function(err) { should.exist(err); + err.should.be.instanceOf(Error); err.message.should.equal('test'); - count.should.equal(3); done(); }); }); From 8f9af8241a76296509276847007940f6aad37eba Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 9 Jun 2016 11:34:50 -0400 Subject: [PATCH 242/299] Bump package version to 3.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c5d1a83c..35b25b7c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "3.0.0", + "version": "3.0.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From ec760dc44e8f69a65bcc5327f21b8a3c03753729 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 9 Jun 2016 15:17:20 -0400 Subject: [PATCH 243/299] docs: update config in services doc --- docs/services.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/services.md b/docs/services.md index 85271347..02eb1117 100644 --- a/docs/services.md +++ b/docs/services.md @@ -41,15 +41,17 @@ var Bitcoin = bitcore.services.Bitcoin; var Web = bitcore.services.Web; var myNode = new bitcore.Node({ - datadir: '/home/user/.bitcore', - network: { - name: 'livenet' - }, - "services": [ + network: 'regtest' + services: [ { name: 'bitcoind', module: Bitcoin, - config: {} + config: { + spawn: { + datadir: '/home//.bitcoin', + exec: '/home//bitcore-node/bin/bitcoind' + } + } }, { name: 'web', From 3dc6860cb360f4716850f47dc0e0dc4e6dae8508 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 9 Jun 2016 16:39:03 -0400 Subject: [PATCH 244/299] bitcoind: connect option for strict ssl This is to be able to configure the RPC client to handle self-signed certificates for development purposes. --- lib/services/bitcoind.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fd0ee0b4..0ecde530 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -925,7 +925,8 @@ Bitcoin.prototype._connectProcess = function(config, callback) { host: config.rpchost || '127.0.0.1', port: config.rpcport, user: config.rpcuser, - pass: config.rpcpassword + pass: config.rpcpassword, + rejectUnauthorized: _.isUndefined(config.rpcstrict) ? true : config.rpcstrict }); self._loadTipFromNode(node, done); From b528c851abd70856425c68db7b98eb7c3af7831e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 10 Jun 2016 10:43:01 -0400 Subject: [PATCH 245/299] test: add additional mempool related utxo tests --- test/services/bitcoind.unit.js | 174 +++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 4f602649..30d89af8 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -2433,6 +2433,180 @@ describe('Bitcoin Service', function() { done(); }); }); + it('three confirmed utxos -> one utxo after mempool', function(done) { + var deltas = [ + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 0 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 1 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 2 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: 100000, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + script: '76a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac', + timestamp: 1461342833133 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = [ + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 0, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + }, + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 1, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + }, + { + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + outputIndex: 2, + script: '76a914f399b4b8894f1153b96fce29f05e6e116eb4c21788ac', + satoshis: 7679241, + height: 207111 + } + ]; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(1); + done(); + }); + }); + it('spending utxos in the mempool', function(done) { + var deltas = [ + { + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + satoshis: 7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707724 + }, + { + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + satoshis: 7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342707724 + }, + { + txid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + satoshis: 7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + timestamp: 1461342707724, + index: 2, + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 0 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 0, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 1 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: -7679241, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342707725, + prevtxid: '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0', + prevout: 2 + }, + { + txid: 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + satoshis: 100000, + address: '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo', + index: 1, + timestamp: 1461342833133 + } + ]; + var bitcoind = new BitcoinService(baseConfig); + var confirmedUtxos = []; + bitcoind.nodes.push({ + client: { + getAddressUtxos: sinon.stub().callsArgWith(1, null, { + result: confirmedUtxos + }), + getAddressMempool: sinon.stub().callsArgWith(1, null, { + result: deltas + }) + } + }); + var options = { + queryMempool: true + }; + var address = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + bitcoind.getAddressUnspentOutputs(address, options, function(err, utxos) { + if (err) { + return done(err); + } + utxos.length.should.equal(1); + utxos[0].address.should.equal(address); + utxos[0].txid.should.equal('e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce'); + utxos[0].outputIndex.should.equal(1); + utxos[0].script.should.equal('76a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac'); + utxos[0].timestamp.should.equal(1461342833133); + done(); + }); + }); it('will update with mempool results spending zero value output (likely never to happen)', function(done) { var deltas = [ { From a2a30b81d8481de9e62c70817e3d7bd412579e26 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 10 Jun 2016 19:05:22 -0400 Subject: [PATCH 246/299] bitcoind: start tryAllClients with the current round-robin index --- lib/services/bitcoind.js | 4 ++-- test/services/bitcoind.unit.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 04dd812c..ddcab71f 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -430,10 +430,10 @@ Bitcoin.prototype._resetCaches = function() { Bitcoin.prototype._tryAllClients = function(func, callback) { var self = this; - var nodesIndex = 0; + var nodesIndex = this.nodesIndex; var retry = function(done) { var client = self.nodes[nodesIndex].client; - nodesIndex++; + nodesIndex = (nodesIndex + 1) % self.nodes.length; func(client, done); }; async.retry({times: this.nodes.length, interval: this.tryAllInterval || 1000}, retry, callback); diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 4eca90ae..99dc6589 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -556,6 +556,37 @@ describe('Bitcoin Service', function() { done(); }); }); + it('will start using the current node index (round-robin)', function(done) { + var bitcoind = new BitcoinService(baseConfig); + bitcoind.tryAllInterval = 1; + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('2')) + } + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('3')) + } + }); + bitcoind.nodes.push({ + client: { + getInfo: sinon.stub().callsArgWith(0, new Error('1')) + } + }); + bitcoind.nodesIndex = 2; + bitcoind._tryAllClients(function(client, next) { + client.getInfo(next); + }, function(err) { + err.should.be.instanceOf(Error); + err.message.should.equal('3'); + bitcoind.nodes[0].client.getInfo.callCount.should.equal(1); + bitcoind.nodes[1].client.getInfo.callCount.should.equal(1); + bitcoind.nodes[2].client.getInfo.callCount.should.equal(1); + bitcoind.nodesIndex.should.equal(2); + done(); + }); + }); it('will get error if all clients fail', function(done) { var bitcoind = new BitcoinService(baseConfig); bitcoind.tryAllInterval = 1; From e5e9d600810141edae1f431277f71f06e97c026d Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 16 Jun 2016 13:34:33 -0400 Subject: [PATCH 247/299] Updated the download script for the latest bitcoind tag. --- scripts/download | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/download b/scripts/download index f142b986..46126283 100755 --- a/scripts/download +++ b/scripts/download @@ -5,9 +5,9 @@ set -e root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` -version="0.12.0" +version="0.12.1" url="https://github.com/bitpay/bitcoin/releases/download" -tag="v0.12-bitcore" +tag="v0.12.1-bitcore-rc1" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From b7f888fc3eec347645b4120686f160c89731ec3a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 16 Jun 2016 13:36:30 -0400 Subject: [PATCH 248/299] web: configure payload size --- lib/services/web.js | 6 +++++- test/services/web.unit.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/services/web.js b/lib/services/web.js index b050b5d5..850accb0 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -37,6 +37,10 @@ var WebService = function(options) { this.httpsOptions = options.httpsOptions || this.node.httpsOptions; this.port = options.port || this.node.port || 3456; + // set the maximum size of json payload, defaults to express default + // see: https://github.com/expressjs/body-parser#limit + this.jsonRequestLimit = options.jsonRequestLimit || '100kb'; + this.enableSocketRPC = _.isUndefined(options.enableSocketRPC) ? WebService.DEFAULT_SOCKET_RPC : options.enableSocketRPC; @@ -59,7 +63,7 @@ WebService.DEFAULT_SOCKET_RPC = true; */ WebService.prototype.start = function(callback) { this.app = express(); - this.app.use(bodyParser.json()); + this.app.use(bodyParser.json({limit: this.jsonRequestLimit})); if(this.https) { this.transformHttpsOptions(); diff --git a/test/services/web.unit.js b/test/services/web.unit.js index 3d0ff593..b57d07b4 100644 --- a/test/services/web.unit.js +++ b/test/services/web.unit.js @@ -46,6 +46,10 @@ describe('WebService', function() { var web3 = new WebService({node: defaultNode}); web3.enableSocketRPC.should.equal(WebService.DEFAULT_SOCKET_RPC); }); + it('will set configuration options for max payload', function() { + var web = new WebService({node: defaultNode, jsonRequestLimit: '200kb'}); + web.jsonRequestLimit.should.equal('200kb'); + }); }); describe('#start', function() { @@ -75,6 +79,39 @@ describe('WebService', function() { done(); }); }); + it('should pass json request limit to json body parser', function(done) { + var node = new EventEmitter(); + var jsonStub = sinon.stub(); + var TestWebService = proxyquire('../../lib/services/web', { + http: { + createServer: sinon.stub() + }, + https: { + createServer: sinon.stub() + }, + fs: fsStub, + express: sinon.stub().returns({ + use: sinon.stub() + }), + 'body-parser': { + json: jsonStub + }, + 'socket.io': { + listen: sinon.stub().returns({ + on: sinon.stub() + }) + } + }); + var web = new TestWebService({node: node}); + web.start(function(err) { + if (err) { + return done(err); + } + jsonStub.callCount.should.equal(1); + jsonStub.args[0][0].limit.should.equal('100kb'); + done(); + }); + }); }); describe('#stop', function() { From f72fe82c6037011220bd7936bf3baa263ce59a00 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 20 Jun 2016 10:25:25 -0400 Subject: [PATCH 249/299] Bump package version to 3.0.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 35b25b7c..7564aea3 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "3.0.1", + "version": "3.0.2", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From f15460a3d99356a4692a5f82171bddb5211a16e3 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 27 Jun 2016 10:57:16 -0400 Subject: [PATCH 250/299] Adjusted tags. --- scripts/download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download b/scripts/download index 46126283..d6c3d7fd 100755 --- a/scripts/download +++ b/scripts/download @@ -7,7 +7,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.1" url="https://github.com/bitpay/bitcoin/releases/download" -tag="v0.12.1-bitcore-rc1" +tag="v0.12.1-bitcore" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From cc4d8d4c5e3173f4176b1f42b73715a03dc6ad33 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 27 Jun 2016 13:04:10 -0400 Subject: [PATCH 251/299] Bump package version to 3.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7564aea3..384dc83f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "3.0.2", + "version": "3.1.0", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 4dc664200b794b3c3d583b09f98d67ede54d25e5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 1 Jul 2016 09:53:08 -0400 Subject: [PATCH 252/299] Update links in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b71ec76a..cd21234e 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ This will create a directory with configuration files for your node and install There are several add-on services available to extend the functionality of Bitcore: -- [Insight API](https://github.com/bitpay/insight-api/tree/v0.3.0) -- [Insight UI](https://github.com/bitpay/insight/tree/v0.3.0) +- [Insight API](https://github.com/bitpay/insight-api) +- [Insight UI](https://github.com/bitpay/insight) - [Bitcore Wallet Service](https://github.com/bitpay/bitcore-wallet-service) ## Documentation From e602eb9a48e5e59094cfa991f06e543388af1444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20B=C3=ADlek?= Date: Mon, 4 Jul 2016 18:20:29 +0200 Subject: [PATCH 253/299] Fixing link to bitcoin service docs --- docs/upgrade.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index c568554a..06993242 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -73,5 +73,5 @@ rpcpassword= **Important**: Once changes have been made you'll also need to add the `reindex=1` option **only for the first startup** to regenerate the indexes. Once this is complete you should be able to remove the `bitcore-node.db` directory with the old indexes. ### API and Service Changes -- Many API methods that were a part of the `db` and `address` services are now a part of the `bitcoind` service. Please see [Bitcoin Service Docs](docs/services/bitcoind.md) for more details. -- The `db` and `address` services are deprecated, most of the functionality still exists. Any services that were extending indexes with the `db` service, will need to manage chain state itself, or build the indexes within `bitcoind`. \ No newline at end of file +- Many API methods that were a part of the `db` and `address` services are now a part of the `bitcoind` service. Please see [Bitcoin Service Docs](services/bitcoind.md) for more details. +- The `db` and `address` services are deprecated, most of the functionality still exists. Any services that were extending indexes with the `db` service, will need to manage chain state itself, or build the indexes within `bitcoind`. From fa79e694cf29282510dbb7f86c7ec334839ea113 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Jul 2016 10:07:23 -0400 Subject: [PATCH 254/299] Fix second link to bitcoin service docs --- docs/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 06993242..fcc2cba1 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -18,7 +18,7 @@ To start reindexing add `reindex=1` during the **first startup only**. ### Configuration Options - The `bitcoin.conf` file in will need to be updated to include additional indexes *(see below)*. -- The `datadir` option is now a part of `bitcoind` spawn configuration, and there is a new option to connect to multiple bitcoind processes (Please see [Bitcoin Service Docs](docs/services/bitcoind.md) for more details). The services `db` and `address` are now a part of the `bitcoind` service. Here is how to update `bitcore-node.json` configuration options: +- The `datadir` option is now a part of `bitcoind` spawn configuration, and there is a new option to connect to multiple bitcoind processes (Please see [Bitcoin Service Docs](services/bitcoind.md) for more details). The services `db` and `address` are now a part of the `bitcoind` service. Here is how to update `bitcore-node.json` configuration options: **Before**: ```json From 716cee68eeb974952799164bcbe0ff9c9bb8d0ab Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Jul 2016 10:11:52 -0400 Subject: [PATCH 255/299] Link directly to repository without redirect --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd21234e..e2089775 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ This will create a directory with configuration files for your node and install There are several add-on services available to extend the functionality of Bitcore: - [Insight API](https://github.com/bitpay/insight-api) -- [Insight UI](https://github.com/bitpay/insight) +- [Insight UI](https://github.com/bitpay/insight-ui) - [Bitcore Wallet Service](https://github.com/bitpay/bitcore-wallet-service) ## Documentation From f6e0783f55f638a8f1ddd136c581bbb8f706ba14 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Jul 2016 10:12:44 -0400 Subject: [PATCH 256/299] Update links to bitcoin bitcore branch --- README.md | 4 ++-- docs/services/bitcoind.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e2089775..0df35b5a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Bitcore Node ============ -A Bitcoin full node for building applications and services with Node.js. A node is extensible and can be configured to run additional services. At the minimum a node has an interface to [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12-bitcore) for more advanced address queries. Additional services can be enabled to make a node more useful such as exposing new APIs, running a block explorer and wallet service. +A Bitcoin full node for building applications and services with Node.js. A node is extensible and can be configured to run additional services. At the minimum a node has an interface to [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore) for more advanced address queries. Additional services can be enabled to make a node more useful such as exposing new APIs, running a block explorer and wallet service. ## Install @@ -10,7 +10,7 @@ npm install -g bitcore-node bitcore-node start ``` -Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12-bitcore). +Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore). ## Prerequisites diff --git a/docs/services/bitcoind.md b/docs/services/bitcoind.md index 238a12bd..99d0a133 100644 --- a/docs/services/bitcoind.md +++ b/docs/services/bitcoind.md @@ -1,6 +1,6 @@ # Bitcoin Service -The Bitcoin Service is a Node.js interface to [Bitcoin Core](https://github.com/bitcoin/bitcoin) for querying information about the bitcoin block chain. It will manage starting and stopping `bitcoind` or connect to several running `bitcoind` processes. It uses a branch of a [branch of Bitcoin Core](https://github.com/bitpay/bitcoin/tree/0.12-bitcore) with additional indexes for querying information about addresses and blocks. Results are cached for performance and there are several additional API methods added for common queries. +The Bitcoin Service is a Node.js interface to [Bitcoin Core](https://github.com/bitcoin/bitcoin) for querying information about the bitcoin block chain. It will manage starting and stopping `bitcoind` or connect to several running `bitcoind` processes. It uses a branch of a [branch of Bitcoin Core](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore) with additional indexes for querying information about addresses and blocks. Results are cached for performance and there are several additional API methods added for common queries. ## Configuration From f913784421fff11d4a68d913cf8e97ee54bad14f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 6 Jul 2016 14:44:29 -0400 Subject: [PATCH 257/299] Bump package version to 3.1.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 384dc83f..e675afd0 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "3.1.0", + "version": "3.1.1", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 75d41acdd60991e8bd016f8513d1393328399704 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 26 Jul 2016 15:48:15 -0400 Subject: [PATCH 258/299] Bump bitcoind tag to v0.12.1-bitcore-2 --- scripts/download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download b/scripts/download index d6c3d7fd..40bdfb48 100755 --- a/scripts/download +++ b/scripts/download @@ -7,7 +7,7 @@ platform=`uname -a | awk '{print tolower($1)}'` arch=`uname -m` version="0.12.1" url="https://github.com/bitpay/bitcoin/releases/download" -tag="v0.12.1-bitcore" +tag="v0.12.1-bitcore-2" if [ "${platform}" == "linux" ]; then if [ "${arch}" == "x86_64" ]; then From 639fdc941ed69e47420772d2091e453aac23dfd5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Jul 2016 10:59:12 -0400 Subject: [PATCH 259/299] Bump package version to 3.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e675afd0..3ca74ae6 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-node", "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", "author": "BitPay ", - "version": "3.1.1", + "version": "3.1.2", "main": "./index.js", "repository": "git://github.com/bitpay/bitcore-node.git", "homepage": "https://github.com/bitpay/bitcore-node", From 0a9d654ca7502fed8bfd57dffa5f17a4bf69e9eb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sun, 28 Aug 2016 14:10:11 +1200 Subject: [PATCH 260/299] Zcash-ify package --- lib/scaffold/create.js | 4 ++-- package.json | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index c521602e..b9631d81 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -14,13 +14,13 @@ var defaultBaseConfig = require('./default-base-config'); var version = '^' + packageFile.version; var BASE_PACKAGE = { - description: 'A full Bitcoin node build with Bitcore', + description: 'A full Zcash node build with Bitcore', repository: 'https://github.com/user/project', license: 'MIT', readme: 'README.md', dependencies: { 'bitcore-lib': '^' + bitcore.version, - 'bitcore-node': version + 'bitcore-node-zcash': version } }; diff --git a/package.json b/package.json index 3ca74ae6..20752fd6 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { - "name": "bitcore-node", - "description": "Full node with extended capabilities using Bitcore and Bitcoin Core", + "name": "bitcore-node-zcash", + "description": "Full node with extended capabilities using Bitcore and Zcash", "author": "BitPay ", "version": "3.1.2", "main": "./index.js", - "repository": "git://github.com/bitpay/bitcore-node.git", - "homepage": "https://github.com/bitpay/bitcore-node", + "repository": "git://github.com/str4d/bitcore-node-zcash.git", + "homepage": "https://github.com/str4d/bitcore-node-zcash", "bugs": { "url": "https://github.com/bitpay/bitcore-node/issues" }, @@ -24,6 +24,10 @@ { "name": "Patrick Nagurny", "email": "patrick@bitpay.com" + }, + { + "name": "Jack Grigg", + "email": "jack@z.cash" } ], "bin": { @@ -40,8 +44,8 @@ "coveralls": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- --recursive -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js" }, "tags": [ - "bitcoin", - "bitcoind" + "zcash", + "zcashd" ], "dependencies": { "async": "^1.3.0", From 10b44b6392a49b4ab7b6318465db30cc5507dbdf Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sun, 28 Aug 2016 14:10:36 +1200 Subject: [PATCH 261/299] Use bitcore-lib-zcash --- index.js | 2 +- lib/logger.js | 2 +- lib/node.js | 2 +- lib/scaffold/add.js | 2 +- lib/scaffold/create.js | 4 ++-- lib/scaffold/find-config.js | 2 +- lib/scaffold/remove.js | 2 +- lib/scaffold/start.js | 2 +- lib/services/bitcoind.js | 2 +- lib/services/web.js | 2 +- package.json | 2 +- regtest/bitcoind.js | 2 +- regtest/cluster.js | 2 +- regtest/node.js | 2 +- regtest/p2p.js | 2 +- test/node.unit.js | 2 +- test/scaffold/add.integration.js | 2 +- test/services/bitcoind.unit.js | 2 +- 18 files changed, 19 insertions(+), 19 deletions(-) diff --git a/index.js b/index.js index 1b37813f..670d50d7 100644 --- a/index.js +++ b/index.js @@ -24,4 +24,4 @@ module.exports.cli.daemon = require('./lib/cli/daemon'); module.exports.cli.bitcore = require('./lib/cli/bitcore'); module.exports.cli.bitcored = require('./lib/cli/bitcored'); -module.exports.lib = require('bitcore-lib'); +module.exports.lib = require('bitcore-lib-zcash'); diff --git a/lib/logger.js b/lib/logger.js index 4084c2cc..edbc5c79 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,6 +1,6 @@ 'use strict'; -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var _ = bitcore.deps._; var colors = require('colors/safe'); diff --git a/lib/node.js b/lib/node.js index d7e95c4f..887c34a1 100644 --- a/lib/node.js +++ b/lib/node.js @@ -3,7 +3,7 @@ var util = require('util'); var EventEmitter = require('events').EventEmitter; var async = require('async'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var Networks = bitcore.Networks; var $ = bitcore.util.preconditions; var _ = bitcore.deps._; diff --git a/lib/scaffold/add.js b/lib/scaffold/add.js index 76f24bdb..1f2d3769 100644 --- a/lib/scaffold/add.js +++ b/lib/scaffold/add.js @@ -4,7 +4,7 @@ var async = require('async'); var fs = require('fs'); var path = require('path'); var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var utils = require('../utils'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index b9631d81..9a6603fb 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -1,7 +1,7 @@ 'use strict'; var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var async = require('async'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; @@ -19,7 +19,7 @@ var BASE_PACKAGE = { license: 'MIT', readme: 'README.md', dependencies: { - 'bitcore-lib': '^' + bitcore.version, + 'bitcore-lib-zcash': '^' + bitcore.version, 'bitcore-node-zcash': version } }; diff --git a/lib/scaffold/find-config.js b/lib/scaffold/find-config.js index a4306e8b..7ac7e359 100644 --- a/lib/scaffold/find-config.js +++ b/lib/scaffold/find-config.js @@ -1,6 +1,6 @@ 'use strict'; -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var path = require('path'); diff --git a/lib/scaffold/remove.js b/lib/scaffold/remove.js index 6d866d6f..db7b4da2 100644 --- a/lib/scaffold/remove.js +++ b/lib/scaffold/remove.js @@ -5,7 +5,7 @@ var fs = require('fs'); var npm = require('npm'); var path = require('path'); var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var utils = require('../utils'); diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index e8654575..c5cd517e 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -3,7 +3,7 @@ var path = require('path'); var BitcoreNode = require('../node'); var index = require('../'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var _ = bitcore.deps._; var log = index.log; var shuttingDown = false; diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 8774498e..b827d539 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -5,7 +5,7 @@ var path = require('path'); var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var zmq = require('zmq'); var async = require('async'); var LRU = require('lru-cache'); diff --git a/lib/services/web.js b/lib/services/web.js index 850accb0..8d1a0228 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -9,7 +9,7 @@ var socketio = require('socket.io'); var inherits = require('util').inherits; var BaseService = require('../service'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var _ = bitcore.deps._; var index = require('../'); var log = index.log; diff --git a/package.json b/package.json index 20752fd6..69630935 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "dependencies": { "async": "^1.3.0", "bitcoind-rpc": "^0.6.0", - "bitcore-lib": "^0.13.13", + "bitcore-lib-zcash": "str4d/bitcore-lib-zcash", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 13346416..9f270507 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -7,7 +7,7 @@ var index = require('..'); var log = index.log; var chai = require('chai'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var BN = bitcore.crypto.BN; var async = require('async'); var rimraf = require('rimraf'); diff --git a/regtest/cluster.js b/regtest/cluster.js index c8ab8754..2397d3e9 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -6,7 +6,7 @@ var spawn = require('child_process').spawn; var BitcoinRPC = require('bitcoind-rpc'); var rimraf = require('rimraf'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var chai = require('chai'); var should = chai.should(); diff --git a/regtest/node.js b/regtest/node.js index 2e8539ab..8d189286 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -9,7 +9,7 @@ var log = index.log; log.debug = function() {}; var chai = require('chai'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var rimraf = require('rimraf'); var node; diff --git a/regtest/p2p.js b/regtest/p2p.js index 4af160bd..e405cfa7 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -10,7 +10,7 @@ var p2p = require('bitcore-p2p'); var Peer = p2p.Peer; var Messages = p2p.Messages; var chai = require('chai'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var Transaction = bitcore.Transaction; var BN = bitcore.crypto.BN; var async = require('async'); diff --git a/test/node.unit.js b/test/node.unit.js index 624ccaf8..633c2bb5 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -2,7 +2,7 @@ var should = require('chai').should(); var sinon = require('sinon'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var Networks = bitcore.Networks; var proxyquire = require('proxyquire'); var util = require('util'); diff --git a/test/scaffold/add.integration.js b/test/scaffold/add.integration.js index 21ef4ad7..fa5b8d5f 100644 --- a/test/scaffold/add.integration.js +++ b/test/scaffold/add.integration.js @@ -94,7 +94,7 @@ describe('#add', function() { var callCount = 0; var oldPackage = { dependencies: { - 'bitcore-lib': '^v0.13.7', + 'bitcore-lib-zcash': '^v0.13.7', 'bitcore-node': '^v0.2.0' } }; diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 57819d12..96c9a8d0 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -6,7 +6,7 @@ var path = require('path'); var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); var crypto = require('crypto'); -var bitcore = require('bitcore-lib'); +var bitcore = require('bitcore-lib-zcash'); var _ = bitcore.deps._; var sinon = require('sinon'); var proxyquire = require('proxyquire'); From 112c6d896d5c96cfe4a1d87548870a5c3edad1f9 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sun, 28 Aug 2016 14:27:01 +1200 Subject: [PATCH 262/299] Zcash-ify daemon, config files etc. --- benchmarks/index.js | 2 +- docs/services.md | 2 +- docs/upgrade.md | 2 +- lib/scaffold/default-base-config.js | 4 +-- lib/scaffold/default-config.js | 2 +- lib/scaffold/start.js | 2 +- lib/services/bitcoind.js | 12 +++---- regtest/bitcoind.js | 2 +- regtest/cluster.js | 8 ++--- regtest/node.js | 2 +- regtest/p2p.js | 2 +- test/scaffold/create.integration.js | 6 ++-- .../default-base-config.integration.js | 4 +-- test/scaffold/default-config.integration.js | 2 +- test/services/bitcoind.unit.js | 32 +++++++++---------- 15 files changed, 42 insertions(+), 42 deletions(-) diff --git a/benchmarks/index.js b/benchmarks/index.js index 71e30864..29bc34ed 100644 --- a/benchmarks/index.js +++ b/benchmarks/index.js @@ -28,7 +28,7 @@ var fixtureData = { var bitcoind = require('../').services.Bitcoin({ node: { - datadir: process.env.HOME + '/.bitcoin', + datadir: process.env.HOME + '/.zcash', network: { name: 'testnet' } diff --git a/docs/services.md b/docs/services.md index 02eb1117..198c3312 100644 --- a/docs/services.md +++ b/docs/services.md @@ -49,7 +49,7 @@ var myNode = new bitcore.Node({ config: { spawn: { datadir: '/home//.bitcoin', - exec: '/home//bitcore-node/bin/bitcoind' + exec: '/home//bitcore-node/bin/zcashd' } } }, diff --git a/docs/upgrade.md b/docs/upgrade.md index fcc2cba1..d07df289 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -48,7 +48,7 @@ To start reindexing add `reindex=1` during the **first startup only**. "bitcoind": { "spawn": { "datadir": "/home//.bitcoin", - "exec": "/home//bitcore-node/bin/bitcoind" + "exec": "/home//bitcore-node/bin/zcashd" } } } diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index 1f584b55..4f641b2c 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -22,8 +22,8 @@ function getDefaultBaseConfig(options) { servicesConfig: { bitcoind: { spawn: { - datadir: options.datadir || path.resolve(process.env.HOME, '.bitcoin'), - exec: path.resolve(__dirname, '../../bin/bitcoind') + datadir: options.datadir || path.resolve(process.env.HOME, '.zcash'), + exec: path.resolve(__dirname, '../../bin/zcashd') } } } diff --git a/lib/scaffold/default-config.js b/lib/scaffold/default-config.js index 7075a7fc..852e9544 100644 --- a/lib/scaffold/default-config.js +++ b/lib/scaffold/default-config.js @@ -38,7 +38,7 @@ function getDefaultConfig(options) { bitcoind: { spawn: { datadir: path.resolve(defaultPath, './data'), - exec: path.resolve(__dirname, '../../bin/bitcoind') + exec: path.resolve(__dirname, '../../bin/zcashd') } } } diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index c5cd517e..b6f9bc28 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -34,7 +34,7 @@ function checkConfigVersion2(fullConfig) { bitcoind: { spawn: { datadir: fullConfig.datadir, - exec: path.resolve(__dirname, '../../bin/bitcoind') + exec: path.resolve(__dirname, '../../bin/zcashd') } } } diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index b827d539..2dc9899d 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -338,7 +338,7 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { this._expandRelativeDatadir(); var spawnOptions = this.options.spawn; - var configPath = path.resolve(spawnOptions.datadir, './bitcoin.conf'); + var configPath = path.resolve(spawnOptions.datadir, './zcash.conf'); log.info('Using bitcoin config file:', configPath); @@ -488,10 +488,10 @@ Bitcoin.prototype._initChain = function(callback) { Bitcoin.prototype._getDefaultConf = function() { var networkOptions = { - rpcport: 8332 + rpcport: 8232 }; if (this.node.network === bitcore.Networks.testnet) { - networkOptions.rpcport = 18332; + networkOptions.rpcport = 18232; } return networkOptions; }; @@ -499,9 +499,9 @@ Bitcoin.prototype._getDefaultConf = function() { Bitcoin.prototype._getNetworkConfigPath = function() { var networkPath; if (this.node.network === bitcore.Networks.testnet) { - networkPath = 'testnet3/bitcoin.conf'; + networkPath = 'testnet3/zcash.conf'; if (this.node.network.regtestEnabled) { - networkPath = 'regtest/bitcoin.conf'; + networkPath = 'regtest/zcash.conf'; } } return networkPath; @@ -796,7 +796,7 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) { Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { var self = this; var spawnOptions = this.options.spawn; - var pidPath = spawnOptions.datadir + '/bitcoind.pid'; + var pidPath = spawnOptions.datadir + '/zcashd.pid'; function stopProcess() { fs.readFile(pidPath, 'utf8', function(err, pid) { diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index 9f270507..fb9cb7c5 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -46,7 +46,7 @@ describe('Bitcoind Functionality', function() { bitcoind = require('../').services.Bitcoin({ spawn: { datadir: datadir, - exec: path.resolve(__dirname, '../bin/bitcoind') + exec: path.resolve(__dirname, '../bin/zcashd') }, node: { network: regtestNetwork, diff --git a/regtest/cluster.js b/regtest/cluster.js index 2397d3e9..56dc1d0c 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -19,11 +19,11 @@ var BitcoinService = index.services.Bitcoin; describe('Bitcoin Cluster', function() { var node; var daemons = []; - var execPath = path.resolve(__dirname, '../bin/bitcoind'); + var execPath = path.resolve(__dirname, '../bin/zcashd'); var nodesConf = [ { datadir: path.resolve(__dirname, './data/node1'), - conf: path.resolve(__dirname, './data/node1/bitcoin.conf'), + conf: path.resolve(__dirname, './data/node1/zcash.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30521, @@ -32,7 +32,7 @@ describe('Bitcoin Cluster', function() { }, { datadir: path.resolve(__dirname, './data/node2'), - conf: path.resolve(__dirname, './data/node2/bitcoin.conf'), + conf: path.resolve(__dirname, './data/node2/zcash.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30522, @@ -41,7 +41,7 @@ describe('Bitcoin Cluster', function() { }, { datadir: path.resolve(__dirname, './data/node3'), - conf: path.resolve(__dirname, './data/node3/bitcoin.conf'), + conf: path.resolve(__dirname, './data/node3/zcash.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30523, diff --git a/regtest/node.js b/regtest/node.js index 8d189286..c5febe19 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -53,7 +53,7 @@ describe('Node Functionality', function() { config: { spawn: { datadir: datadir, - exec: path.resolve(__dirname, '../bin/bitcoind') + exec: path.resolve(__dirname, '../bin/zcashd') } } } diff --git a/regtest/p2p.js b/regtest/p2p.js index e405cfa7..cc6b61b9 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -52,7 +52,7 @@ describe('P2P Functionality', function() { bitcoind = require('../').services.Bitcoin({ spawn: { datadir: datadir, - exec: path.resolve(__dirname, '../bin/bitcoind') + exec: path.resolve(__dirname, '../bin/zcashd') }, node: { network: bitcore.Networks.testnet diff --git a/test/scaffold/create.integration.js b/test/scaffold/create.integration.js index f2c61523..ccf7c742 100644 --- a/test/scaffold/create.integration.js +++ b/test/scaffold/create.integration.js @@ -33,7 +33,7 @@ describe('#create', function() { if (err) { throw err; } - mkdirp(testDir + '/.bitcoin', function(err) { + mkdirp(testDir + '/.zcash', function(err) { if (err) { throw err; } @@ -104,7 +104,7 @@ describe('#create', function() { dirname: 'mynode3', name: 'My Node 3', isGlobal: true, - datadir: '../.bitcoin' + datadir: '../.zcash' }, function(err) { if (err) { throw err; @@ -139,7 +139,7 @@ describe('#create', function() { dirname: 'mynode4', name: 'My Node 4', isGlobal: false, - datadir: '../.bitcoin' + datadir: '../.zcash' }, function(err) { should.exist(err); err.message.should.equal('There was an error installing dependencies.'); diff --git a/test/scaffold/default-base-config.integration.js b/test/scaffold/default-base-config.integration.js index b622c742..72b994f8 100644 --- a/test/scaffold/default-base-config.integration.js +++ b/test/scaffold/default-base-config.integration.js @@ -14,8 +14,8 @@ describe('#defaultBaseConfig', function() { info.config.port.should.equal(3001); info.config.services.should.deep.equal(['bitcoind', 'web']); var bitcoind = info.config.servicesConfig.bitcoind; - bitcoind.spawn.datadir.should.equal(home + '/.bitcoin'); - bitcoind.spawn.exec.should.equal(path.resolve(__dirname, '../../bin/bitcoind')); + bitcoind.spawn.datadir.should.equal(home + '/.zcash'); + bitcoind.spawn.exec.should.equal(path.resolve(__dirname, '../../bin/zcashd')); }); it('be able to specify a network', function() { var info = defaultBaseConfig({network: 'testnet'}); diff --git a/test/scaffold/default-config.integration.js b/test/scaffold/default-config.integration.js index 83e674e5..9b5338d1 100644 --- a/test/scaffold/default-config.integration.js +++ b/test/scaffold/default-config.integration.js @@ -6,7 +6,7 @@ var sinon = require('sinon'); var proxyquire = require('proxyquire'); describe('#defaultConfig', function() { - var expectedExecPath = path.resolve(__dirname, '../../bin/bitcoind'); + var expectedExecPath = path.resolve(__dirname, '../../bin/zcashd'); it('will return expected configuration', function() { var config = JSON.stringify({ diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 96c9a8d0..0a75654f 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -18,13 +18,13 @@ var log = index.log; var errors = index.errors; var Transaction = bitcore.Transaction; -var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/bitcoin.conf'))); +var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/zcash.conf'))); var BitcoinService = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync } }); -var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.bitcoin.conf'), 'utf8'); +var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.zcash.conf'), 'utf8'); describe('Bitcoin Service', function() { var txhex = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000'; @@ -357,7 +357,7 @@ describe('Bitcoin Service', function() { afterEach(function() { sandbox.restore(); }); - it('will parse a bitcoin.conf file', function() { + it('will parse a zcash.conf file', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync, @@ -369,7 +369,7 @@ describe('Bitcoin Service', function() { } }); var bitcoind = new TestBitcoin(baseConfig); - bitcoind.options.spawn.datadir = '/tmp/.bitcoin'; + bitcoind.options.spawn.datadir = '/tmp/.zcash'; var node = {}; bitcoind._loadSpawnConfiguration(node); should.exist(bitcoind.spawn.config); @@ -423,7 +423,7 @@ describe('Bitcoin Service', function() { it('should throw an exception if txindex isn\'t enabled in the configuration', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { - readFileSync: sinon.stub().returns(fs.readFileSync(__dirname + '/../data/badbitcoin.conf')), + readFileSync: sinon.stub().returns(fs.readFileSync(__dirname + '/../data/badzcash.conf')), existsSync: sinon.stub().returns(true), }, mkdirp: { @@ -466,7 +466,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new TestBitcoin(config); - bitcoind.options.spawn.datadir = '/tmp/.bitcoin'; + bitcoind.options.spawn.datadir = '/tmp/.zcash'; var node = {}; bitcoind._loadSpawnConfiguration(node); }); @@ -480,7 +480,7 @@ describe('Bitcoin Service', function() { afterEach(function() { sandbox.restore(); }); - it('should warn the user if reindex is set to 1 in the bitcoin.conf file', function() { + it('should warn the user if reindex is set to 1 in the zcash.conf file', function() { var bitcoind = new BitcoinService(baseConfig); var config = { txindex: 1, @@ -829,7 +829,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new BitcoinService(config); - bitcoind._getNetworkConfigPath().should.equal('testnet3/bitcoin.conf'); + bitcoind._getNetworkConfigPath().should.equal('testnet3/zcash.conf'); }); it('will get default rpc port for regtest', function() { bitcore.Networks.enableRegtest(); @@ -843,7 +843,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new BitcoinService(config); - bitcoind._getNetworkConfigPath().should.equal('regtest/bitcoin.conf'); + bitcoind._getNetworkConfigPath().should.equal('regtest/zcash.conf'); }); }); @@ -1749,7 +1749,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.configPath = 'testdir/zcash.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; @@ -1766,7 +1766,7 @@ describe('Bitcoin Service', function() { spawn.callCount.should.equal(1); spawn.args[0][0].should.equal('testexec'); spawn.args[0][1].should.deep.equal([ - '--conf=testdir/bitcoin.conf', + '--conf=testdir/zcash.conf', '--datadir=testdir', '--testnet' ]); @@ -1800,7 +1800,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1838,7 +1838,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1885,7 +1885,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/bitcoin.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1924,7 +1924,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.configPath = 'testdir/zcash.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; @@ -1954,7 +1954,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/bitcoin.conf'; + bitcoind.spawn.configPath = 'testdir/zcash.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; From 3ecbe216970e49a39ff6f4f2b37a727de1da3d78 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sun, 28 Aug 2016 14:50:53 +1200 Subject: [PATCH 263/299] Zcash-ify console output --- benchmarks/index.js | 6 +++--- lib/scaffold/start.js | 4 ++-- lib/services/bitcoind.js | 46 ++++++++++++++++++++-------------------- regtest/bitcoind.js | 6 +++--- regtest/p2p.js | 4 ++-- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/benchmarks/index.js b/benchmarks/index.js index 29bc34ed..8e5973ce 100644 --- a/benchmarks/index.js +++ b/benchmarks/index.js @@ -5,7 +5,7 @@ var bitcoin = require('bitcoin'); var async = require('async'); var maxTime = 20; -console.log('Bitcoin Service native interface vs. Bitcoin JSON RPC interface'); +console.log('Zcash Service native interface vs. Zcash JSON RPC interface'); console.log('----------------------------------------------------------------------'); // To run the benchmarks a fully synced Bitcore Core directory is needed. The RPC comands @@ -43,12 +43,12 @@ bitcoind.start(function(err) { if (err) { throw err; } - console.log('Bitcoin Core started'); + console.log('Zcash started'); }); bitcoind.on('ready', function() { - console.log('Bitcoin Core ready'); + console.log('Zcash ready'); var client = new bitcoin.Client({ host: 'localhost', diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index b6f9bc28..27248941 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -22,8 +22,8 @@ function checkConfigVersion2(fullConfig) { if (!datadirUndefined || addressDefined || dbDefined) { console.warn('\nConfiguration file is not compatible with this version. \n' + - 'A reindex for bitcoind is necessary for this upgrade with the "reindex=1" bitcoin.conf option. \n' + - 'There are changes necessary in both bitcoin.conf and bitcore-node.json. \n\n' + + 'A reindex for zcashd is necessary for this upgrade with the "reindex=1" zcash.conf option. \n' + + 'There are changes necessary in both zcash.conf and bitcore-node.json. \n\n' + 'To upgrade please see the details below and documentation at: \n' + 'https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md \n'); diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 2dc9899d..36233e70 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -331,16 +331,16 @@ Bitcoin.prototype._expandRelativeDatadir = function() { Bitcoin.prototype._loadSpawnConfiguration = function(node) { /* jshint maxstatements: 25 */ - $.checkArgument(this.options.spawn, 'Please specify "spawn" in bitcoind config options'); - $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in bitcoind config options'); - $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in bitcoind config options'); + $.checkArgument(this.options.spawn, 'Please specify "spawn" in zcashd config options'); + $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in zcashd config options'); + $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in zcashd config options'); this._expandRelativeDatadir(); var spawnOptions = this.options.spawn; var configPath = path.resolve(spawnOptions.datadir, './zcash.conf'); - log.info('Using bitcoin config file:', configPath); + log.info('Using zcash config file:', configPath); this.spawn = {}; this.spawn.datadir = this.options.spawn.datadir; @@ -395,29 +395,29 @@ Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { $.checkState( spawnConfig.server && spawnConfig.server === 1, - '"server" option is required to communicate to bitcoind from bitcore. ' + + '"server" option is required to communicate to zcashd from bitcore. ' + 'Please add "server=1" to your configuration and restart' ); $.checkState( spawnConfig.zmqpubrawtx, - '"zmqpubrawtx" option is required to get event updates from bitcoind. ' + + '"zmqpubrawtx" option is required to get event updates from zcashd. ' + 'Please add "zmqpubrawtx=tcp://127.0.0.1:" to your configuration and restart' ); $.checkState( spawnConfig.zmqpubhashblock, - '"zmqpubhashblock" option is required to get event updates from bitcoind. ' + + '"zmqpubhashblock" option is required to get event updates from zcashd. ' + 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' ); $.checkState( (spawnConfig.zmqpubhashblock === spawnConfig.zmqpubrawtx), - '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in bitcoin.conf' + '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in zcash.conf' ); if (spawnConfig.reindex && spawnConfig.reindex === 1) { - log.warn('Reindex option is currently enabled. This means that bitcoind is undergoing a reindex. ' + + log.warn('Reindex option is currently enabled. This means that zcashd is undergoing a reindex. ' + 'The reindex flag will start the index from beginning every time the node is started, so it ' + 'should be removed after the reindex has been initiated. Once the reindex is complete, the rest ' + 'of bitcore-node services will start.'); @@ -477,7 +477,7 @@ Bitcoin.prototype._initChain = function(callback) { } self.genesisBuffer = blockBuffer; self.emit('ready'); - log.info('Bitcoin Daemon Ready'); + log.info('Zcash Daemon Ready'); callback(); }); }); @@ -581,7 +581,7 @@ Bitcoin.prototype._updateTip = function(node, message) { if (Math.round(percentage) >= 100) { self.emit('synced', self.height); } - log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); + log.info('Zcash Height:', self.height, 'Percentage:', percentage.toFixed(2)); } }); } @@ -759,7 +759,7 @@ Bitcoin.prototype._checkReindex = function(node, callback) { } var percentSynced = response.result.verificationprogress * 100; - log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); + log.info('Zcash Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); if (Math.round(percentSynced) >= 100) { node._reindex = false; @@ -812,11 +812,11 @@ Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { return callback(null); } try { - log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); + log.warn('Stopping existing spawned zcash process with pid: ' + pid); self._process.kill(pid, 'SIGINT'); } catch(err) { if (err && err.code === 'ESRCH') { - log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid); + log.warn('Unclean zcash process shutdown, process not found with pid: ' + pid); return callback(null); } else if(err) { return callback(err); @@ -858,7 +858,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { return callback(err); } - log.info('Starting bitcoin process'); + log.info('Starting zcash process'); self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'}); self.spawn.process.on('error', function(err) { @@ -867,14 +867,14 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { self.spawn.process.once('exit', function(code) { if (!self.node.stopping) { - log.warn('Bitcoin process unexpectedly exited with code:', code); - log.warn('Restarting bitcoin child process in ' + self.spawnRestartTime + 'ms'); + log.warn('Zcash process unexpectedly exited with code:', code); + log.warn('Restarting zcash child process in ' + self.spawnRestartTime + 'ms'); setTimeout(function() { self._spawnChildProcess(function(err) { if (err) { return self.emit('error', err); } - log.warn('Bitcoin process restarted'); + log.warn('Zcash process restarted'); }); }, self.spawnRestartTime); } @@ -903,7 +903,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { return callback(err); } if (exitShutdown) { - return callback(new Error('Stopping while trying to spawn bitcoind.')); + return callback(new Error('Stopping while trying to spawn zcashd.')); } self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); @@ -949,7 +949,7 @@ Bitcoin.prototype._connectProcess = function(config, callback) { return callback(err); } if (exitShutdown) { - return callback(new Error('Stopping while trying to connect to bitcoind.')); + return callback(new Error('Stopping while trying to connect to zcashd.')); } self._initZmqSubSocket(node, config.zmqpubrawtx); @@ -1000,7 +1000,7 @@ Bitcoin.prototype.start = function(callback) { return callback(err); } if (self.nodes.length === 0) { - return callback(new Error('Bitcoin configuration options "spawn" or "connect" are expected')); + return callback(new Error('Zcash configuration options "spawn" or "connect" are expected')); } self._initChain(callback); }); @@ -2080,7 +2080,7 @@ Bitcoin.prototype.stop = function(callback) { if (!exited) { exited = true; if (code !== 0) { - var error = new Error('bitcoind spawned process exited with status code: ' + code); + var error = new Error('zcashd spawned process exited with status code: ' + code); error.code = code; return callback(error); } else { @@ -2092,7 +2092,7 @@ Bitcoin.prototype.stop = function(callback) { setTimeout(function() { if (!exited) { exited = true; - return callback(new Error('bitcoind process did not exit')); + return callback(new Error('zcashd process did not exit')); } }, this.shutdownTimeout).unref(); } else { diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index fb9cb7c5..fc6e9d56 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -26,7 +26,7 @@ var coinbasePrivateKey; var privateKey = bitcore.PrivateKey(); var destKey = bitcore.PrivateKey(); -describe('Bitcoind Functionality', function() { +describe('Zcashd Functionality', function() { before(function(done) { this.timeout(60000); @@ -60,10 +60,10 @@ describe('Bitcoind Functionality', function() { log.error('error="%s"', err.message); }); - log.info('Waiting for Bitcoin Core to initialize...'); + log.info('Waiting for Zcash to initialize...'); bitcoind.start(function() { - log.info('Bitcoind started'); + log.info('Zcashd started'); client = new BitcoinRPC({ protocol: 'http', diff --git a/regtest/p2p.js b/regtest/p2p.js index cc6b61b9..95c759ce 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -63,13 +63,13 @@ describe('P2P Functionality', function() { log.error('error="%s"', err.message); }); - log.info('Waiting for Bitcoin Core to initialize...'); + log.info('Waiting for Zcash to initialize...'); bitcoind.start(function(err) { if (err) { throw err; } - log.info('Bitcoind started'); + log.info('Zcashd started'); client = new BitcoinRPC({ protocol: 'http', From 433a08d070178eb9c7aaa0cd6a45a85bf64f77b5 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 29 Aug 2016 21:01:16 +1200 Subject: [PATCH 264/299] Add JoinSplit public values to getDetailedTransaction() --- lib/services/bitcoind.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 36233e70..fcab1bfd 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1947,6 +1947,20 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { } } + function addJoinSplitsToTx(tx, result) { + tx.joinSplits = []; + var netJoinSplitZatoshis = 0; + for (var jsIndex = 0; jsIndex < result.vjoinsplit.length; jsIndex++) { + var jsdesc = result.vjoinsplit[jsIndex]; + netJoinSplitZatoshis += jsdesc.vpub_newZat - jsdesc.vpub_oldZat; + tx.joinSplits.push({ + oldZatoshis: jsdesc.vpub_oldZat, + newZatoshis: jsdesc.vpub_newZat, + }); + } + return netJoinSplitZatoshis; + } + if (tx) { return setImmediate(function() { callback(null, tx); @@ -1975,8 +1989,13 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { addInputsToTx(tx, result); addOutputsToTx(tx, result); + var netJoinSplitZatoshis = 0; + if (tx.version >= 2) { + netJoinSplitZatoshis = addJoinSplitsToTx(tx, result); + } + if (!tx.coinbase) { - tx.feeSatoshis = tx.inputSatoshis - tx.outputSatoshis; + tx.feeSatoshis = tx.inputSatoshis - tx.outputSatoshis + netJoinSplitZatoshis; } else { tx.feeSatoshis = 0; } From f5b864f29e1e1a41f22db99e542f725d2b3b1523 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 30 Aug 2016 15:42:10 +1200 Subject: [PATCH 265/299] Remove hosted download --- package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package.json b/package.json index 69630935..54b33851 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,9 @@ } ], "bin": { - "bitcore-node": "./bin/bitcore-node", - "bitcoind": "./bin/bitcoind" + "bitcore-node": "./bin/bitcore-node" }, "scripts": { - "preinstall": "./scripts/download", - "verify": "./scripts/download --skip-bitcoin-download --verify-bitcoin-download", "test": "mocha -R spec --recursive", "regtest": "./scripts/regtest", "jshint": "jshint --reporter=node_modules/jshint-stylish ./lib", From 9212ef3c9e155f11da868d1b8203207658bef7af Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 30 Aug 2016 17:15:45 +1200 Subject: [PATCH 266/299] Create nodes using GitHub links to bitcore-*-zcash packages --- lib/scaffold/create.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index 9a6603fb..2e465e6d 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -19,8 +19,8 @@ var BASE_PACKAGE = { license: 'MIT', readme: 'README.md', dependencies: { - 'bitcore-lib-zcash': '^' + bitcore.version, - 'bitcore-node-zcash': version + 'bitcore-lib-zcash': 'str4d/bitcore-lib-zcash', + 'bitcore-node-zcash': 'str4d/bitcore-node-zcash' } }; From f1e821dc05214355312cfd426b8832d805491709 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sat, 26 May 2018 18:16:28 +0200 Subject: [PATCH 267/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 54b33851..b31aa174 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { - "name": "bitcore-node-zcash", - "description": "Full node with extended capabilities using Bitcore and Zcash", + "name": "bitcore-node-zero", + "description": "Full node with extended capabilities using Bitcore and Zero", "author": "BitPay ", "version": "3.1.2", "main": "./index.js", - "repository": "git://github.com/str4d/bitcore-node-zcash.git", - "homepage": "https://github.com/str4d/bitcore-node-zcash", + "repository": "git://github.com/ProphetAlgorithms/bitcore-node-zero.git", + "homepage": "https://github.com/ProphetAlgorithms/bitcore-node-zero", "bugs": { "url": "https://github.com/bitpay/bitcore-node/issues" }, @@ -41,13 +41,13 @@ "coveralls": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- --recursive -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js" }, "tags": [ - "zcash", + "zero", "zcashd" ], "dependencies": { "async": "^1.3.0", "bitcoind-rpc": "^0.6.0", - "bitcore-lib-zcash": "str4d/bitcore-lib-zcash", + "bitcore-lib-zero": "ProphetAlgorithms/bitcore-lib-zero", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", From 4a279a457d5ac89448ed96819ba90f274979ffef Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sat, 26 May 2018 18:22:38 +0200 Subject: [PATCH 268/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 670d50d7..a9c5b377 100644 --- a/index.js +++ b/index.js @@ -24,4 +24,4 @@ module.exports.cli.daemon = require('./lib/cli/daemon'); module.exports.cli.bitcore = require('./lib/cli/bitcore'); module.exports.cli.bitcored = require('./lib/cli/bitcored'); -module.exports.lib = require('bitcore-lib-zcash'); +module.exports.lib = require('bitcore-lib-zero'); From 567a37dfc5e69b8f3ca6ba040312c46a3e236512 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sat, 26 May 2018 19:47:56 +0200 Subject: [PATCH 269/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0df35b5a..daf6b77a 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ A Bitcoin full node for building applications and services with Node.js. A node ## Install ```bash -npm install -g bitcore-node -bitcore-node start +npm install ProphetAlgorithms/bitcore-node-zero +./node_modules/bitcore-node-zero/bin/bitcore-node start ``` Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore). @@ -25,10 +25,10 @@ Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and Bitcore includes a Command Line Interface (CLI) for managing, configuring and interfacing with your Bitcore Node. ```bash -bitcore-node create -d mynode +./node_modules/bitcore-node-zero/bin/bitcore-node create -d mynode cd mynode -bitcore-node install -bitcore-node install https://github.com/yourname/helloworld +./node_modules/bitcore-node-zero/bin/bitcore-node install +./node_modules/bitcore-node-zero/bin/bitcore-node install https://github.com/yourname/helloworld ``` This will create a directory with configuration files for your node and install the necessary dependencies. For more information about (and developing) services, please see the [Service Documentation](docs/services.md). @@ -37,8 +37,8 @@ This will create a directory with configuration files for your node and install There are several add-on services available to extend the functionality of Bitcore: -- [Insight API](https://github.com/bitpay/insight-api) -- [Insight UI](https://github.com/bitpay/insight-ui) +- [Insight API Zero](https://github.com/ProphetAlgorithms/insight-api-zero) +- [Insight UI Zero](https://github.com/ProphetAlgorithms/insight-ui-zero) - [Bitcore Wallet Service](https://github.com/bitpay/bitcore-wallet-service) ## Documentation From 4fbee6b900adaf368a28d8b9786ef1ced00457a9 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:08:19 +0200 Subject: [PATCH 270/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/services/bitcoind.js | 71 ++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index fcab1bfd..312f0b6a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -5,7 +5,7 @@ var path = require('path'); var spawn = require('child_process').spawn; var util = require('util'); var mkdirp = require('mkdirp'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var zmq = require('zmq'); var async = require('async'); var LRU = require('lru-cache'); @@ -78,18 +78,31 @@ Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY = 5; Bitcoin.DEFAULT_CONFIG_SETTINGS = { - server: 1, - whitelist: '127.0.0.1', - txindex: 1, - addressindex: 1, - timestampindex: 1, - spentindex: 1, + rpcuser: 'ZeroNodeUsername', + rpcpassword: 'ZeroNodePassword', + rpcport: '23800', + addnode: '213.239.212.246:23801', + addnode: '64.237.50.236:23801', + addnode: '51.255.95.53:23801', + addnode: '79.137.70.151:23801', + addnode: '178.213.233.173:23801', + addnode: '86.31.59.86:23801', + addnode: '138.197.149.3:23801', + addnode: '145.239.71.6:23801', + addnode: '174.138.15.68:23801', + addnode: 'zeroseed.cryptoforge.cc:23801', + addnode: '84.19.36.203:23801', + addnode: 'zerocurrency.prophetalgorithms.com:23801', + txindex: '1', + addressindex: '1', + timestampindex: '1', + spentindex: '1', zmqpubrawtx: 'tcp://127.0.0.1:28332', zmqpubhashblock: 'tcp://127.0.0.1:28332', rpcallowip: '127.0.0.1', - rpcuser: 'bitcoin', - rpcpassword: 'local321', - uacomment: 'bitcore' + uacomment: 'bitcore', + server: '1', + showmetrics: '0' }; Bitcoin.prototype._initDefaults = function(options) { @@ -338,9 +351,9 @@ Bitcoin.prototype._loadSpawnConfiguration = function(node) { this._expandRelativeDatadir(); var spawnOptions = this.options.spawn; - var configPath = path.resolve(spawnOptions.datadir, './zcash.conf'); + var configPath = path.resolve(spawnOptions.datadir, './zero.conf'); - log.info('Using zcash config file:', configPath); + log.info('Using zero config file:', configPath); this.spawn = {}; this.spawn.datadir = this.options.spawn.datadir; @@ -413,7 +426,7 @@ Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { $.checkState( (spawnConfig.zmqpubhashblock === spawnConfig.zmqpubrawtx), - '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in zcash.conf' + '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in zero.conf' ); if (spawnConfig.reindex && spawnConfig.reindex === 1) { @@ -477,7 +490,7 @@ Bitcoin.prototype._initChain = function(callback) { } self.genesisBuffer = blockBuffer; self.emit('ready'); - log.info('Zcash Daemon Ready'); + log.info('Zero Daemon Ready'); callback(); }); }); @@ -488,10 +501,10 @@ Bitcoin.prototype._initChain = function(callback) { Bitcoin.prototype._getDefaultConf = function() { var networkOptions = { - rpcport: 8232 + rpcport: 23800 }; if (this.node.network === bitcore.Networks.testnet) { - networkOptions.rpcport = 18232; + networkOptions.rpcport = 23802; } return networkOptions; }; @@ -499,9 +512,9 @@ Bitcoin.prototype._getDefaultConf = function() { Bitcoin.prototype._getNetworkConfigPath = function() { var networkPath; if (this.node.network === bitcore.Networks.testnet) { - networkPath = 'testnet3/zcash.conf'; + networkPath = 'testnet3/zero.conf'; if (this.node.network.regtestEnabled) { - networkPath = 'regtest/zcash.conf'; + networkPath = 'regtest/zero.conf'; } } return networkPath; @@ -581,7 +594,7 @@ Bitcoin.prototype._updateTip = function(node, message) { if (Math.round(percentage) >= 100) { self.emit('synced', self.height); } - log.info('Zcash Height:', self.height, 'Percentage:', percentage.toFixed(2)); + log.info('Zero Height:', self.height, 'Percentage:', percentage.toFixed(2)); } }); } @@ -759,7 +772,7 @@ Bitcoin.prototype._checkReindex = function(node, callback) { } var percentSynced = response.result.verificationprogress * 100; - log.info('Zcash Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); + log.info('Zero Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); if (Math.round(percentSynced) >= 100) { node._reindex = false; @@ -812,11 +825,11 @@ Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { return callback(null); } try { - log.warn('Stopping existing spawned zcash process with pid: ' + pid); + log.warn('Stopping existing spawned zero process with pid: ' + pid); self._process.kill(pid, 'SIGINT'); } catch(err) { if (err && err.code === 'ESRCH') { - log.warn('Unclean zcash process shutdown, process not found with pid: ' + pid); + log.warn('Unclean zero process shutdown, process not found with pid: ' + pid); return callback(null); } else if(err) { return callback(err); @@ -858,7 +871,7 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { return callback(err); } - log.info('Starting zcash process'); + log.info('Starting zero process'); self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'}); self.spawn.process.on('error', function(err) { @@ -867,14 +880,14 @@ Bitcoin.prototype._spawnChildProcess = function(callback) { self.spawn.process.once('exit', function(code) { if (!self.node.stopping) { - log.warn('Zcash process unexpectedly exited with code:', code); - log.warn('Restarting zcash child process in ' + self.spawnRestartTime + 'ms'); + log.warn('Zero process unexpectedly exited with code:', code); + log.warn('Restarting zero child process in ' + self.spawnRestartTime + 'ms'); setTimeout(function() { self._spawnChildProcess(function(err) { if (err) { return self.emit('error', err); } - log.warn('Zcash process restarted'); + log.warn('Zero process restarted'); }); }, self.spawnRestartTime); } @@ -1000,7 +1013,7 @@ Bitcoin.prototype.start = function(callback) { return callback(err); } if (self.nodes.length === 0) { - return callback(new Error('Zcash configuration options "spawn" or "connect" are expected')); + return callback(new Error('Zero configuration options "spawn" or "connect" are expected')); } self._initChain(callback); }); @@ -1930,13 +1943,13 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { tx.outputSatoshis = 0; for(var outputIndex = 0; outputIndex < result.vout.length; outputIndex++) { var out = result.vout[outputIndex]; - tx.outputSatoshis += out.valueSat; + tx.outputSatoshis += out.valueZat; var address = null; if (out.scriptPubKey && out.scriptPubKey.addresses && out.scriptPubKey.addresses.length === 1) { address = out.scriptPubKey.addresses[0]; } tx.outputs.push({ - satoshis: out.valueSat, + satoshis: out.valueZat, script: out.scriptPubKey.hex, scriptAsm: out.scriptPubKey.asm, spentTxId: out.spentTxId, From 3c9ec4d0d8438bfbfb4c49dc1d03bd023f290bd8 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:11:23 +0200 Subject: [PATCH 271/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/services/web.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/web.js b/lib/services/web.js index 8d1a0228..08ce7620 100644 --- a/lib/services/web.js +++ b/lib/services/web.js @@ -9,7 +9,7 @@ var socketio = require('socket.io'); var inherits = require('util').inherits; var BaseService = require('../service'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var _ = bitcore.deps._; var index = require('../'); var log = index.log; From ac43e891ae73e24666e9c7c754d0f6fdf90371e1 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:12:51 +0200 Subject: [PATCH 272/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/add.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scaffold/add.js b/lib/scaffold/add.js index 1f2d3769..921cc0cb 100644 --- a/lib/scaffold/add.js +++ b/lib/scaffold/add.js @@ -4,7 +4,7 @@ var async = require('async'); var fs = require('fs'); var path = require('path'); var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var utils = require('../utils'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; From ea5735a9fc4329e95a7162a99648fd9d4cb6402a Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:15:14 +0200 Subject: [PATCH 273/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/create.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index 2e465e6d..dd9f9c40 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -1,7 +1,7 @@ 'use strict'; var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var async = require('async'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; @@ -14,13 +14,13 @@ var defaultBaseConfig = require('./default-base-config'); var version = '^' + packageFile.version; var BASE_PACKAGE = { - description: 'A full Zcash node build with Bitcore', + description: 'A full Zero node build with Bitcore', repository: 'https://github.com/user/project', license: 'MIT', readme: 'README.md', dependencies: { - 'bitcore-lib-zcash': 'str4d/bitcore-lib-zcash', - 'bitcore-node-zcash': 'str4d/bitcore-node-zcash' + 'bitcore-lib-zero': 'ProphetAlgorithms/bitcore-lib-zero', + 'bitcore-node-zero': 'ProphetAlgorithms/bitcore-node-zero' } }; From cac59c2e30392c6a9e4093bfc5a6a63de3130e04 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:16:53 +0200 Subject: [PATCH 274/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/default-base-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scaffold/default-base-config.js b/lib/scaffold/default-base-config.js index 4f641b2c..bdcd8b4a 100644 --- a/lib/scaffold/default-base-config.js +++ b/lib/scaffold/default-base-config.js @@ -22,7 +22,7 @@ function getDefaultBaseConfig(options) { servicesConfig: { bitcoind: { spawn: { - datadir: options.datadir || path.resolve(process.env.HOME, '.zcash'), + datadir: options.datadir || path.resolve(process.env.HOME, '.zero'), exec: path.resolve(__dirname, '../../bin/zcashd') } } From 5564e1da9672c9022e65ce793432d040b45acff0 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:20:12 +0200 Subject: [PATCH 275/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/find-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scaffold/find-config.js b/lib/scaffold/find-config.js index 7ac7e359..235969a4 100644 --- a/lib/scaffold/find-config.js +++ b/lib/scaffold/find-config.js @@ -1,6 +1,6 @@ 'use strict'; -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var path = require('path'); From 8a0a49bf8f749671b3757dfbd30ae58acf4dc911 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:21:35 +0200 Subject: [PATCH 276/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/remove.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scaffold/remove.js b/lib/scaffold/remove.js index db7b4da2..e4f30d3c 100644 --- a/lib/scaffold/remove.js +++ b/lib/scaffold/remove.js @@ -5,7 +5,7 @@ var fs = require('fs'); var npm = require('npm'); var path = require('path'); var spawn = require('child_process').spawn; -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var utils = require('../utils'); From df255d245ca2f162605dce87472d859be0f100f5 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:23:10 +0200 Subject: [PATCH 277/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/scaffold/start.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 27248941..38fd8710 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -3,7 +3,7 @@ var path = require('path'); var BitcoreNode = require('../node'); var index = require('../'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var _ = bitcore.deps._; var log = index.log; var shuttingDown = false; @@ -22,8 +22,8 @@ function checkConfigVersion2(fullConfig) { if (!datadirUndefined || addressDefined || dbDefined) { console.warn('\nConfiguration file is not compatible with this version. \n' + - 'A reindex for zcashd is necessary for this upgrade with the "reindex=1" zcash.conf option. \n' + - 'There are changes necessary in both zcash.conf and bitcore-node.json. \n\n' + + 'A reindex for zcashd is necessary for this upgrade with the "reindex=1" zero.conf option. \n' + + 'There are changes necessary in both zero.conf and bitcore-node.json. \n\n' + 'To upgrade please see the details below and documentation at: \n' + 'https://github.com/bitpay/bitcore-node/blob/bitcoind/docs/upgrade.md \n'); From bb6df202da14e1e4b7b6772d4c130a324f0a93ad Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:29:02 +0200 Subject: [PATCH 278/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logger.js b/lib/logger.js index edbc5c79..f28e7615 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,6 +1,6 @@ 'use strict'; -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var _ = bitcore.deps._; var colors = require('colors/safe'); From 1dbd1a29b953844e89b57315a4d8ed4c77163d43 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:30:19 +0200 Subject: [PATCH 279/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- lib/node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/node.js b/lib/node.js index 887c34a1..1827a312 100644 --- a/lib/node.js +++ b/lib/node.js @@ -3,7 +3,7 @@ var util = require('util'); var EventEmitter = require('events').EventEmitter; var async = require('async'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var Networks = bitcore.Networks; var $ = bitcore.util.preconditions; var _ = bitcore.deps._; From 3655d06ca854401bbf52cfa0c11e0597929f2785 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:33:25 +0200 Subject: [PATCH 280/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- benchmarks/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/index.js b/benchmarks/index.js index 8e5973ce..3d11e7e9 100644 --- a/benchmarks/index.js +++ b/benchmarks/index.js @@ -5,7 +5,7 @@ var bitcoin = require('bitcoin'); var async = require('async'); var maxTime = 20; -console.log('Zcash Service native interface vs. Zcash JSON RPC interface'); +console.log('Zero Service native interface vs. Zero JSON RPC interface'); console.log('----------------------------------------------------------------------'); // To run the benchmarks a fully synced Bitcore Core directory is needed. The RPC comands @@ -28,7 +28,7 @@ var fixtureData = { var bitcoind = require('../').services.Bitcoin({ node: { - datadir: process.env.HOME + '/.zcash', + datadir: process.env.HOME + '/.zero', network: { name: 'testnet' } @@ -43,12 +43,12 @@ bitcoind.start(function(err) { if (err) { throw err; } - console.log('Zcash started'); + console.log('Zero started'); }); bitcoind.on('ready', function() { - console.log('Zcash ready'); + console.log('Zero ready'); var client = new bitcoin.Client({ host: 'localhost', From 6cde057a4bafb552d0818058f7aa7151a2dc387e Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:42:35 +0200 Subject: [PATCH 281/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- regtest/bitcoind.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regtest/bitcoind.js b/regtest/bitcoind.js index fc6e9d56..7e79e36c 100644 --- a/regtest/bitcoind.js +++ b/regtest/bitcoind.js @@ -7,7 +7,7 @@ var index = require('..'); var log = index.log; var chai = require('chai'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var BN = bitcore.crypto.BN; var async = require('async'); var rimraf = require('rimraf'); @@ -60,7 +60,7 @@ describe('Zcashd Functionality', function() { log.error('error="%s"', err.message); }); - log.info('Waiting for Zcash to initialize...'); + log.info('Waiting for Zero to initialize...'); bitcoind.start(function() { log.info('Zcashd started'); From 517602c208e70f6bef715b4eb91aa9f2eca1515c Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:43:42 +0200 Subject: [PATCH 282/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- regtest/cluster.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/regtest/cluster.js b/regtest/cluster.js index 56dc1d0c..8bcbc736 100644 --- a/regtest/cluster.js +++ b/regtest/cluster.js @@ -6,7 +6,7 @@ var spawn = require('child_process').spawn; var BitcoinRPC = require('bitcoind-rpc'); var rimraf = require('rimraf'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var chai = require('chai'); var should = chai.should(); @@ -23,7 +23,7 @@ describe('Bitcoin Cluster', function() { var nodesConf = [ { datadir: path.resolve(__dirname, './data/node1'), - conf: path.resolve(__dirname, './data/node1/zcash.conf'), + conf: path.resolve(__dirname, './data/node1/zero.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30521, @@ -32,7 +32,7 @@ describe('Bitcoin Cluster', function() { }, { datadir: path.resolve(__dirname, './data/node2'), - conf: path.resolve(__dirname, './data/node2/zcash.conf'), + conf: path.resolve(__dirname, './data/node2/zero.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30522, @@ -41,7 +41,7 @@ describe('Bitcoin Cluster', function() { }, { datadir: path.resolve(__dirname, './data/node3'), - conf: path.resolve(__dirname, './data/node3/zcash.conf'), + conf: path.resolve(__dirname, './data/node3/zero.conf'), rpcuser: 'bitcoin', rpcpassword: 'local321', rpcport: 30523, From b22cea6697359d8028214474c422b7d54a259941 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:45:09 +0200 Subject: [PATCH 283/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- regtest/node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regtest/node.js b/regtest/node.js index c5febe19..272696da 100644 --- a/regtest/node.js +++ b/regtest/node.js @@ -9,7 +9,7 @@ var log = index.log; log.debug = function() {}; var chai = require('chai'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var rimraf = require('rimraf'); var node; From c4d4729a532621df98275cb29a2d6455abd318b5 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:46:20 +0200 Subject: [PATCH 284/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- regtest/p2p.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regtest/p2p.js b/regtest/p2p.js index 95c759ce..c83fd639 100644 --- a/regtest/p2p.js +++ b/regtest/p2p.js @@ -10,7 +10,7 @@ var p2p = require('bitcore-p2p'); var Peer = p2p.Peer; var Messages = p2p.Messages; var chai = require('chai'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var Transaction = bitcore.Transaction; var BN = bitcore.crypto.BN; var async = require('async'); @@ -63,7 +63,7 @@ describe('P2P Functionality', function() { log.error('error="%s"', err.message); }); - log.info('Waiting for Zcash to initialize...'); + log.info('Waiting for Zero to initialize...'); bitcoind.start(function(err) { if (err) { From 7f5b21f21e7e60b9376f54027bc918de4fe7d593 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:49:08 +0200 Subject: [PATCH 285/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/scaffold/add.integration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scaffold/add.integration.js b/test/scaffold/add.integration.js index fa5b8d5f..4a34e6b0 100644 --- a/test/scaffold/add.integration.js +++ b/test/scaffold/add.integration.js @@ -94,7 +94,7 @@ describe('#add', function() { var callCount = 0; var oldPackage = { dependencies: { - 'bitcore-lib-zcash': '^v0.13.7', + 'bitcore-lib-zero': '^v0.13.7', 'bitcore-node': '^v0.2.0' } }; From 6eb796d6ad83f94caa4c43418aa5fbcdad975606 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:53:05 +0200 Subject: [PATCH 286/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/scaffold/create.integration.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/scaffold/create.integration.js b/test/scaffold/create.integration.js index ccf7c742..bc57d1ff 100644 --- a/test/scaffold/create.integration.js +++ b/test/scaffold/create.integration.js @@ -33,7 +33,7 @@ describe('#create', function() { if (err) { throw err; } - mkdirp(testDir + '/.zcash', function(err) { + mkdirp(testDir + '/.zero', function(err) { if (err) { throw err; } @@ -104,7 +104,7 @@ describe('#create', function() { dirname: 'mynode3', name: 'My Node 3', isGlobal: true, - datadir: '../.zcash' + datadir: '../.zero' }, function(err) { if (err) { throw err; @@ -139,7 +139,7 @@ describe('#create', function() { dirname: 'mynode4', name: 'My Node 4', isGlobal: false, - datadir: '../.zcash' + datadir: '../.zero' }, function(err) { should.exist(err); err.message.should.equal('There was an error installing dependencies.'); From df692057b8a89475c11aadb2d021b838328bd5c2 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:54:08 +0200 Subject: [PATCH 287/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/scaffold/default-base-config.integration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scaffold/default-base-config.integration.js b/test/scaffold/default-base-config.integration.js index 72b994f8..03f87be5 100644 --- a/test/scaffold/default-base-config.integration.js +++ b/test/scaffold/default-base-config.integration.js @@ -14,7 +14,7 @@ describe('#defaultBaseConfig', function() { info.config.port.should.equal(3001); info.config.services.should.deep.equal(['bitcoind', 'web']); var bitcoind = info.config.servicesConfig.bitcoind; - bitcoind.spawn.datadir.should.equal(home + '/.zcash'); + bitcoind.spawn.datadir.should.equal(home + '/.zero'); bitcoind.spawn.exec.should.equal(path.resolve(__dirname, '../../bin/zcashd')); }); it('be able to specify a network', function() { From 06cc5b5fe20d59db76b1a936c353aa81eaefd3b9 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 16:56:54 +0200 Subject: [PATCH 288/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/scaffold/default-config.integration.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/scaffold/default-config.integration.js b/test/scaffold/default-config.integration.js index 9b5338d1..bb6a91cc 100644 --- a/test/scaffold/default-config.integration.js +++ b/test/scaffold/default-config.integration.js @@ -58,8 +58,8 @@ describe('#defaultConfig', function() { services: [ 'bitcoind', 'web', - 'insight-api', - 'insight-ui' + 'insight-api-zero', + 'insight-ui-zero' ], servicesConfig: { bitcoind: { @@ -87,7 +87,7 @@ describe('#defaultConfig', function() { }); var home = process.env.HOME; var info = defaultConfig({ - additionalServices: ['insight-api', 'insight-ui'] + additionalServices: ['insight-api-zero', 'insight-ui-zero'] }); info.path.should.equal(home + '/.bitcore'); info.config.network.should.equal('livenet'); @@ -95,8 +95,8 @@ describe('#defaultConfig', function() { info.config.services.should.deep.equal([ 'bitcoind', 'web', - 'insight-api', - 'insight-ui' + 'insight-api-zero', + 'insight-ui-zero' ]); var bitcoind = info.config.servicesConfig.bitcoind; should.exist(bitcoind); From ede2698c4d01fe9ddac3d20c42af426fd9fa6e38 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 17:06:48 +0200 Subject: [PATCH 289/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/services/bitcoind.unit.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/services/bitcoind.unit.js b/test/services/bitcoind.unit.js index 0a75654f..d5ad33dd 100644 --- a/test/services/bitcoind.unit.js +++ b/test/services/bitcoind.unit.js @@ -6,7 +6,7 @@ var path = require('path'); var EventEmitter = require('events').EventEmitter; var should = require('chai').should(); var crypto = require('crypto'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var _ = bitcore.deps._; var sinon = require('sinon'); var proxyquire = require('proxyquire'); @@ -18,13 +18,13 @@ var log = index.log; var errors = index.errors; var Transaction = bitcore.Transaction; -var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/zcash.conf'))); +var readFileSync = sinon.stub().returns(fs.readFileSync(path.resolve(__dirname, '../data/zero.conf'))); var BitcoinService = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync } }); -var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.zcash.conf'), 'utf8'); +var defaultBitcoinConf = fs.readFileSync(path.resolve(__dirname, '../data/default.zero.conf'), 'utf8'); describe('Bitcoin Service', function() { var txhex = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000'; @@ -357,7 +357,7 @@ describe('Bitcoin Service', function() { afterEach(function() { sandbox.restore(); }); - it('will parse a zcash.conf file', function() { + it('will parse a zero.conf file', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { readFileSync: readFileSync, @@ -369,7 +369,7 @@ describe('Bitcoin Service', function() { } }); var bitcoind = new TestBitcoin(baseConfig); - bitcoind.options.spawn.datadir = '/tmp/.zcash'; + bitcoind.options.spawn.datadir = '/tmp/.zero'; var node = {}; bitcoind._loadSpawnConfiguration(node); should.exist(bitcoind.spawn.config); @@ -423,7 +423,7 @@ describe('Bitcoin Service', function() { it('should throw an exception if txindex isn\'t enabled in the configuration', function() { var TestBitcoin = proxyquire('../../lib/services/bitcoind', { fs: { - readFileSync: sinon.stub().returns(fs.readFileSync(__dirname + '/../data/badzcash.conf')), + readFileSync: sinon.stub().returns(fs.readFileSync(__dirname + '/../data/badzero.conf')), existsSync: sinon.stub().returns(true), }, mkdirp: { @@ -466,7 +466,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new TestBitcoin(config); - bitcoind.options.spawn.datadir = '/tmp/.zcash'; + bitcoind.options.spawn.datadir = '/tmp/.zero'; var node = {}; bitcoind._loadSpawnConfiguration(node); }); @@ -480,7 +480,7 @@ describe('Bitcoin Service', function() { afterEach(function() { sandbox.restore(); }); - it('should warn the user if reindex is set to 1 in the zcash.conf file', function() { + it('should warn the user if reindex is set to 1 in the zero.conf file', function() { var bitcoind = new BitcoinService(baseConfig); var config = { txindex: 1, @@ -829,7 +829,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new BitcoinService(config); - bitcoind._getNetworkConfigPath().should.equal('testnet3/zcash.conf'); + bitcoind._getNetworkConfigPath().should.equal('testnet3/zero.conf'); }); it('will get default rpc port for regtest', function() { bitcore.Networks.enableRegtest(); @@ -843,7 +843,7 @@ describe('Bitcoin Service', function() { } }; var bitcoind = new BitcoinService(config); - bitcoind._getNetworkConfigPath().should.equal('regtest/zcash.conf'); + bitcoind._getNetworkConfigPath().should.equal('regtest/zero.conf'); }); }); @@ -1749,7 +1749,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/zcash.conf'; + bitcoind.spawn.configPath = 'testdir/zero.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; @@ -1766,7 +1766,7 @@ describe('Bitcoin Service', function() { spawn.callCount.should.equal(1); spawn.args[0][0].should.equal('testexec'); spawn.args[0][1].should.deep.equal([ - '--conf=testdir/zcash.conf', + '--conf=testdir/zero.conf', '--datadir=testdir', '--testnet' ]); @@ -1800,7 +1800,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zero.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1838,7 +1838,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zero.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1885,7 +1885,7 @@ describe('Bitcoin Service', function() { bitcoind.spawn = {}; bitcoind.spawn.exec = 'bitcoind'; bitcoind.spawn.datadir = '/tmp/bitcoin'; - bitcoind.spawn.configPath = '/tmp/bitcoin/zcash.conf'; + bitcoind.spawn.configPath = '/tmp/bitcoin/zero.conf'; bitcoind.spawn.config = {}; bitcoind.spawnRestartTime = 1; bitcoind._loadTipFromNode = sinon.stub().callsArg(1); @@ -1924,7 +1924,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/zcash.conf'; + bitcoind.spawn.configPath = 'testdir/zero.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; @@ -1954,7 +1954,7 @@ describe('Bitcoin Service', function() { bitcoind._loadSpawnConfiguration = sinon.stub(); bitcoind.spawn = {}; bitcoind.spawn.exec = 'testexec'; - bitcoind.spawn.configPath = 'testdir/zcash.conf'; + bitcoind.spawn.configPath = 'testdir/zero.conf'; bitcoind.spawn.datadir = 'testdir'; bitcoind.spawn.config = {}; bitcoind.spawn.config.rpcport = 20001; From 5fa9a5dfb903b54d599bd45d5b543d516c68bcdc Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Sun, 27 May 2018 17:11:55 +0200 Subject: [PATCH 290/299] changes for the Zero node enable bitcore-node to work with the Zero node and to create the bitcore-node-zero package --- test/node.unit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/node.unit.js b/test/node.unit.js index 633c2bb5..4a8bc592 100644 --- a/test/node.unit.js +++ b/test/node.unit.js @@ -2,7 +2,7 @@ var should = require('chai').should(); var sinon = require('sinon'); -var bitcore = require('bitcore-lib-zcash'); +var bitcore = require('bitcore-lib-zero'); var Networks = bitcore.Networks; var proxyquire = require('proxyquire'); var util = require('util'); From 378f40be85df4f82b97f4668e03f9f5639b9ed54 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Tue, 29 May 2018 19:35:19 +0200 Subject: [PATCH 291/299] getDefaultConfig: list of objects -> array add multiple options with the same name in the zero configuration file --- lib/services/bitcoind.js | 56 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 312f0b6a..48c95c8a 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -77,33 +77,33 @@ Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY = 5; -Bitcoin.DEFAULT_CONFIG_SETTINGS = { - rpcuser: 'ZeroNodeUsername', - rpcpassword: 'ZeroNodePassword', - rpcport: '23800', - addnode: '213.239.212.246:23801', - addnode: '64.237.50.236:23801', - addnode: '51.255.95.53:23801', - addnode: '79.137.70.151:23801', - addnode: '178.213.233.173:23801', - addnode: '86.31.59.86:23801', - addnode: '138.197.149.3:23801', - addnode: '145.239.71.6:23801', - addnode: '174.138.15.68:23801', - addnode: 'zeroseed.cryptoforge.cc:23801', - addnode: '84.19.36.203:23801', - addnode: 'zerocurrency.prophetalgorithms.com:23801', - txindex: '1', - addressindex: '1', - timestampindex: '1', - spentindex: '1', - zmqpubrawtx: 'tcp://127.0.0.1:28332', - zmqpubhashblock: 'tcp://127.0.0.1:28332', - rpcallowip: '127.0.0.1', - uacomment: 'bitcore', - server: '1', - showmetrics: '0' -}; +Bitcoin.DEFAULT_CONFIG_SETTINGS = [ + "rpcuser=ZeroNodeUsername", + "rpcpassword=ZeroNodePassword", + "rpcport=23800", + "addnode=213.239.212.246:23801", + "addnode=64.237.50.236:23801", + "addnode=51.255.95.53:23801", + "addnode=79.137.70.151:23801", + "addnode=178.213.233.173:23801", + "addnode=86.31.59.86:23801", + "addnode=138.197.149.3:23801", + "addnode=145.239.71.6:23801", + "addnode=174.138.15.68:23801", + "addnode=zeroseed.cryptoforge.cc:23801", + "addnode=84.19.36.203:23801", + "addnode=zerocurrency.prophetalgorithms.com:23801", + "txindex=1", + "addressindex=1", + "timestampindex=1", + "spentindex=1", + "zmqpubrawtx=tcp://127.0.0.1:28332", + "zmqpubhashblock=tcp://127.0.0.1:28332", + "rpcallowip=127.0.0.1", + "uacomment=bitcore", + "server=1", + "showmetrics=0" +]; Bitcoin.prototype._initDefaults = function(options) { /* jshint maxcomplexity: 15 */ @@ -307,7 +307,7 @@ Bitcoin.prototype._getDefaultConfig = function() { var config = ''; var defaults = Bitcoin.DEFAULT_CONFIG_SETTINGS; for(var key in defaults) { - config += key + '=' + defaults[key] + '\n'; + config += defaults[key] + '\n'; } return config; }; From d2eb2af8ba49bbf00bcdc90b18a8f1db200fb0f2 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Fri, 1 Jun 2018 17:19:34 +0200 Subject: [PATCH 292/299] Update README.md for Zero node --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index daf6b77a..2b4c3da8 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,25 @@ npm install ProphetAlgorithms/bitcore-node-zero Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore). +## Install bitcore-node-zero with insight-api-zero and insight-ui-zero + +```bash +npm install ProphetAlgorithms/bitcore-node-zero +./node_modules/bitcore-node-zero/bin/bitcore-node create mynode +cd mynode +./node_modules/bitcore-node-zero/bin/bitcore-node install ProphetAlgorithms/insight-api-zero ProphetAlgorithms/insight-ui-zero +``` +Now change the values of rpcuser and rpcpassword with values of your choice in the file bitcoind.js, located from the base path in: ./mynode/node_modules/bitcore-node-zero/lib/services/ +Copy the executables of the Zero daemon to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ +If you have changed directories, go back to: ./mynode/ and run the command: + +```bash +./node_modules/bitcore-node-zero/bin/bitcore-node start +``` + +Now all the necessary services should be working, wait for the synchronization of all the blocks by the Zero daemon. You can type in the browser's address bar: http://localhost:3001/insight/ , if everything went well you should see the Zero Insight home page. + + ## Prerequisites - GNU/Linux x86_32/x86_64, or OSX 64bit *(for bitcoind distributed binaries)* @@ -39,7 +58,7 @@ There are several add-on services available to extend the functionality of Bitco - [Insight API Zero](https://github.com/ProphetAlgorithms/insight-api-zero) - [Insight UI Zero](https://github.com/ProphetAlgorithms/insight-ui-zero) -- [Bitcore Wallet Service](https://github.com/bitpay/bitcore-wallet-service) + ## Documentation From e1051c9c479d27789e5d13861a1b3c6512b31d1b Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Fri, 1 Jun 2018 17:27:32 +0200 Subject: [PATCH 293/299] Update README.md for Zero node --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2b4c3da8..e2ea832f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ npm install ProphetAlgorithms/bitcore-node-zero Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and x86_64 Mac OS X. Upon npm install, the binaries for your platform will be downloaded. For more detailed installation instructions, or if you want to compile the project yourself, then please see the Bitcore branch of [Bitcoin Core with additional indexing](https://github.com/bitpay/bitcoin/tree/0.12.1-bitcore). -## Install bitcore-node-zero with insight-api-zero and insight-ui-zero +## Install bitcore-node-zero with insight-api-zero and insight-ui-zero (tested with nodejs v4) ```bash npm install ProphetAlgorithms/bitcore-node-zero @@ -20,8 +20,8 @@ npm install ProphetAlgorithms/bitcore-node-zero cd mynode ./node_modules/bitcore-node-zero/bin/bitcore-node install ProphetAlgorithms/insight-api-zero ProphetAlgorithms/insight-ui-zero ``` -Now change the values of rpcuser and rpcpassword with values of your choice in the file bitcoind.js, located from the base path in: ./mynode/node_modules/bitcore-node-zero/lib/services/ -Copy the executables of the Zero daemon to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ +Now change the values of rpcuser and rpcpassword with values of your choice in the file bitcoind.js, located from the base path in: ./mynode/node_modules/bitcore-node-zero/lib/services/ . +Copy the executables of the Zero daemon to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ . If you have changed directories, go back to: ./mynode/ and run the command: ```bash From d7266cbf5575554ef6926b8431a6ddfb52977bf7 Mon Sep 17 00:00:00 2001 From: ProphetAlgorithms <38349760+ProphetAlgorithms@users.noreply.github.com> Date: Fri, 1 Jun 2018 17:38:59 +0200 Subject: [PATCH 294/299] Update README.md for Zero daemon --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e2ea832f..1ab2caa6 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ cd mynode ./node_modules/bitcore-node-zero/bin/bitcore-node install ProphetAlgorithms/insight-api-zero ProphetAlgorithms/insight-ui-zero ``` Now change the values of rpcuser and rpcpassword with values of your choice in the file bitcoind.js, located from the base path in: ./mynode/node_modules/bitcore-node-zero/lib/services/ . -Copy the executables of the Zero daemon to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ . +Copy the executables of the Zero daemon (you need the version of Zero daemon patched with the addition of rpc calls needed to bitcore-node-zero - https://github.com/ProphetAlgorithms/zero-1.0.14-1-bitcore) to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ . If you have changed directories, go back to: ./mynode/ and run the command: ```bash From 39efc07d24f776d47e24b54b31f8afc3a6cffa73 Mon Sep 17 00:00:00 2001 From: Mark Sailor Date: Sun, 28 Oct 2018 17:31:36 -0700 Subject: [PATCH 295/299] Add Support for Overwinter --- lib/services/bitcoind.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 48c95c8a..f60b8747 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1993,6 +1993,7 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { version: result.version, hash: txid, locktime: result.locktime, + fOverwintered: result.overwintered, }; if (result.vin[0] && result.vin[0].coinbase) { @@ -2013,6 +2014,11 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { tx.feeSatoshis = 0; } + if (tx.fOverwintered) { + tx.nVersionGroupId = parseInt(result.versiongroupid, 16); + tx.nExpiryHeight = result.exiryheight; + } + self.transactionDetailedCache.set(txid, tx); done(null, tx); From a9db0cc9885e4bd8c1dd509c26fa602475005369 Mon Sep 17 00:00:00 2001 From: Mark Sailor Date: Sun, 28 Oct 2018 17:31:58 -0700 Subject: [PATCH 296/299] Add Support for Sapling --- lib/services/bitcoind.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index f60b8747..53467612 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -2019,6 +2019,19 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { tx.nExpiryHeight = result.exiryheight; } + // Sapling START + if (tx.fOverwintered && tx.version >= 4) { + tx.valueBalance = result.valueBalance; + tx.spendDescs = result.vShieldedSpend; + tx.outputDescs = result.vShieldedOutput; + if (result.bindingSig) { + tx.bindingSig = result.bindingSig; + } + // Update tx.feeSatoshis with custom explorer JSON field 'valueBalanceZat' + tx.feeSatoshis = tx.feeSatoshis + result.valueBalanceZat; + } + // Sapling END + self.transactionDetailedCache.set(txid, tx); done(null, tx); From 51e05a9259dc9c626d30f038d82aacea7d7f33fc Mon Sep 17 00:00:00 2001 From: Mark Sailor Date: Sun, 28 Oct 2018 17:52:18 -0700 Subject: [PATCH 297/299] Update package, readme, dependencies --- README.md | 12 ++++++------ lib/scaffold/create.js | 4 ++-- package.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1ab2caa6..868ea9e0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Bitcoin full node for building applications and services with Node.js. A node ## Install ```bash -npm install ProphetAlgorithms/bitcore-node-zero +npm install zerocurrencycoin/bitcore-node-zero ./node_modules/bitcore-node-zero/bin/bitcore-node start ``` @@ -15,13 +15,13 @@ Note: For your convenience, we distribute bitcoind binaries for x86_64 Linux and ## Install bitcore-node-zero with insight-api-zero and insight-ui-zero (tested with nodejs v4) ```bash -npm install ProphetAlgorithms/bitcore-node-zero +npm install zerocurrencycoin/bitcore-node-zero ./node_modules/bitcore-node-zero/bin/bitcore-node create mynode cd mynode -./node_modules/bitcore-node-zero/bin/bitcore-node install ProphetAlgorithms/insight-api-zero ProphetAlgorithms/insight-ui-zero +./node_modules/bitcore-node-zero/bin/bitcore-node install zerocurrencycoin/insight-api-zero zerocurrencycoin/insight-ui-zero ``` Now change the values of rpcuser and rpcpassword with values of your choice in the file bitcoind.js, located from the base path in: ./mynode/node_modules/bitcore-node-zero/lib/services/ . -Copy the executables of the Zero daemon (you need the version of Zero daemon patched with the addition of rpc calls needed to bitcore-node-zero - https://github.com/ProphetAlgorithms/zero-1.0.14-1-bitcore) to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ . +Copy the executables of the Zero daemon (you need the version of Zero daemon patched with the addition of rpc calls needed to bitcore-node-zero - https://github.com/zerocurrencycoin/zero-1.0.14-1-bitcore) to the folder located from the base path in: ./node_modules/bitcore-node-zero/bin/ . If you have changed directories, go back to: ./mynode/ and run the command: ```bash @@ -56,8 +56,8 @@ This will create a directory with configuration files for your node and install There are several add-on services available to extend the functionality of Bitcore: -- [Insight API Zero](https://github.com/ProphetAlgorithms/insight-api-zero) -- [Insight UI Zero](https://github.com/ProphetAlgorithms/insight-ui-zero) +- [Insight API Zero](https://github.com/zerocurrencycoin/insight-api-zero) +- [Insight UI Zero](https://github.com/zerocurrencycoin/insight-ui-zero) ## Documentation diff --git a/lib/scaffold/create.js b/lib/scaffold/create.js index dd9f9c40..84212d06 100644 --- a/lib/scaffold/create.js +++ b/lib/scaffold/create.js @@ -19,8 +19,8 @@ var BASE_PACKAGE = { license: 'MIT', readme: 'README.md', dependencies: { - 'bitcore-lib-zero': 'ProphetAlgorithms/bitcore-lib-zero', - 'bitcore-node-zero': 'ProphetAlgorithms/bitcore-node-zero' + 'bitcore-lib-zero': 'zerocurrencycoin/bitcore-lib-zero', + 'bitcore-node-zero': 'zerocurrencycoin/bitcore-node-zero' } }; diff --git a/package.json b/package.json index b31aa174..98e4cdec 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "author": "BitPay ", "version": "3.1.2", "main": "./index.js", - "repository": "git://github.com/ProphetAlgorithms/bitcore-node-zero.git", - "homepage": "https://github.com/ProphetAlgorithms/bitcore-node-zero", + "repository": "git://github.com/zerocurrencycoin/bitcore-node-zero.git", + "homepage": "https://github.com/zerocurrencycoin/bitcore-node-zero", "bugs": { - "url": "https://github.com/bitpay/bitcore-node/issues" + "url": "https://github.com/zerocurrencycoin/bitcore-node-zero/issues" }, "contributors": [ { @@ -47,7 +47,7 @@ "dependencies": { "async": "^1.3.0", "bitcoind-rpc": "^0.6.0", - "bitcore-lib-zero": "ProphetAlgorithms/bitcore-lib-zero", + "bitcore-lib-zero": "zerocurrencycoin/bitcore-lib-zero", "body-parser": "^1.13.3", "colors": "^1.1.2", "commander": "^2.8.1", From 2ef6d2bbb1d6c8a2db7a4cfc8734e6b33e060abe Mon Sep 17 00:00:00 2001 From: Mark Sailor Date: Mon, 29 Oct 2018 21:04:25 -0700 Subject: [PATCH 298/299] Change valueSat to valueZat --- lib/services/bitcoind.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 53467612..9f2ed002 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -1916,7 +1916,7 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { for(var inputIndex = 0; inputIndex < result.vin.length; inputIndex++) { var input = result.vin[inputIndex]; if (!tx.coinbase) { - tx.inputSatoshis += input.valueSat; + tx.inputSatoshis += input.valueZat; } var script = null; var scriptAsm = null; @@ -1933,7 +1933,7 @@ Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { scriptAsm: scriptAsm || null, sequence: input.sequence, address: input.address || null, - satoshis: _.isUndefined(input.valueSat) ? null : input.valueSat + satoshis: _.isUndefined(input.valueZat) ? null : input.valueZat }); } } From e98134b159f8d6b0fd22be2676ca358ce19e5822 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2020 03:16:29 +0000 Subject: [PATCH 299/299] build(deps): bump npm from 2.15.12 to 6.14.6 Bumps [npm](https://github.com/npm/cli) from 2.15.12 to 6.14.6. - [Release notes](https://github.com/npm/cli/releases) - [Changelog](https://github.com/npm/cli/blob/latest/CHANGELOG.md) - [Commits](https://github.com/npm/cli/compare/v2.15.12...v6.14.6) Signed-off-by: dependabot[bot] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98e4cdec..9b35422e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "liftoff": "^2.2.0", "lru-cache": "^4.0.1", "mkdirp": "0.5.0", - "npm": "^2.14.1", + "npm": "^6.14.6", "path-is-absolute": "^1.0.0", "semver": "^5.0.1", "socket.io": "^1.4.5",