From e5bcf23f910fb991829baf513ca5b8a4d9cdcf51 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 27 Dec 2023 11:46:59 +0400 Subject: [PATCH 01/13] test: add standalone wallet to the NodeContext. --- test/auction-rpc-test.js | 3 + test/node-http-test.js | 20 +++---- test/node-rescan-test.js | 5 +- test/node-rpc-test.js | 13 +++- test/util/node-context.js | 96 +++++++++++++++++++++++++----- test/util/nodes-context.js | 118 +++++++++++++++++++++++++++++-------- 6 files changed, 200 insertions(+), 55 deletions(-) diff --git a/test/auction-rpc-test.js b/test/auction-rpc-test.js index 9a8092987..d55742655 100644 --- a/test/auction-rpc-test.js +++ b/test/auction-rpc-test.js @@ -23,6 +23,9 @@ class TestUtil { bip37: true, wallet: true }); + + this.nodeCtx.init(); + this.network = this.nodeCtx.network; this.txs = {}; this.blocks = {}; diff --git a/test/node-http-test.js b/test/node-http-test.js index 0bf8c16b5..1fbcca6e8 100644 --- a/test/node-http-test.js +++ b/test/node-http-test.js @@ -28,9 +28,9 @@ describe('Node HTTP', function() { beforeEach(async () => { nodeCtx = new NodeContext(); - nclient = nodeCtx.nclient; await nodeCtx.open(); + nclient = nodeCtx.nclient; }); afterEach(async () => { @@ -74,9 +74,8 @@ describe('Node HTTP', function() { beforeEach(async () => { nodeCtx = new NodeContext(); - nclient = nodeCtx.nclient; - await nodeCtx.open(); + nclient = nodeCtx.nclient; }); afterEach(async () => { @@ -261,9 +260,8 @@ describe('Node HTTP', function() { network: 'regtest' }); - const interval = nodeCtx.network.names.treeInterval; - await nodeCtx.open(); + const interval = nodeCtx.network.names.treeInterval; const nclient = nodeCtx.nclient; const node = nodeCtx.node; @@ -315,9 +313,9 @@ describe('Node HTTP', function() { network: 'regtest' }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -337,9 +335,9 @@ describe('Node HTTP', function() { network: 'regtest', listen: true }); - const {network, nclient} = nodeCtx; await nodeCtx.open(); + const {network, nclient} = nodeCtx; const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -359,9 +357,9 @@ describe('Node HTTP', function() { network: 'main' }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -382,9 +380,9 @@ describe('Node HTTP', function() { listen: true }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -412,9 +410,9 @@ describe('Node HTTP', function() { publicBrontidePort }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -443,6 +441,8 @@ describe('Node HTTP', function() { rejectAbsurdFees: false }); + nodeCtx.init(); + const {network, nclient} = nodeCtx; const {treeInterval} = network.names; diff --git a/test/node-rescan-test.js b/test/node-rescan-test.js index c651a3e57..3d6fca10c 100644 --- a/test/node-rescan-test.js +++ b/test/node-rescan-test.js @@ -66,11 +66,10 @@ describe('Node Rescan Interactive API', function() { before(async () => { nodeCtx = new NodeContext(); - const {network} = nodeCtx; - - funderWallet = new MemWallet({ network }); await nodeCtx.open(); + const {network} = nodeCtx; + funderWallet = new MemWallet({ network }); nodeCtx.on('connect', (entry, block) => { funderWallet.addBlock(entry, block.txs); diff --git a/test/node-rpc-test.js b/test/node-rpc-test.js index 0b1428010..9701d4398 100644 --- a/test/node-rpc-test.js +++ b/test/node-rpc-test.js @@ -37,6 +37,7 @@ describe('RPC', function() { describe('getblockchaininfo', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -58,6 +59,7 @@ describe('RPC', function() { describe('getrawmempool', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -83,10 +85,9 @@ describe('RPC', function() { name: 'node-rpc-test' }); + await nodeCtx.open(); nclient = nodeCtx.nclient; node = nodeCtx.node; - - await nodeCtx.open(); }); after(async () => { @@ -221,6 +222,7 @@ describe('RPC', function() { ...nodeOptions, spv: true }); + await nodeCtx.open(); await assert.rejects(async () => { @@ -265,7 +267,6 @@ describe('RPC', function() { // default - prune: false nodeCtx = new NodeContext(nodeOptions); await nodeCtx.open(); - const {miner, nclient} = nodeCtx; const addr = 'rs1q4rvs9pp9496qawp2zyqpz3s90fjfk362q92vq8'; @@ -320,6 +321,7 @@ describe('RPC', function() { describe('mining', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const { miner, chain, @@ -505,6 +507,7 @@ describe('RPC', function() { ...nodeOptions, indexTX: true }); + nodeCtx.init(); const { miner, @@ -588,6 +591,7 @@ describe('RPC', function() { describe('networking', function() { const nodeCtx = new NodeContext({ ...nodeOptions, bip37: true }); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -607,6 +611,7 @@ describe('RPC', function() { describe('DNS Utility', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -762,6 +767,8 @@ describe('RPC', function() { wallet: true }); + nodeCtx.init(); + const { node, nclient, diff --git a/test/util/node-context.js b/test/util/node-context.js index c0a49db1c..aa2efd3eb 100644 --- a/test/util/node-context.js +++ b/test/util/node-context.js @@ -3,10 +3,11 @@ const assert = require('bsert'); const common = require('./common'); const fs = require('bfile'); +const Network = require('../../lib/protocol/network'); const SPVNode = require('../../lib/node/spvnode'); const FullNode = require('../../lib/node/fullnode'); +const WalletNode = require('../../lib/wallet/node'); const plugin = require('../../lib/wallet/plugin'); -const Network = require('../../lib/protocol/network'); const {NodeClient, WalletClient} = require('../../lib/client'); const Logger = require('blgr'); @@ -23,7 +24,7 @@ class NodeContext { constructor(options = {}) { this.name = 'node-test'; this.options = {}; - this.node = null; + this.prefix = null; this.opened = false; this.logger = new Logger({ console: true, @@ -31,13 +32,15 @@ class NodeContext { level: 'none' }); + this.initted = false; + this.node = null; + this.walletNode = null; this.nclient = null; this.wclient = null; this.clients = []; this.fromOptions(options); - this.init(); } fromOptions(options) { @@ -54,6 +57,11 @@ class NodeContext { walletHttpPort: null }; + if (options.name != null) { + assert(typeof options.name === 'string'); + this.name = options.name; + } + if (options.network != null) fnodeOptions.network = Network.get(options.network).type; @@ -66,17 +74,23 @@ class NodeContext { } if (options.prefix != null) { - fnodeOptions.prefix = this.prefix; + fnodeOptions.prefix = options.prefix; fnodeOptions.memory = false; + this.prefix = fnodeOptions.prefix; } if (options.memory != null) { - assert(!fnodeOptions.prefix, 'Can not set prefix with memory.'); + assert(typeof options.memory === 'boolean'); + assert(!(options.memory && options.prefix), + 'Can not set prefix with memory.'); + fnodeOptions.memory = options.memory; } - if (!this.memory && !this.prefix) + if (!fnodeOptions.memory && !fnodeOptions.prefix) { fnodeOptions.prefix = common.testdir(this.name); + this.prefix = fnodeOptions.prefix; + } if (options.wallet != null) fnodeOptions.wallet = options.wallet; @@ -101,23 +115,45 @@ class NodeContext { fnodeOptions.timeout = options.timeout; } + if (options.standalone != null) { + assert(typeof options.standalone === 'boolean'); + fnodeOptions.standalone = options.standalone; + } + this.options = fnodeOptions; } init() { + if (this.initted) + return; + if (this.options.spv) this.node = new SPVNode(this.options); else this.node = new FullNode(this.options); - if (this.options.wallet) + if (this.options.wallet && !this.options.standalone) { this.node.use(plugin); + } else if (this.options.wallet && this.options.standalone) { + this.walletNode = new WalletNode({ + ...this.options, + + nodeHost: '127.0.0.1', + nodePort: this.options.httpPort, + nodeApiKey: this.options.apiKey, + + httpPort: this.options.walletHttpPort, + apiKey: this.options.apiKey + }); + } // Initial wallets. this.nclient = this.nodeClient(); if (this.options.wallet) this.wclient = this.walletClient(); + + this.initted = true; } get network() { @@ -145,6 +181,12 @@ class NodeContext { } get wdb() { + if (!this.options.wallet) + return null; + + if (this.walletNode) + return this.walletNode.wdb; + return this.node.get('walletdb').wdb; } @@ -177,16 +219,26 @@ class NodeContext { */ async open() { + this.init(); + if (this.opened) return; if (this.prefix) await fs.mkdirp(this.prefix); + const open = common.forEvent(this.node, 'open'); await this.node.ensure(); await this.node.open(); await this.node.connect(); this.node.startSync(); + await open; + + if (this.walletNode) { + const walletOpen = common.forEvent(this.walletNode, 'open'); + await this.walletNode.open(); + await walletOpen; + } if (this.wclient) await this.wclient.open(); @@ -200,7 +252,6 @@ class NodeContext { if (!this.opened) return; - const close = common.forEvent(this.node, 'close'); const closeClients = []; for (const client of this.clients) { @@ -209,10 +260,22 @@ class NodeContext { } await Promise.all(closeClients); + + if (this.walletNode) { + const walletClose = common.forEvent(this.walletNode, 'close'); + await this.walletNode.close(); + await walletClose; + } + + const close = common.forEvent(this.node, 'close'); await this.node.close(); await close; + this.node = null; + this.wclient = null; + this.nclient = null; this.opened = false; + this.initted = false; } async destroy() { @@ -224,8 +287,8 @@ class NodeContext { * Helpers */ - enableLogging() { - this.logger.setLevel('debug'); + enableLogging(level = 'debug') { + this.logger.setLevel(level); } disableLogging() { @@ -299,17 +362,20 @@ class NodeContext { * Mine blocks and wait for connect. * @param {Number} count * @param {Address} address + * @param {ChainEntry} [tip=chain.tip] - Tip to mine on * @returns {Promise} - Block hashes */ - async mineBlocks(count, address) { + async mineBlocks(count, address, tip) { assert(this.open); - const blockEvents = common.forEvent(this.node, 'block', count); - const hashes = await this.nodeRPC.generateToAddress([count, address]); - await blockEvents; + if (!tip) + tip = this.chain.tip; - return hashes; + for (let i = 0; i < count; i++) { + const block = await this.miner.mineBlock(tip, address); + tip = await this.chain.add(block); + } } } diff --git a/test/util/nodes-context.js b/test/util/nodes-context.js index 0be0f325c..221b88208 100644 --- a/test/util/nodes-context.js +++ b/test/util/nodes-context.js @@ -19,7 +19,7 @@ class NodesContext { } addNode(options = {}) { - const index = this.nodeCtxs.length; + const index = this.nodeCtxs.length + 1; let seedPort = this.network.port + index - 1; @@ -30,17 +30,21 @@ class NodesContext { const brontidePort = this.network.brontidePort + index; const httpPort = this.network.rpcPort + index + 100; const walletHttpPort = this.network.walletPort + index + 200; + const nsPort = this.network.nsPort + index; + const rsPort = this.network.rsPort + index + 100; const nodeCtx = new NodeContext({ + listen: true, + ...options, // override name: `node-${index}`, network: this.network, - listen: true, - publicHost: '127.0.0.1', - publicPort: port, + port: port, brontidePort: brontidePort, + rsPort: rsPort, + nsPort: nsPort, httpPort: httpPort, walletHttpPort: walletHttpPort, seeds: [ @@ -49,9 +53,19 @@ class NodesContext { }); this.nodeCtxs.push(nodeCtx); + return nodeCtx; } - open() { + /** + * Open all or specific nodes. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + open(index = -1) { + if (index !== -1) + return this.context(index).open(); + const jobs = []; for (const nodeCtx of this.nodeCtxs) @@ -60,7 +74,16 @@ class NodesContext { return Promise.all(jobs); } - close() { + /** + * Close all or specific nodes. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + close(index = -1) { + if (index !== -1) + return this.context(index).close(); + const jobs = []; for (const nodeCtx of this.nodeCtxs) @@ -69,21 +92,52 @@ class NodesContext { return Promise.all(jobs); } + /** + * Destroy specific or all nodes. Clean up directories on the disk. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + destroy(index = -1) { + if (index !== -1) + return this.context(index).destroy(); + + const jobs = []; + + for (const nodeCtx of this.nodeCtxs) + jobs.push(nodeCtx.destroy()); + + return Promise.all(jobs); + } + + /** + * Connect all nodes. + * @returns {Promise} + */ + async connect() { for (const nodeCtx of this.nodeCtxs) { await nodeCtx.node.connect(); - await new Promise(r => setTimeout(r, 1000)); + await nodeCtx.node.startSync(); } } + /** + * Disconnect all nodes. + * @returns {Promise} + */ + async disconnect() { for (let i = this.nodeCtxs.length - 1; i >= 0; i--) { - const node = this.nodeCtxs[i]; + const node = this.nodeCtxs[i].node; await node.disconnect(); - await new Promise(r => setTimeout(r, 1000)); } } + /** + * Start syncing. + */ + startSync() { for (const nodeCtx of this.nodeCtxs) { nodeCtx.chain.synced = true; @@ -92,32 +146,48 @@ class NodesContext { } } + /** + * Stop syncing. + */ + stopSync() { for (const nodeCtx of this.nodeCtxs) nodeCtx.stopSync(); } - async generate(index, blocks) { - const nodeCtx = this.nodeCtxs[index]; - - assert(nodeCtx); - - for (let i = 0; i < blocks; i++) { - const block = await nodeCtx.miner.mineBlock(); - await nodeCtx.chain.add(block); - } + /** + * Mine blocks. + * @param {Number} index + * @param {Number} blocks + * @param {String} address + * @param {ChainEntry} [tip=chain.tip] + * @returns {Promise} + */ + + async generate(index, blocks, address, tip) { + return this.context(index).mineBlocks(blocks, address, tip); } + /** + * Get NodeCtx for the node. + * @param {Number} index + * @returns {NodeContext} + */ + context(index) { - const node = this.nodeCtxs[index]; - assert(node); - return node; + const nodeCtx = this.nodeCtxs[index]; + assert(nodeCtx); + return nodeCtx; } + /** + * Get height for the node. + * @param {Number} index + * @returns {Number} + */ + height(index) { - const nodeCtx = this.nodeCtxs[index]; - assert(nodeCtx); - return nodeCtx.height; + return this.context(index).height; } } From 570023651376d18238f33effe0df2e79aa6e0350 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 27 Dec 2023 11:52:32 +0400 Subject: [PATCH 02/13] test: move balance object to utils. --- test/util/balance.js | 144 ++++++++++++++++++++++++++++++++++++ test/wallet-balance-test.js | 119 ++++++----------------------- 2 files changed, 168 insertions(+), 95 deletions(-) create mode 100644 test/util/balance.js diff --git a/test/util/balance.js b/test/util/balance.js new file mode 100644 index 000000000..d53b612bf --- /dev/null +++ b/test/util/balance.js @@ -0,0 +1,144 @@ +'use strict'; + +const assert = require('bsert'); +const Wallet = require('../../lib/wallet/wallet'); +const WalletClient = require('../../lib/client/wallet'); + +/** + * @property {Number} tx + * @property {Number} coin + * @property {Number} confirmed + * @property {Number} unconfirmed + * @property {Number} ulocked - unconfirmed locked + * @property {Number} clocked - confirmed locked + */ + +class Balance { + constructor(options) { + options = options || {}; + + this.tx = options.tx || 0; + this.coin = options.coin || 0; + this.confirmed = options.confirmed || 0; + this.unconfirmed = options.unconfirmed || 0; + this.ulocked = options.ulocked || 0; + this.clocked = options.clocked || 0; + } + + clone() { + return new Balance(this); + } + + cloneWithDelta(obj) { + return this.clone().apply(obj); + } + + fromBalance(obj) { + this.tx = obj.tx; + this.coin = obj.coin; + this.confirmed = obj.confirmed; + this.unconfirmed = obj.unconfirmed; + this.ulocked = obj.lockedUnconfirmed; + this.clocked = obj.lockedConfirmed; + + return this; + } + + apply(balance) { + this.tx += balance.tx || 0; + this.coin += balance.coin || 0; + this.confirmed += balance.confirmed || 0; + this.unconfirmed += balance.unconfirmed || 0; + this.ulocked += balance.ulocked || 0; + this.clocked += balance.clocked || 0; + + return this; + } + + diff(balance) { + return new Balance({ + tx: this.tx - balance.tx, + coin: this.coin - balance.coin, + confirmed: this.confirmed - balance.confirmed, + unconfirmed: this.unconfirmed - balance.unconfirmed, + ulocked: this.ulocked - balance.ulocked, + clocked: this.clocked - balance.clocked + }); + } + + static fromBalance(wbalance) { + return new this().fromBalance(wbalance); + } +} + +/** + * @param {Wallet} wallet + * @param {String} accountName + * @returns {Promise} + */ + +async function getWalletBalance(wallet, accountName) { + assert(wallet instanceof Wallet); + const balance = await wallet.getBalance(accountName); + return Balance.fromBalance(balance.getJSON(true)); +} + +/** + * @param {WalletClient} wclient + * @param {String} id + * @param {String} accountName + * @returns {Promise} + */ + +async function getWClientBalance(wclient, id, accountName) { + assert(wclient instanceof WalletClient); + const balance = await wclient.getBalance(id, accountName); + return Balance.fromBalance(balance); +} + +/** + * @param {WalletClient.Wallet} balance + * @param {String} accountName + * @returns {Promise} + */ + +async function getWClientWalletBalance(wallet, accountName) { + assert(wallet instanceof WalletClient.Wallet); + const balance = await wallet.getBalance(accountName); + return Balance.fromBalance(balance); +} + +async function getBalance(wallet, accountName) { + if (wallet instanceof WalletClient.Wallet) + return getWClientWalletBalance(wallet, accountName); + + return getWalletBalance(wallet, accountName); +} + +/** + * @param {Wallet} wallet + * @param {String} accountName + * @param {Balance} expectedBalance + * @param {String} message + * @returns {Promise} + */ + +async function assertBalanceEquals(wallet, accountName, expectedBalance, message) { + const balance = await getBalance(wallet, accountName); + assert.deepStrictEqual(balance, expectedBalance, message); +} + +async function assertWClientBalanceEquals(wclient, id, accountName, expectedBalance, message) { + const balance = await getWClientBalance(wclient, id, accountName); + assert.deepStrictEqual(balance, expectedBalance, message); +} + +exports.Balance = Balance; + +exports.getBalance = getBalance; +exports.getWalletBalance = getWalletBalance; +exports.getWClientBalance = getWClientBalance; +exports.getWClientWalletBalance = getWClientWalletBalance; + +exports.assertBalanceEquals = assertBalanceEquals; +exports.assertWClientBalanceEquals = assertWClientBalanceEquals; diff --git a/test/wallet-balance-test.js b/test/wallet-balance-test.js index 66b5902f5..065f8c99c 100644 --- a/test/wallet-balance-test.js +++ b/test/wallet-balance-test.js @@ -11,6 +11,7 @@ const Output = require('../lib/primitives/output'); const {Resource} = require('../lib/dns/resource'); const {types, grindName} = require('../lib/covenants/rules'); const {forEventCondition} = require('./util/common'); +const {Balance, assertBalanceEquals} = require('./util/balance'); /** * Wallet balance tracking tests. @@ -68,65 +69,9 @@ const DEFAULT_ACCOUNT = 'default'; const ALT_ACCOUNT = 'alt'; // Balances -/** - * @property {Number} tx - * @property {Number} coin - * @property {Number} confirmed - * @property {Number} unconfirmed - * @property {Number} ulocked - unconfirmed locked - * @property {Number} clocked - confirmed locked - */ - -class BalanceObj { - constructor(options) { - options = options || {}; - - this.tx = options.tx || 0; - this.coin = options.coin || 0; - this.confirmed = options.confirmed || 0; - this.unconfirmed = options.unconfirmed || 0; - this.ulocked = options.ulocked || 0; - this.clocked = options.clocked || 0; - } - - clone() { - return new BalanceObj(this); - } - - cloneWithDelta(obj) { - return this.clone().apply(obj); - } - - fromBalance(obj) { - this.tx = obj.tx; - this.coin = obj.coin; - this.confirmed = obj.confirmed; - this.unconfirmed = obj.unconfirmed; - this.ulocked = obj.lockedUnconfirmed; - this.clocked = obj.lockedConfirmed; - - return this; - } - - apply(balance) { - this.tx += balance.tx || 0; - this.coin += balance.coin || 0; - this.confirmed += balance.confirmed || 0; - this.unconfirmed += balance.unconfirmed || 0; - this.ulocked += balance.ulocked || 0; - this.clocked += balance.clocked || 0; - - return this; - } - - static fromBalance(wbalance) { - return new this().fromBalance(wbalance); - } -} - const INIT_BLOCKS = treeInterval; const INIT_FUND = 10e6; -const NULL_BALANCE = new BalanceObj({ +const NULL_BALANCE = new Balance({ tx: 0, coin: 0, unconfirmed: 0, @@ -135,7 +80,7 @@ const NULL_BALANCE = new BalanceObj({ clocked: 0 }); -const INIT_BALANCE = new BalanceObj({ +const INIT_BALANCE = new Balance({ tx: 1, coin: 1, unconfirmed: INIT_FUND, @@ -202,38 +147,22 @@ async function resign(wallet, mtx) { */ /** - * @returns {Promise} + * @returns {Promise} */ -async function getBalanceObj(wallet, accountName) { - const balance = await wallet.getBalance(accountName); - return BalanceObj.fromBalance(balance.getJSON(true)); -} - -async function assertBalance(wallet, accountName, expected, message) { - const balance = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance, expected, message); - - // recalculate balance test - await wallet.recalculateBalances(); - const balance2 = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance2, expected, message); -} - -async function assertRecalcBalance(wallet, accountName, expected, message) { +async function assertRecalcBalanceEquals(wallet, accountName, expected, message) { await wallet.recalculateBalances(); - const balance = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance, expected, message); + assertBalanceEquals(wallet, accountName, expected, message); } /** - * @param {BalanceObj} balance - * @param {BalanceObj} delta - * @returns {BalanceObj} + * @param {Balance} balance + * @param {Balance} delta + * @returns {Balance} */ function applyDelta(balance, delta) { - return balance.clone().apply(delta); + return balance.cloneWithDelta(delta); } describe('Wallet Balance', function() { @@ -364,14 +293,14 @@ describe('Wallet Balance', function() { /** * @typedef {Object} TestBalances - * @property {BalanceObj} TestBalances.initialBalance - * @property {BalanceObj} TestBalances.sentBalance - * @property {BalanceObj} TestBalances.confirmedBalance - * @property {BalanceObj} TestBalances.unconfirmedBalance - * @property {BalanceObj} TestBalances.eraseBalance - * @property {BalanceObj} TestBalances.blockConfirmedBalance - * @property {BalanceObj} TestBalances.blockUnconfirmedBalance - * @property {BalanceObj} [TestBalances.blockFinalConfirmedBalance] + * @property {Balance} TestBalances.initialBalance + * @property {Balance} TestBalances.sentBalance + * @property {Balance} TestBalances.confirmedBalance + * @property {Balance} TestBalances.unconfirmedBalance + * @property {Balance} TestBalances.eraseBalance + * @property {Balance} TestBalances.blockConfirmedBalance + * @property {Balance} TestBalances.blockUnconfirmedBalance + * @property {Balance} [TestBalances.blockFinalConfirmedBalance] */ /** @@ -481,14 +410,14 @@ describe('Wallet Balance', function() { for (const [key, [balanceName, name]] of Object.entries(BALANCE_CHECK_MAP)) { checks[key] = async (wallet) => { - await assertBalance( + await assertBalanceEquals( wallet, DEFAULT_ACCOUNT, defBalances[balanceName], `${name} balance is incorrect in the account ${DEFAULT_ACCOUNT}.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, DEFAULT_ACCOUNT, defBalances[balanceName], @@ -497,14 +426,14 @@ describe('Wallet Balance', function() { ); if (altBalances != null) { - await assertBalance( + await assertBalanceEquals( wallet, ALT_ACCOUNT, altBalances[balanceName], `${name} balance is incorrect in the account ${ALT_ACCOUNT}.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, ALT_ACCOUNT, altBalances[balanceName], @@ -513,14 +442,14 @@ describe('Wallet Balance', function() { ); } - await assertBalance( + await assertBalanceEquals( wallet, -1, walletBalances[balanceName], `${name} balance is incorrect for the wallet.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, -1, walletBalances[balanceName], From dcafb2046bb51f6aca49bcded19d70675ec8210d Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 27 Dec 2023 12:02:55 +0400 Subject: [PATCH 03/13] wallet: WalletNode now emits 'open' and 'close' events. wallet-node: fix standalone wallet get entry requests when entry is not found on chain. client: expose Wallet class. --- CHANGELOG.md | 1 + lib/client/wallet.js | 2 + lib/wallet/client.js | 4 + lib/wallet/node.js | 2 + test/wallet-rescan-test.js | 269 ++++++++++++++++++++++++++++++++++++- 5 files changed, 276 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a52e3b1..d53891960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ process and allows parallel rescans. #### Wallet API - Add migration that recalculates txdb balances to fix any inconsistencies. +- WalletNode now emits `open` and `close` events. - WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`. - WalletDB - `open()` no longer calls `connect` and needs separate call `connect`. diff --git a/lib/client/wallet.js b/lib/client/wallet.js index 1a027ce45..784b21d17 100644 --- a/lib/client/wallet.js +++ b/lib/client/wallet.js @@ -1600,4 +1600,6 @@ class Wallet extends EventEmitter { * Expose */ +WalletClient.Wallet = Wallet; + module.exports = WalletClient; diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 92bb9a9f8..67bd3995a 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -94,6 +94,9 @@ class WalletClient extends NodeClient { */ function parseEntry(data) { + if (!data) + return null; + // 32 hash // 4 height // 4 nonce @@ -123,6 +126,7 @@ function parseEntry(data) { function parseBlock(entry, txs) { const block = parseEntry(entry); + assert(block); const out = []; for (const tx of txs) diff --git a/lib/wallet/node.js b/lib/wallet/node.js index 53707c661..ff4816320 100644 --- a/lib/wallet/node.js +++ b/lib/wallet/node.js @@ -113,6 +113,7 @@ class WalletNode extends Node { await this.handleOpen(); this.logger.info('Wallet node is loaded.'); + this.emit('open'); } /** @@ -134,6 +135,7 @@ class WalletNode extends Node { await this.wdb.disconnect(); await this.wdb.close(); await this.handleClose(); + this.emit('close'); } } diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index 0be8a1e1a..28b2560f4 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -2,8 +2,14 @@ const assert = require('bsert'); const Network = require('../lib/protocol/network'); +const NodesContext = require('./util/nodes-context'); const NodeContext = require('./util/node-context'); +const {forEvent, forEventCondition} = require('./util/common'); +const {Balance, getWClientBalance} = require('./util/balance'); +// Definitions: +// Gapped txs/addresses - addresses with lookahead + 1 gap when deriving. +// // Setup: // - Standalone Node (no wallet) responsible for progressing network. // - Wallet Node (with wallet) responsible for rescanning. @@ -22,13 +28,270 @@ const NodeContext = require('./util/node-context'); // recovery is impossible. This tests situation where in block // derivation depth is lower than wallet lookahead. -// TODO: Rewrite using util/node from the interactive rescan test. // TODO: Add the standalone Wallet variation. // TODO: Add initial rescan test. +const combinations = [ + { SPV: false, STANDALONE: false, name: 'Full/Plugin' }, + { SPV: false, STANDALONE: true, name: 'Full/Standalone' }, + { SPV: true, STANDALONE: false, name: 'SPV/Plugin' } + // Not supported. + // { SPV: true, STANDALONE: true, name: 'SPV/Standalone' } +]; + describe('Wallet rescan', function() { const network = Network.get('regtest'); + for (const {SPV, STANDALONE, name} of combinations) { + describe(`Initial sync/rescan (${name} Integration)`, function() { + // Test wallet plugin/standalone is disabled and re-enabled after some time: + // 1. Normal received blocks. + // 2. Reorged after wallet was closed. + // NOTE: Node is not closed, only wallet. + + const MINER = 0; + const WALLET = 1; + const WALLET_NO_WALLET = 2; + + /** @type {NodesContext} */ + let nodes; + let wnodeCtx, noWnodeCtx; + let minerWallet, minerAddress; + let testAddress; + + before(async () => { + nodes = new NodesContext(network, 1); + + // MINER = 0 + nodes.init({ + wallet: true, + noDNS: true, + bip37: true + }); + + // WALLET = 1 + wnodeCtx = nodes.addNode({ + noDNS: true, + wallet: true, + + standalone: STANDALONE, + spv: SPV, + + // We need to store on disk in order to test + // recovery on restart + memory: false + }); + + // WALLET_NO_WALLET = 2 + // Wallet node that uses same chain above one + // just does not start wallet. + noWnodeCtx = nodes.addNode({ + noDNS: true, + wallet: false, + prefix: wnodeCtx.prefix, + memory: false, + spv: SPV + }); + + // only open two at a time. + await nodes.open(MINER); + await nodes.open(WALLET); + + minerWallet = nodes.context(MINER).wclient.wallet('primary'); + minerAddress = (await minerWallet.createAddress('default')).address; + + const testWallet = wnodeCtx.wclient.wallet('primary'); + testAddress = (await testWallet.createAddress('default')).address; + + await nodes.close(WALLET); + }); + + after(async () => { + await nodes.close(); + await nodes.destroy(); + }); + + afterEach(async () => { + await nodes.close(WALLET); + await nodes.close(WALLET_NO_WALLET); + }); + + it('should fund and spend to wallet', async () => { + await wnodeCtx.open(); + + const txEvent = forEvent(wnodeCtx.wdb, 'tx'); + + // fund wallet. + await nodes.generate(MINER, 9, minerAddress); + + // Send TX to the test wallet. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 1e6 + }] + }); + + await nodes.generate(MINER, 1, minerAddress); + await txEvent; + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + assert.deepStrictEqual(balance, new Balance({ + coin: 1, + tx: 1, + confirmed: 1e6, + unconfirmed: 1e6 + })); + }); + + it('should rescan/resync after wallet was off', async () => { + // replace wallet node with new one w/o wallet. + await noWnodeCtx.open(); + + await nodes.generate(MINER, 10, minerAddress); + + // Mine in the last block that we will be reorging. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 2e6 + }] + }); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + await nodeSync; + + // Disable wallet + await noWnodeCtx.close(); + + // sync node. + let eventsToWait; + + wnodeCtx.init(); + + // For spv we don't wait for sync done, as it will do the full rescan + // and reset the SPVNode as well. It does not depend on the accumulated + // blocks. + if (SPV) { + eventsToWait = [ + // This will happen right away, as scan will just call reset + forEvent(wnodeCtx.wdb, 'sync done'), + // This is what matters for the rescan. + forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + }), + // Make sure node gets resets. + forEvent(wnodeCtx.node, 'reset') + ]; + } else { + eventsToWait = [ + forEvent(wnodeCtx.wdb, 'sync done') + ]; + } + + await wnodeCtx.open(); + await Promise.all(eventsToWait); + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + assert.deepStrictEqual(balance, new Balance({ + coin: 2, + tx: 2, + confirmed: 1e6 + 2e6, + unconfirmed: 1e6 + 2e6 + })); + + await wnodeCtx.close(); + }); + + it('should rescan/resync after wallet was off and node reorged', async () => { + const minerCtx = nodes.context(MINER); + + await noWnodeCtx.open(); + + // Reorg the network + const tip = minerCtx.chain.tip; + const block = await minerCtx.chain.getBlock(tip.hash); + + // Last block contained our tx from previous test. (integration) + assert.strictEqual(block.txs.length, 2); + + const reorgEvent = forEvent(minerCtx.node, 'reorganize'); + const forkTip = await minerCtx.chain.getPrevious(tip); + + // REORG + await nodes.generate(MINER, 2, minerAddress, forkTip); + // Reset mempool/Get rid of tx after reorg. + await nodes.context(MINER).mempool.reset(); + await nodes.generate(MINER, 2, minerAddress); + await reorgEvent; + + // Send another tx, with different output. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 3e6 + }] + }); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + await nodeSync; + + await noWnodeCtx.close(); + + wnodeCtx.init(); + + // initial sync + let eventsToWait; + if (SPV) { + eventsToWait = [ + // This will happen right away, as scan will just call reset + forEvent(wnodeCtx.wdb, 'sync done'), + // This is what matters for the rescan. + forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + }), + // Make sure node gets resets. + forEvent(wnodeCtx.node, 'reset'), + forEvent(wnodeCtx.wdb, 'unconfirmed') + ]; + } else { + eventsToWait = [ + forEvent(wnodeCtx.wdb, 'sync done'), + forEvent(wnodeCtx.wdb, 'unconfirmed') + ]; + } + await wnodeCtx.open(); + await Promise.all(eventsToWait); + + assert.strictEqual(wnodeCtx.height, nodes.height(MINER)); + assert.strictEqual(wnodeCtx.wdb.state.height, wnodeCtx.height); + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + + // previous transaction should get unconfirmed. + assert.deepStrictEqual(balance, new Balance({ + coin: 3, + tx: 3, + confirmed: 1e6 + 3e6, + unconfirmed: 1e6 + 2e6 + 3e6 + })); + + await wnodeCtx.close(); + }); + }); + } + describe('Deadlock', function() { const nodeCtx = new NodeContext({ memory: true, @@ -39,6 +302,8 @@ describe('Wallet rescan', function() { let address, node, wdb; before(async () => { + nodeCtx.init(); + node = nodeCtx.node; wdb = nodeCtx.wdb; @@ -51,7 +316,7 @@ describe('Wallet rescan', function() { }); it('should generate 10 blocks', async () => { - await node.rpc.generateToAddress([10, address.toString(network)]); + await nodeCtx.mineBlocks(10, address); }); it('should rescan when receiving a block', async () => { From 8affe9570daecfbb434a56577ece6f95ab1c060a Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 27 Dec 2023 15:39:00 +0400 Subject: [PATCH 04/13] node: add fullLock option to the interactive rescan. Interactive rescan by default does per block scan lock. This enables parallel rescans, as well as chain sync while rescan is in progress. But in specific cases, it may be more beneficial to stop the node from syncing while the rescan is in progress. --- CHANGELOG.md | 2 +- lib/blockchain/chain.js | 40 +++++++++++++++-- lib/client/node.js | 5 ++- lib/node/fullnode.js | 6 ++- lib/node/http.js | 8 ++-- test/node-rescan-test.js | 96 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 138 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d53891960..5410c7541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ process and allows parallel rescans. - `compactInterval` - what is the current compaction interval config. - `nextCompaction` - when will the next compaction trigger after restart. - `lastCompaction` - when was the last compaction run. - - Introduce `scan interactive` hook (start, filter) + - Introduce `scan interactive` hook (start, filter, fullLock) ### Node HTTP Client: - Introduce `scanInteractive` method that starts interactive rescan. diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index aa5bb4c5a..f537143e6 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2273,9 +2273,39 @@ class Chain extends AsyncEmitter { * @param {BloomFilter} filter - Starting bloom filter containing tx, * address and name hashes. * @param {Function} iter - Iterator. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async scanInteractive(start, filter, iter, fullLock = false) { + if (fullLock) { + const unlock = await this.locker.lock(); + try { + // We lock the whole chain, no longer lock per block scan. + return await this._scanInteractive(start, filter, iter, false); + } catch (e) { + this.logger.debug('Scan(interactive) errored. Error: %s', e.message); + throw e; + } finally { + unlock(); + } + } + + return this._scanInteractive(start, filter, iter, true); + } + + /** + * Interactive scan the blockchain for transactions containing specified + * address hashes. Allows repeat and abort. + * @param {Hash|Number} start - Block hash or height to start at. + * @param {BloomFilter} filter - Starting bloom filter containing tx, + * address and name hashes. + * @param {Function} iter - Iterator. + * @param {Boolean} [lockPerScan=true] - if we should lock per block scan. + * @returns {Promise} */ - async scanInteractive(start, filter, iter) { + async _scanInteractive(start, filter, iter, lockPerScan = true) { if (start == null) start = this.network.genesis.hash; @@ -2287,7 +2317,10 @@ class Chain extends AsyncEmitter { let hash = start; while (hash != null) { - const unlock = await this.locker.lock(); + let unlock; + + if (lockPerScan) + unlock = await this.locker.lock(); try { const {entry, txs} = await this.db.scanBlock(hash, filter); @@ -2333,7 +2366,8 @@ class Chain extends AsyncEmitter { this.logger.debug('Scan(interactive) errored. Error: %s', e.message); throw e; } finally { - unlock(); + if (lockPerScan) + unlock(); } } } diff --git a/lib/client/node.js b/lib/client/node.js index 631eb3d02..aae765d5c 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -370,16 +370,17 @@ class NodeClient extends Client { * Rescan for any missed transactions. (Interactive) * @param {Number|Hash} start - Start block. * @param {BloomFilter} [filter] + * @param {Boolean} [fullLock=false] * @returns {Promise} */ - rescanInteractive(start, filter = null) { + rescanInteractive(start, filter = null, fullLock = false) { if (start == null) start = 0; assert(typeof start === 'number' || Buffer.isBuffer(start)); - return this.call('rescan interactive', start, filter); + return this.call('rescan interactive', start, filter, fullLock); } } diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index de7fcc2b0..0e8bbdcb8 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -369,11 +369,13 @@ class FullNode extends Node { * @param {Number|Hash} start - Start block. * @param {BloomFilter} filter * @param {Function} iter - Iterator. + * @param {Boolean} [fullLock=false] - lock the whole chain instead of per + * scan. * @returns {Promise} */ - scanInteractive(start, filter, iter) { - return this.chain.scanInteractive(start, filter, iter); + scanInteractive(start, filter, iter, fullLock = false) { + return this.chain.scanInteractive(start, filter, iter, fullLock); } /** diff --git a/lib/node/http.js b/lib/node/http.js index e82b482bd..93277edb2 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -712,6 +712,7 @@ class HTTP extends Server { const valid = new Validator(args); const start = valid.uintbhash(0); const rawFilter = valid.buf(1); + const fullLock = valid.bool(2, false); let filter = socket.filter; if (start == null) @@ -720,7 +721,7 @@ class HTTP extends Server { if (rawFilter) filter = BloomFilter.fromRaw(rawFilter); - return this.scanInteractive(socket, start, filter); + return this.scanInteractive(socket, start, filter, fullLock); }); } @@ -859,10 +860,11 @@ class HTTP extends Server { * @param {WebSocket} socket * @param {Hash} start * @param {BloomFilter} filter + * @param {Boolean} [fullLock=false] * @returns {Promise} */ - async scanInteractive(socket, start, filter) { + async scanInteractive(socket, start, filter, fullLock = false) { const iter = async (entry, txs) => { const block = entry.encode(); const raw = []; @@ -921,7 +923,7 @@ class HTTP extends Server { }; try { - await this.node.scanInteractive(start, filter, iter); + await this.node.scanInteractive(start, filter, iter, fullLock); } catch (err) { return socket.call('block rescan interactive abort', err.message); } diff --git a/test/node-rescan-test.js b/test/node-rescan-test.js index 3d6fca10c..86584b7b2 100644 --- a/test/node-rescan-test.js +++ b/test/node-rescan-test.js @@ -399,20 +399,57 @@ describe('Node Rescan Interactive API', function() { node.scanInteractive(startHeight, null, getIter(counter2)) ]); - assert.strictEqual(counter1.count, 10); - assert.strictEqual(counter2.count, 10); + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); - // Chain gets locked per block, so we should see alternating events. + // Chain gets locked per block by default, so we should see alternating events. // Because they start in parallel, but id1 starts first they will be // getting events in alternating older (first one gets lock, second waits, // second gets lock, first waits, etc.) - for (let i = 0; i < 10; i++) { + for (let i = 0; i < RESCAN_DEPTH; i++) { assert.strictEqual(events[i].id, 1); assert.strictEqual(events[i + 1].id, 2); i++; } }); + it('should rescan in series', async () => { + const {node} = nodeCtx; + const startHeight = nodeCtx.height - RESCAN_DEPTH + 1; + + const events = []; + const getIter = (counterObj) => { + return async (entry, txs) => { + assert.strictEqual(entry.height, startHeight + counterObj.count); + assert.strictEqual(txs.length, 4); + + events.push({ ...counterObj }); + counterObj.count++; + + return { + type: scanActions.NEXT + }; + }; + }; + + const counter1 = { id: 1, count: 0 }; + const counter2 = { id: 2, count: 0 }; + await Promise.all([ + node.scanInteractive(startHeight, null, getIter(counter1), true), + node.scanInteractive(startHeight, null, getIter(counter2), true) + ]); + + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); + + // We lock the whole chain for this test, so we should see events + // from one to other. + for (let i = 0; i < RESCAN_DEPTH; i++) { + assert.strictEqual(events[i].id, 1); + assert.strictEqual(events[i + RESCAN_DEPTH].id, 2); + } + }); + describe('HTTP', function() { let client = null; @@ -456,7 +493,7 @@ describe('Node Rescan Interactive API', function() { filter = test.filter.encode(); await client.rescanInteractive(startHeight, filter); - assert.strictEqual(count, 10); + assert.strictEqual(count, RESCAN_DEPTH); count = 0; if (test.filter) @@ -757,20 +794,63 @@ describe('Node Rescan Interactive API', function() { client2.rescanInteractive(startHeight) ]); - assert.strictEqual(counter1.count, 10); - assert.strictEqual(counter2.count, 10); + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); // Chain gets locked per block, so we should see alternating events. // Because they start in parallel, but id1 starts first they will be // getting events in alternating older (first one gets lock, second waits, // second gets lock, first waits, etc.) - for (let i = 0; i < 10; i++) { + for (let i = 0; i < RESCAN_DEPTH; i++) { assert.strictEqual(events[i].id, 1); assert.strictEqual(events[i + 1].id, 2); i++; } }); + it('should rescan in series', async () => { + const client2 = nodeCtx.nodeClient(); + await client2.open(); + + const startHeight = nodeCtx.height - RESCAN_DEPTH + 1; + const events = []; + const counter1 = { id: 1, count: 0 }; + const counter2 = { id: 2, count: 0 }; + + const getIter = (counterObj) => { + return async (rawEntry, rawTXs) => { + const [entry, txs] = parseBlock(rawEntry, rawTXs); + assert.strictEqual(entry.height, startHeight + counterObj.count); + assert.strictEqual(txs.length, 4); + + events.push({ ...counterObj }); + counterObj.count++; + + return { + type: scanActions.NEXT + }; + }; + }; + + client.hook('block rescan interactive', getIter(counter1)); + client2.hook('block rescan interactive', getIter(counter2)); + + await Promise.all([ + client.rescanInteractive(startHeight, null, true), + client2.rescanInteractive(startHeight, null, true) + ]); + + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); + + // We lock the whole chain for this test, so we should see events + // from one to other. + for (let i = 0; i < RESCAN_DEPTH; i++) { + assert.strictEqual(events[i].id, 1); + assert.strictEqual(events[i + RESCAN_DEPTH].id, 2); + } + }); + // Make sure the client closing does not cause the chain locker to get // indefinitely locked. (https://github.com/bcoin-org/bsock/pull/11) it('should stop rescan when client closes', async () => { From 0632bb502769439e0465808dde4dff8eaa1c2e44 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 1 Feb 2024 17:23:28 +0400 Subject: [PATCH 05/13] wdb: add reorg guard in disconnect. --- lib/wallet/walletdb.js | 3 ++ test/wallet-rescan-test.js | 77 +++++++++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 1e0568007..16cf0aa1f 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -159,6 +159,9 @@ class WalletDB extends EventEmitter { }); this.client.bind('block disconnect', async (entry) => { + if (this.rescanning) + return; + try { await this.removeBlock(entry); } catch (e) { diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index 28b2560f4..92381d746 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -3,7 +3,6 @@ const assert = require('bsert'); const Network = require('../lib/protocol/network'); const NodesContext = require('./util/nodes-context'); -const NodeContext = require('./util/node-context'); const {forEvent, forEventCondition} = require('./util/common'); const {Balance, getWClientBalance} = require('./util/balance'); @@ -293,54 +292,96 @@ describe('Wallet rescan', function() { } describe('Deadlock', function() { - const nodeCtx = new NodeContext({ - memory: true, - network: 'regtest', - wallet: true - }); - - let address, node, wdb; + const nodes = new NodesContext(network, 1); + let minerCtx; + let nodeCtx, address, node, wdb; before(async () => { - nodeCtx.init(); + nodes.init({ + memory: true, + wallet: false + }); + + nodes.addNode({ + memory: true, + wallet: true + }); + + await nodes.open(); + minerCtx = nodes.context(0); + nodeCtx = nodes.context(1); node = nodeCtx.node; wdb = nodeCtx.wdb; - await nodeCtx.open(); address = await wdb.primary.receiveAddress(); }); after(async () => { - await nodeCtx.close(); + await nodes.close(); }); - it('should generate 10 blocks', async () => { - await nodeCtx.mineBlocks(10, address); + it('should generate 20 blocks', async () => { + await minerCtx.mineBlocks(20, address); + await forEventCondition(nodeCtx.chain, 'connect', (entry) => { + return entry.height === 20; + }); }); it('should rescan when receiving a block', async () => { const preTip = await wdb.getTip(); await Promise.all([ - node.rpc.generateToAddress([1, address.toString(network)]), + minerCtx.mineBlocks(5, address), wdb.rescan(0) ]); const wdbTip = await wdb.getTip(); - assert.strictEqual(wdbTip.height, preTip.height + 1); + assert.strictEqual(wdbTip.height, preTip.height + 5); }); - it('should rescan when receiving a block', async () => { + it('should rescan when receiving blocks', async () => { const preTip = await wdb.getTip(); + const minerHeight = minerCtx.height; + const BLOCKS = 50; + + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === minerHeight + BLOCKS; + }); await Promise.all([ wdb.rescan(0), - node.rpc.generateToAddress([1, address.toString(network)]) + minerCtx.mineBlocks(BLOCKS, address) ]); + await blocks; + + const tip = await wdb.getTip(); + + assert.strictEqual(tip.height, preTip.height + BLOCKS); + }); + + it('should rescan when chain is reorging', async () => { + const minerHeight = minerCtx.height; + const BLOCKS = 50; + const reorgHeight = minerHeight - 10; + const newHeight = minerHeight + 40; + + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === newHeight; + }, 10000); + + const reorgEntry = await minerCtx.chain.getEntry(reorgHeight); + + await Promise.all([ + wdb.rescan(0), + minerCtx.mineBlocks(BLOCKS, address, reorgEntry) + ]); + + await blocks; + const tip = await wdb.getTip(); - assert.strictEqual(tip.height, preTip.height + 1); + assert.strictEqual(tip.height, newHeight); }); }); }); From 1cf623c804a095a7ea882a076d2ae33f99e0fbb1 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Fri, 2 Feb 2024 12:17:47 +0400 Subject: [PATCH 06/13] test: add standalone wallet and alternate chain rescan test for wallet. --- test/wallet-rescan-test.js | 95 ++++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index 92381d746..e39b27d71 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -38,6 +38,8 @@ const combinations = [ // { SPV: true, STANDALONE: true, name: 'SPV/Standalone' } ]; +const noSPVcombinations = combinations.filter(c => !c.SPV); + describe('Wallet rescan', function() { const network = Network.get('regtest'); @@ -291,20 +293,23 @@ describe('Wallet rescan', function() { }); } - describe('Deadlock', function() { + for (const {STANDALONE, name} of noSPVcombinations) { + describe(`Deadlock (${name} Integration)`, function() { + this.timeout(10000); const nodes = new NodesContext(network, 1); let minerCtx; let nodeCtx, address, node, wdb; before(async () => { nodes.init({ - memory: true, + memory: false, wallet: false }); nodes.addNode({ - memory: true, - wallet: true + memory: false, + wallet: true, + standalone: STANDALONE }); await nodes.open(); @@ -322,20 +327,37 @@ describe('Wallet rescan', function() { }); it('should generate 20 blocks', async () => { - await minerCtx.mineBlocks(20, address); - await forEventCondition(nodeCtx.chain, 'connect', (entry) => { - return entry.height === 20; + const BLOCKS = 20; + const chainBlocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === BLOCKS; }); + + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === BLOCKS; + }); + + await minerCtx.mineBlocks(BLOCKS, address); + await chainBlocks; + await wdbBlocks; }); it('should rescan when receiving a block', async () => { const preTip = await wdb.getTip(); + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === preTip.height + 5; + }); + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === preTip.height + 5; + }); await Promise.all([ minerCtx.mineBlocks(5, address), wdb.rescan(0) ]); + await blocks; + await wdbBlocks; + const wdbTip = await wdb.getTip(); assert.strictEqual(wdbTip.height, preTip.height + 5); }); @@ -349,12 +371,20 @@ describe('Wallet rescan', function() { return entry.height === minerHeight + BLOCKS; }); - await Promise.all([ - wdb.rescan(0), + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === minerHeight + BLOCKS; + }); + + const promises = [ minerCtx.mineBlocks(BLOCKS, address) - ]); + ]; + + await forEvent(node.chain, 'connect'); + promises.push(wdb.rescan(0)); + await Promise.all(promises); await blocks; + await wdbBlocks; const tip = await wdb.getTip(); @@ -371,17 +401,56 @@ describe('Wallet rescan', function() { return entry.height === newHeight; }, 10000); + const walletBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === newHeight; + }, 10000); + const reorgEntry = await minerCtx.chain.getEntry(reorgHeight); - await Promise.all([ - wdb.rescan(0), + const promises = [ minerCtx.mineBlocks(BLOCKS, address, reorgEntry) - ]); + ]; + + // We start rescan only after first disconnect is detected to ensure + // wallet guard is set. + await forEvent(node.chain, 'disconnect'); + promises.push(wdb.rescan(0)); + await Promise.all(promises); await blocks; + await walletBlocks; const tip = await wdb.getTip(); assert.strictEqual(tip.height, newHeight); }); + + // Rescanning alternate chain. + it('should rescan when chain is reorging (alternate chain)', async () => { + const minerHeight = minerCtx.height; + const BLOCKS = 50; + const reorgHeight = minerHeight - 20; + + const reorgEntry = await minerCtx.chain.getEntry(reorgHeight); + const mineBlocks = minerCtx.mineBlocks(BLOCKS, address, reorgEntry); + + // We start rescan only after first disconnect is detected to ensure + // wallet guard is set. + await forEvent(node.chain, 'disconnect'); + let err; + try { + // Because we are rescanning within the rescan blocks, + // these blocks will end up in alternate chain, resulting + // in error. + await wdb.rescan(minerHeight - 5); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'Cannot rescan an alternate chain.'); + + await mineBlocks; + }); }); + } }); From a405ea9009a574fb7d8e0e84cd60d0876f1d749c Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Tue, 6 Feb 2024 19:13:17 +0400 Subject: [PATCH 07/13] wdb: Add sanity checks to the wdb.addBlock. wdb/client: fix standalone ChainEntry deserialization and add prevBlock. --- lib/wallet/client.js | 15 ++- lib/wallet/walletdb.js | 48 +++++++- test/util/wallet.js | 36 +++--- test/wallet-coinselection-test.js | 34 ++++-- test/wallet-unit-test.js | 185 +++++++++++++++++++++++++++++- 5 files changed, 283 insertions(+), 35 deletions(-) diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 67bd3995a..b67891ee5 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -11,6 +11,7 @@ const NodeClient = require('../client/node'); const TX = require('../primitives/tx'); const Coin = require('../primitives/coin'); const NameState = require('../covenants/namestate'); +const {encoding} = require('bufio'); const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), @@ -115,12 +116,18 @@ function parseEntry(data) { assert(Buffer.isBuffer(data)); // Just enough to read the three data below - assert(data.length >= 44); + assert(data.length >= 80); + + const hash = data.slice(0, 32); + const height = encoding.readU32(data, 32); + const time = encoding.readU64(data, 40); + const prevBlock = data.slice(48, 80); return { - hash: data.slice(0, 32), - height: data.readUInt32LE(32), - time: data.readUInt32LE(40) + hash, + height, + time, + prevBlock }; } diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 16cf0aa1f..3caa33cf1 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -517,7 +517,7 @@ class WalletDB extends EventEmitter { * Rescan blockchain from a given height. * Needs this.rescanning = true to be set from the caller. * @private - * @param {Number?} height + * @param {Number} [height=this.state.startHeight] * @returns {Promise} */ @@ -2181,7 +2181,7 @@ class WalletDB extends EventEmitter { /** * Get a wallet block meta. - * @param {Hash} hash + * @param {Number} height * @returns {Promise} */ @@ -2305,7 +2305,7 @@ class WalletDB extends EventEmitter { * @private * @param {ChainEntry} entry * @param {TX[]} txs - * @returns {Promise} + * @returns {Promise} - Number of transactions added. */ async _addBlock(entry, txs) { @@ -2315,7 +2315,17 @@ class WalletDB extends EventEmitter { this.logger.warning( 'WalletDB is connecting low blocks (%d).', tip.height); - return 0; + + const block = await this.getBlock(tip.height); + assert(block); + + if (!entry.hash.equals(block.hash)) { + // Maybe we run syncChain here. + this.logger.warning( + 'Unusual reorg at low height (%d).', + tip.height); + } + return -1; } if (tip.height >= this.network.block.slowHeight) @@ -2329,9 +2339,37 @@ class WalletDB extends EventEmitter { // updated before the block was fully // processed (in the case of a crash). this.logger.warning('Already saw WalletDB block (%d).', tip.height); + + const block = await this.getBlock(tip.height); + assert(block); + + if (!entry.hash.equals(block.hash)) { + this.logger.warning( + 'Unusual reorg at the same height (%d).', + tip.height); + + // Maybe we can run syncChain here. + return -1; + } } else if (tip.height !== this.state.height + 1) { await this._rescan(this.state.height); - return 0; + return -1; + } + + let block; + + if (tip.height > 2) { + block = await this.getBlock(tip.height - 1); + assert(block); + } + + if (block && !block.hash.equals(entry.prevBlock)) { + // We can trigger syncChain here as well. + this.logger.warning( + 'Unusual reorg at height (%d).', + tip.height); + + return -1; } const walletTxs = []; diff --git a/test/util/wallet.js b/test/util/wallet.js index 812ea34de..255024790 100644 --- a/test/util/wallet.js +++ b/test/util/wallet.js @@ -1,18 +1,20 @@ 'use strict'; +const assert = require('bsert'); const blake2b = require('bcrypto/lib/blake2b'); const random = require('bcrypto/lib/random'); -const Block = require('../../lib/primitives/block'); const ChainEntry = require('../../lib/blockchain/chainentry'); const Input = require('../../lib/primitives/input'); const Outpoint = require('../../lib/primitives/outpoint'); +const {ZERO_HASH} = require('../../lib/protocol/consensus'); const walletUtils = exports; -walletUtils.fakeBlock = (height) => { - const prev = blake2b.digest(fromU32((height - 1) >>> 0)); - const hash = blake2b.digest(fromU32(height >>> 0)); - const root = blake2b.digest(fromU32((height | 0x80000000) >>> 0)); +walletUtils.fakeBlock = (height, prevSeed = 0, seed = prevSeed) => { + assert(height >= 0); + const prev = height === 0 ? ZERO_HASH : blake2b.digest(fromU32(((height - 1) ^ prevSeed) >>> 0)); + const hash = blake2b.digest(fromU32((height ^ seed) >>> 0)); + const root = blake2b.digest(fromU32((height | 0x80000000 ^ seed) >>> 0)); return { hash: hash, @@ -36,22 +38,26 @@ walletUtils.dummyInput = () => { return Input.fromOutpoint(new Outpoint(hash, 0)); }; -walletUtils.nextBlock = (wdb) => { - return walletUtils.fakeBlock(wdb.state.height + 1); +walletUtils.nextBlock = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeBlock(wdb.state.height + 1, prevSeed, seed); }; -walletUtils.curBlock = (wdb) => { - return walletUtils.fakeBlock(wdb.state.height); +walletUtils.curBlock = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeBlock(wdb.state.height, prevSeed, seed); }; -walletUtils.nextEntry = (wdb) => { - const cur = walletUtils.curEntry(wdb); - const next = new Block(walletUtils.nextBlock(wdb)); - return ChainEntry.fromBlock(next, cur); +walletUtils.fakeEntry = (height, prevSeed = 0, curSeed = prevSeed) => { + const cur = walletUtils.fakeBlock(height, prevSeed, curSeed); + return new ChainEntry(cur);; }; -walletUtils.curEntry = (wdb) => { - return new ChainEntry(walletUtils.curBlock(wdb)); +walletUtils.nextEntry = (wdb, curSeed = 0, nextSeed = curSeed) => { + const next = walletUtils.nextBlock(wdb, curSeed, nextSeed); + return new ChainEntry(next); +}; + +walletUtils.curEntry = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeEntry(wdb.state.height, seed); }; function fromU32(num) { diff --git a/test/wallet-coinselection-test.js b/test/wallet-coinselection-test.js index f4cf9e940..ab8d13c41 100644 --- a/test/wallet-coinselection-test.js +++ b/test/wallet-coinselection-test.js @@ -7,11 +7,30 @@ const { WalletDB, policy } = require('..'); +const {BlockMeta} = require('../lib/wallet/records'); // Use main instead of regtest because (deprecated) // CoinSelector.MAX_FEE was network agnostic const network = Network.get('main'); +function dummyBlock(tipHeight) { + const height = tipHeight + 1; + const hash = Buffer.alloc(32); + hash.writeUInt16BE(height); + + const prevHash = Buffer.alloc(32); + prevHash.writeUInt16BE(tipHeight); + + const dummyBlock = { + hash, + height, + time: Date.now(), + prevBlock: prevHash + }; + + return dummyBlock; +} + async function fundWallet(wallet, amounts) { assert(Array.isArray(amounts)); @@ -21,15 +40,8 @@ async function fundWallet(wallet, amounts) { mtx.addOutput(addr, amt); } - const height = wallet.wdb.height + 1; - const hash = Buffer.alloc(32); - hash.writeUInt16BE(height); - const dummyBlock = { - hash, - height, - time: Date.now() - }; - await wallet.wdb.addBlock(dummyBlock, [mtx.toTX()]); + const dummy = dummyBlock(wallet.wdb.height); + await wallet.wdb.addBlock(dummy, [mtx.toTX()]); } describe('Wallet Coin Selection', function () { @@ -41,6 +53,10 @@ describe('Wallet Coin Selection', function () { await wdb.open(); wdb.height = network.txStart + 1; wdb.state.height = wdb.height; + + const dummy = dummyBlock(network.txStart + 1); + const record = BlockMeta.fromEntry(dummy); + await wdb.setTip(record); wallet = wdb.primary; }); diff --git a/test/wallet-unit-test.js b/test/wallet-unit-test.js index ce7749a73..04315cfc4 100644 --- a/test/wallet-unit-test.js +++ b/test/wallet-unit-test.js @@ -10,15 +10,19 @@ const { Mnemonic, WalletDB, Network, - wallet: { Wallet } + wallet: { Wallet }, + MTX } = require('../lib/hsd'); const Account = require('../lib/wallet/account'); +const wutils = require('./util/wallet'); +const {nextEntry, fakeEntry} = require('./util/wallet'); +const MemWallet = require('./util/memwallet'); const mnemonics = require('./data/mnemonic-english.json'); const network = Network.get('main'); describe('Wallet Unit Tests', () => { - describe('constructor', () => { + describe('constructor', function() { // abandon, abandon... about const phrase = mnemonics[0][1]; const passphrase = mnemonics[0][2]; @@ -346,4 +350,181 @@ describe('Wallet Unit Tests', () => { } }); }); + + describe('addBlock', function() { + const ALT_SEED = 0xdeadbeef; + + let wdb, wallet, memwallet; + + beforeEach(async () => { + wdb = new WalletDB({ + network: network.type, + memory: true + }); + + await wdb.open(); + wallet = wdb.primary; + + memwallet = new MemWallet({ + network + }); + + for (let i = 0; i < 10; i++) { + const entry = nextEntry(wdb); + await wdb.addBlock(entry, []); + } + }); + + afterEach(async () => { + await wdb.close(); + wdb = null; + }); + + // Move forward + it('should progress with 10 block', async () => { + const tip = await wdb.getTip(); + + for (let i = 0; i < 10; i++) { + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, 0); + assert.equal(wdb.height, entry.height); + } + + assert.strictEqual(wdb.height, tip.height + 10); + }); + + it('should return number of transactions added (owned)', async () => { + const tip = await wdb.getTip(); + const wtx = await fakeWTX(wallet); + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, [wtx]); + + assert.strictEqual(added, 1); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should return number of transactions added (none)', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, []); + + assert.strictEqual(added, 0); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should fail to add block on unusual reorg', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb, ALT_SEED, ALT_SEED); + + // TODO: Detect sync chain is correct. + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, -1); + assert.strictEqual(wdb.height, tip.height); + }); + + // Same block + it('should re-add the same block', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb); + const wtx1 = await fakeWTX(wallet); + const wtx2 = await fakeWTX(wallet); + + const added1 = await wdb.addBlock(entry, [wtx1]); + assert.strictEqual(added1, 1); + assert.equal(wdb.height, tip.height + 1); + + // Same TX wont show up second time. + const added2 = await wdb.addBlock(entry, [wtx1]); + assert.strictEqual(added2, 0); + assert.equal(wdb.height, tip.height + 1); + + const added3 = await wdb.addBlock(entry, [wtx1, wtx2]); + assert.strictEqual(added3, 1); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should ignore txs not owned by wallet', async () => { + const tip = await wdb.getTip(); + const addr = memwallet.getReceive().toString(network); + const tx = fakeTX(addr); + + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, [tx]); + assert.strictEqual(added, 0); + + assert.strictEqual(wdb.height, tip.height + 1); + }); + + // This should not happen, but there should be guards in place. + it('should resync if the block is the same', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height, 0, ALT_SEED); + + // TODO: Detect sync chain is correct. + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, -1); + }); + + // LOW BLOCKS + it('should ignore blocks before tip', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height - 1); + const wtx = await fakeWTX(wallet); + + // ignore low blocks. + const added = await wdb.addBlock(entry, [wtx]); + assert.strictEqual(added, -1); + assert.strictEqual(wdb.height, tip.height); + }); + + it('should sync chain blocks before tip on unusual low block reorg', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height - 1, 0, ALT_SEED); + const wtx = await fakeWTX(wallet); + + // TODO: Detect sync chain is correct. + + // ignore low blocks. + const added = await wdb.addBlock(entry, [wtx]); + assert.strictEqual(added, -1); + assert.strictEqual(wdb.height, tip.height); + }); + + // HIGH BLOCKS + it('should rescan for missed blocks', async () => { + const tip = await wdb.getTip(); + // next + 1 + const entry = fakeEntry(tip.height + 2); + + let rescan = false; + let rescanHash = null; + + wdb.client.rescan = async (hash) => { + rescan = true; + rescanHash = hash; + }; + + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, -1); + + assert.strictEqual(rescan, true); + assert.bufferEqual(rescanHash, tip.hash); + }); + }); }); + +function fakeTX(addr) { + const tx = new MTX(); + tx.addInput(wutils.dummyInput()); + tx.addOutput({ + address: addr, + value: 5460 + }); + return tx.toTX(); +} + +async function fakeWTX(wallet) { + const addr = await wallet.receiveAddress(); + return fakeTX(addr); +} From 4d9127dfb8964b6477099f039e7bfea20312b344 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Mon, 12 Feb 2024 22:52:35 +0400 Subject: [PATCH 08/13] test: add rescan and addBlock tests when addresses are gapped. --- test/wallet-rescan-test.js | 341 ++++++++++++++++++++++++++++++++++++- 1 file changed, 336 insertions(+), 5 deletions(-) diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index e39b27d71..920e0ae37 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -2,9 +2,11 @@ const assert = require('bsert'); const Network = require('../lib/protocol/network'); +const Address = require('../lib/primitives/address'); +const HDPublicKey = require('../lib/hd/public'); const NodesContext = require('./util/nodes-context'); const {forEvent, forEventCondition} = require('./util/common'); -const {Balance, getWClientBalance} = require('./util/balance'); +const {Balance, getWClientBalance, getBalance} = require('./util/balance'); // Definitions: // Gapped txs/addresses - addresses with lookahead + 1 gap when deriving. @@ -27,9 +29,6 @@ const {Balance, getWClientBalance} = require('./util/balance'); // recovery is impossible. This tests situation where in block // derivation depth is lower than wallet lookahead. -// TODO: Add the standalone Wallet variation. -// TODO: Add initial rescan test. - const combinations = [ { SPV: false, STANDALONE: false, name: 'Full/Plugin' }, { SPV: false, STANDALONE: true, name: 'Full/Standalone' }, @@ -40,9 +39,341 @@ const combinations = [ const noSPVcombinations = combinations.filter(c => !c.SPV); -describe('Wallet rescan', function() { +describe('Wallet rescan/addBlock', function() { const network = Network.get('regtest'); + // TODO: Add SPV tests. + for (const {SPV, STANDALONE, name} of noSPVcombinations) { + describe(`rescan/addBlock gapped addresses (${name} Integration)`, function() { + const TEST_LOOKAHEAD = 20; + + const MAIN = 0; + const TEST_ADDBLOCK = 1; + const TEST_RESCAN = 2; + + const WALLET_NAME = 'test'; + const ACCOUNT = 'default'; + + const network = Network.get('regtest'); + + /** @type {NodesContext} */ + let nodes; + let minerWallet, minerAddress; + let main, addBlock, rescan; + + const deriveAddresses = async (wallet, depth) => { + const accInfo = await wallet.getAccount('default'); + let currentDepth = accInfo.receiveDepth; + + if (depth <= currentDepth) + return; + + while (currentDepth !== depth) { + const addr = await wallet.createAddress('default'); + currentDepth = addr.index; + } + }; + + const getAddress = async (wallet, depth = -1) => { + const accInfo = await wallet.getAccount('default'); + const {accountKey, lookahead} = accInfo; + + if (depth === -1) + depth = accInfo.receiveDepth; + + const XPUBKey = HDPublicKey.fromBase58(accountKey, network); + const key = XPUBKey.derive(0).derive(depth).publicKey; + const address = Address.fromPubkey(key); + + const gappedDepth = depth + lookahead + 1; + return {address, depth, gappedDepth}; + }; + + const generateGappedAddresses = async (wallet, count) => { + let depth = -1; + + const addresses = []; + + // generate gapped addresses. + for (let i = 0; i < count; i++) { + const addrInfo = await getAddress(wallet, depth); + + addresses.push({ + address: addrInfo.address, + depth: addrInfo.depth, + gappedDepth: addrInfo.gappedDepth + }); + + await deriveAddresses(wallet, depth); + depth = addrInfo.gappedDepth; + } + + return addresses; + }; + + before(async () => { + // Initial node is the one that progresses the network. + nodes = new NodesContext(network, 1); + // MAIN_WALLET = 0 + nodes.init({ + wallet: true, + standalone: true, + memory: true, + noDNS: true + }); + + // Add the testing node. + // TEST_ADDBLOCK = 1 + nodes.addNode({ + spv: SPV, + wallet: true, + memory: true, + standalone: STANDALONE, + noDNS: true + }); + + // Add the rescan test node. + // TEST_RESCAN = 2 + nodes.addNode({ + spv: SPV, + wallet: true, + memory: true, + standalone: STANDALONE, + noDNS: true + }); + + await nodes.open(); + + const mainWClient = nodes.context(MAIN).wclient; + minerWallet = nodes.context(MAIN).wclient.wallet('primary'); + minerAddress = (await minerWallet.createAddress('default')).address; + + const mainWallet = await mainWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD + }); + assert(mainWallet); + + const master = await mainWClient.getMaster(WALLET_NAME); + + const addBlockWClient = nodes.context(TEST_ADDBLOCK).wclient; + const addBlockWalletResult = await addBlockWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD, + mnemonic: master.mnemonic.phrase + }); + assert(addBlockWalletResult); + + const rescanWClient = nodes.context(TEST_RESCAN).wclient; + const rescanWalletResult = await rescanWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD, + mnemonic: master.mnemonic.phrase + }); + assert(rescanWalletResult); + + main = {}; + main.client = mainWClient.wallet(WALLET_NAME); + await main.client.open(); + main.wdb = nodes.context(MAIN).wdb; + + addBlock = {}; + addBlock.client = addBlockWClient.wallet(WALLET_NAME); + await addBlock.client.open(); + addBlock.wdb = nodes.context(TEST_ADDBLOCK).wdb; + + rescan = {}; + rescan.client = rescanWClient.wallet(WALLET_NAME); + await rescan.client.open(); + rescan.wdb = nodes.context(TEST_RESCAN).wdb; + + await nodes.generate(MAIN, 10, minerAddress); + }); + + after(async () => { + await nodes.close(); + await nodes.destroy(); + }); + + // Prepare for the rescan and addBlock tests. + it('should send gapped txs on each block', async () => { + const expectedRescanBalance = await getBalance(main.client, ACCOUNT); + const blocks = 5; + + // 1 address per block, all of them gapped. + // Start after first gap, make sure rescan has no clue. + const all = await generateGappedAddresses(main.client, blocks + 1); + const addresses = all.slice(1); + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const mainWalletBlocks = forEvent(main.wdb, 'block connect', blocks); + const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect', blocks); + const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect', blocks); + + for (let i = 0; i < blocks; i++) { + await minerWallet.send({ + outputs: [{ + address: addresses[i].address.toString(network), + value: 1e6 + }] + }); + + await nodes.generate(MAIN, 1, minerAddress); + } + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + // before the rescan test. + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); + + it('should receive gapped txs on each block (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); + + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); + }); + + it('should receive gapped txs on each block (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + // give rescan first address. + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + + it('should send gapped txs in the same block', async () => { + const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); + const txCount = 5; + + const all = await generateGappedAddresses(main.client, txCount + 1); + const addresses = all.slice(1); + + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const mainWalletBlocks = forEvent(main.wdb, 'block connect'); + const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect'); + const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect'); + + for (const {address} of addresses) { + await minerWallet.send({ + outputs: [{ + address: address.toString(network), + value: 1e6 + }] + }); + } + + await nodes.generate(MAIN, 1, minerAddress); + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); + + it.skip('should receive gapped txs in the same block (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); + + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); + }); + + it.skip('should receive gapped txs in the same block (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + + it('should send gapped outputs in the same tx', async () => { + const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); + const outCount = 5; + + const all = await generateGappedAddresses(main.client, outCount + 1); + const addresses = all.slice(1); + + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const mainWalletBlocks = forEvent(main.wdb, 'block connect'); + const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect'); + const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect'); + + const outputs = addresses.map(({address}) => ({ + address: address.toString(network), + value: 1e6 + })); + + await minerWallet.send({outputs}); + await nodes.generate(MAIN, 1, minerAddress); + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); + + it.skip('should receive gapped outputs in the same tx (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); + + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); + }); + + it.skip('should receive gapped outputs in the same tx (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + }); + } + for (const {SPV, STANDALONE, name} of combinations) { describe(`Initial sync/rescan (${name} Integration)`, function() { // Test wallet plugin/standalone is disabled and re-enabled after some time: From 81821ff17aba5ebaf2a09b002a74cc268ac56240 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Tue, 13 Feb 2024 21:04:28 +0400 Subject: [PATCH 09/13] wallet: add filterUpdated to wallet.add, wdb.addTX and wdb.addBlock. --- lib/wallet/account.js | 2 +- lib/wallet/wallet.js | 31 ++++++++++------ lib/wallet/walletdb.js | 60 ++++++++++++++++++++++++------- test/wallet-auction-test.js | 8 +++-- test/wallet-unit-test.js | 72 ++++++++++++++++++++++++++----------- 5 files changed, 125 insertions(+), 48 deletions(-) diff --git a/lib/wallet/account.js b/lib/wallet/account.js index d60c7903b..582d3a747 100644 --- a/lib/wallet/account.js +++ b/lib/wallet/account.js @@ -512,7 +512,7 @@ class Account extends bio.Struct { * Allocate new lookahead addresses if necessary. * @param {Number} receiveDepth * @param {Number} changeDepth - * @returns {Promise} - Returns {@link WalletKey}. + * @returns {Promise} */ async syncDepth(b, receive, change) { diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 1d5a8ce79..9c23aa88b 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -46,6 +46,12 @@ const Outpoint = require('../primitives/outpoint'); const EMPTY = Buffer.alloc(0); +/** + * @typedef {Object} AddResult + * @property {Details} details + * @property {WalletKey[]} derived + */ + /** * Wallet * @alias module:wallet.Wallet @@ -4360,7 +4366,7 @@ class Wallet extends EventEmitter { * This is used for deriving new addresses when * a confirmed transaction is seen. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} - derived rings. */ async syncOutputDepth(tx) { @@ -4737,7 +4743,7 @@ class Wallet extends EventEmitter { /** * Add a transaction to the wallets TX history. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async add(tx, block) { @@ -4754,21 +4760,26 @@ class Wallet extends EventEmitter { * Potentially resolves orphans. * @private * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async _add(tx, block) { const details = await this.txdb.add(tx, block); - if (details) { - const derived = await this.syncOutputDepth(tx); - if (derived.length > 0) { - this.wdb.emit('address', this, derived); - this.emit('address', derived); - } + if (!details) + return null; + + const derived = await this.syncOutputDepth(tx); + + if (derived.length > 0) { + this.wdb.emit('address', this, derived); + this.emit('address', derived); } - return details; + return { + details, + derived + }; } /** diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 3caa33cf1..9c8098fc2 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -31,6 +31,8 @@ const tlayout = layouts.txdb; const {states} = require('../covenants/namestate'); const util = require('../utils/util'); +/** @typedef {import('../primitives/tx')} TX */ + const { ChainState, BlockMeta, @@ -38,6 +40,17 @@ const { MapRecord } = records; +/** + * @typedef {Object} AddBlockResult + * @property {Number} txs - Number of transactions added on this add. + * @property {Boolean} filterUpdated - Whether the bloom filter was updated. + */ + +/** + * @typedef {Object} AddTXResult + * @property {Number} wids - Wallet IDs affected. + * @property {Boolean} filterUpdated - Whether the bloom filter was updated. + /** * WalletDB * @alias module:wallet.WalletDB @@ -2287,7 +2300,8 @@ class WalletDB extends EventEmitter { /** * Add a block's transactions and write the new best hash. * @param {ChainEntry} entry - * @returns {Promise} + * @param {TX[]} txs + * @returns {Promise} */ async addBlock(entry, txs) { @@ -2305,7 +2319,7 @@ class WalletDB extends EventEmitter { * @private * @param {ChainEntry} entry * @param {TX[]} txs - * @returns {Promise} - Number of transactions added. + * @returns {Promise} */ async _addBlock(entry, txs) { @@ -2325,7 +2339,8 @@ class WalletDB extends EventEmitter { 'Unusual reorg at low height (%d).', tip.height); } - return -1; + + return null; } if (tip.height >= this.network.block.slowHeight) @@ -2349,11 +2364,11 @@ class WalletDB extends EventEmitter { tip.height); // Maybe we can run syncChain here. - return -1; + return null; } } else if (tip.height !== this.state.height + 1) { await this._rescan(this.state.height); - return -1; + return null; } let block; @@ -2369,10 +2384,11 @@ class WalletDB extends EventEmitter { 'Unusual reorg at height (%d).', tip.height); - return -1; + return null; } const walletTxs = []; + let filterUpdated = false; try { // We set the state as confirming so that @@ -2382,8 +2398,13 @@ class WalletDB extends EventEmitter { this.confirming = true; for (const tx of txs) { - if (await this._addTX(tx, tip)) { + const txadded = await this._addTX(tx, tip); + + if (txadded) { walletTxs.push(tx); + + if (txadded.filterUpdated) + filterUpdated = true; } } @@ -2401,7 +2422,10 @@ class WalletDB extends EventEmitter { this.emit('block connect', entry, walletTxs); - return walletTxs.length; + return { + txs: walletTxs.length, + filterUpdated: filterUpdated + }; } /** @@ -2506,7 +2530,7 @@ class WalletDB extends EventEmitter { * to wallet IDs, potentially store orphans, resolve * orphans, or confirm a transaction. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async addTX(tx) { @@ -2523,7 +2547,7 @@ class WalletDB extends EventEmitter { * @private * @param {TX} tx * @param {BlockMeta} block - * @returns {Promise} + * @returns {Promise} */ async _addTX(tx, block) { @@ -2539,6 +2563,7 @@ class WalletDB extends EventEmitter { wids.size, tx.txid()); let result = false; + let filterUpdated = false; // Insert the transaction // into every matching wallet. @@ -2547,18 +2572,27 @@ class WalletDB extends EventEmitter { assert(wallet); - if (await wallet.add(tx, block)) { + const wadded = await wallet.add(tx, block); + + if (wadded) { + result = true; + + if (wadded.derived.length > 0) + filterUpdated = true; + this.logger.info( 'Added transaction to wallet in WalletDB: %s (%d).', wallet.id, wid); - result = true; } } if (!result) return null; - return wids; + return { + wids, + filterUpdated + }; } /** diff --git a/test/wallet-auction-test.js b/test/wallet-auction-test.js index 267e111dd..ece157689 100644 --- a/test/wallet-auction-test.js +++ b/test/wallet-auction-test.js @@ -131,8 +131,9 @@ describe('Wallet Auction', function() { const openMTX = openTXs[openIndex++]; const tx = openMTX.toTX(); const addResult = await wdb.addTX(tx); - assert.strictEqual(addResult.size, 1); - assert.ok(addResult.has(wallet.wid)); + assert.ok(addResult); + assert.strictEqual(addResult.wids.size, 1); + assert.ok(addResult.wids.has(wallet.wid)); const pending = await wallet.getPending(); assert.strictEqual(pending.length, 1); @@ -305,7 +306,8 @@ describe('Wallet Auction', function() { // double opens are properly removed. await wallet.sign(spendMTX); const added = await wdb.addTX(spendMTX.toTX()); - assert.strictEqual(added.size, 1); + assert.ok(added); + assert.strictEqual(added.wids.size, 1); }); it('should mine enough blocks to expire auction (again)', async () => { diff --git a/test/wallet-unit-test.js b/test/wallet-unit-test.js index 04315cfc4..d991085b8 100644 --- a/test/wallet-unit-test.js +++ b/test/wallet-unit-test.js @@ -5,19 +5,19 @@ const blake2b = require('bcrypto/lib/blake2b'); const base58 = require('bcrypto/lib/encoding/base58'); const random = require('bcrypto/lib/random'); const bio = require('bufio'); -const { - HDPrivateKey, - Mnemonic, - WalletDB, - Network, - wallet: { Wallet }, - MTX -} = require('../lib/hsd'); +const Network = require('../lib/protocol/network'); +const MTX = require('../lib/primitives/mtx'); +const HDPrivateKey = require('../lib/hd/private'); +const Mnemonic = require('../lib/hd/mnemonic'); +const WalletDB = require('../lib/wallet/walletdb'); +const Wallet = require('../lib/wallet/wallet'); const Account = require('../lib/wallet/account'); const wutils = require('./util/wallet'); const {nextEntry, fakeEntry} = require('./util/wallet'); const MemWallet = require('./util/memwallet'); +/** @typedef {import('../lib/primitives/tx')} TX */ + const mnemonics = require('./data/mnemonic-english.json'); const network = Network.get('main'); @@ -354,7 +354,12 @@ describe('Wallet Unit Tests', () => { describe('addBlock', function() { const ALT_SEED = 0xdeadbeef; - let wdb, wallet, memwallet; + /** @type {WalletDB} */ + let wdb; + /** @type {Wallet} */ + let wallet; + /** @type {MemWallet} */ + let memwallet; beforeEach(async () => { wdb = new WalletDB({ @@ -387,7 +392,9 @@ describe('Wallet Unit Tests', () => { for (let i = 0; i < 10; i++) { const entry = nextEntry(wdb); const added = await wdb.addBlock(entry, []); - assert.strictEqual(added, 0); + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); assert.equal(wdb.height, entry.height); } @@ -400,7 +407,9 @@ describe('Wallet Unit Tests', () => { const entry = nextEntry(wdb); const added = await wdb.addBlock(entry, [wtx]); - assert.strictEqual(added, 1); + assert.ok(added); + assert.strictEqual(added.txs, 1); + assert.strictEqual(added.filterUpdated, true); assert.equal(wdb.height, tip.height + 1); }); @@ -409,7 +418,9 @@ describe('Wallet Unit Tests', () => { const entry = nextEntry(wdb); const added = await wdb.addBlock(entry, []); - assert.strictEqual(added, 0); + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); assert.equal(wdb.height, tip.height + 1); }); @@ -419,7 +430,7 @@ describe('Wallet Unit Tests', () => { // TODO: Detect sync chain is correct. const added = await wdb.addBlock(entry, []); - assert.strictEqual(added, -1); + assert.strictEqual(added, null); assert.strictEqual(wdb.height, tip.height); }); @@ -431,16 +442,23 @@ describe('Wallet Unit Tests', () => { const wtx2 = await fakeWTX(wallet); const added1 = await wdb.addBlock(entry, [wtx1]); - assert.strictEqual(added1, 1); + assert.ok(added1); + assert.strictEqual(added1.txs, 1); + assert.strictEqual(added1.filterUpdated, true); assert.equal(wdb.height, tip.height + 1); // Same TX wont show up second time. const added2 = await wdb.addBlock(entry, [wtx1]); - assert.strictEqual(added2, 0); + assert.ok(added2); + assert.strictEqual(added2.txs, 0); + assert.strictEqual(added2.filterUpdated, false); assert.equal(wdb.height, tip.height + 1); const added3 = await wdb.addBlock(entry, [wtx1, wtx2]); - assert.strictEqual(added3, 1); + assert.ok(added3); + assert.strictEqual(added3.txs, 1); + // Both txs are using the same address. + assert.strictEqual(added3.filterUpdated, false); assert.equal(wdb.height, tip.height + 1); }); @@ -451,7 +469,9 @@ describe('Wallet Unit Tests', () => { const entry = nextEntry(wdb); const added = await wdb.addBlock(entry, [tx]); - assert.strictEqual(added, 0); + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); assert.strictEqual(wdb.height, tip.height + 1); }); @@ -463,7 +483,7 @@ describe('Wallet Unit Tests', () => { // TODO: Detect sync chain is correct. const added = await wdb.addBlock(entry, []); - assert.strictEqual(added, -1); + assert.strictEqual(added, null); }); // LOW BLOCKS @@ -474,7 +494,7 @@ describe('Wallet Unit Tests', () => { // ignore low blocks. const added = await wdb.addBlock(entry, [wtx]); - assert.strictEqual(added, -1); + assert.strictEqual(added, null); assert.strictEqual(wdb.height, tip.height); }); @@ -487,7 +507,7 @@ describe('Wallet Unit Tests', () => { // ignore low blocks. const added = await wdb.addBlock(entry, [wtx]); - assert.strictEqual(added, -1); + assert.strictEqual(added, null); assert.strictEqual(wdb.height, tip.height); }); @@ -506,7 +526,7 @@ describe('Wallet Unit Tests', () => { }; const added = await wdb.addBlock(entry, []); - assert.strictEqual(added, -1); + assert.strictEqual(added, null); assert.strictEqual(rescan, true); assert.bufferEqual(rescanHash, tip.hash); @@ -514,6 +534,11 @@ describe('Wallet Unit Tests', () => { }); }); +/** + * @param {String} addr + * @returns {TX} + */ + function fakeTX(addr) { const tx = new MTX(); tx.addInput(wutils.dummyInput()); @@ -524,6 +549,11 @@ function fakeTX(addr) { return tx.toTX(); } +/** + * @param {Wallet} wallet + * @returns {Promise} + */ + async function fakeWTX(wallet) { const addr = await wallet.receiveAddress(); return fakeTX(addr); From c8592d121d0805c96140bca46d25088dfa23c4a2 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 14 Feb 2024 17:35:53 +0400 Subject: [PATCH 10/13] wallet: Fix node client interface for hooks. bsock hooks that nodeclient tries to imitate, are handlers set on specific event. They are expected to return results to the caller. --- lib/wallet/nodeclient.js | 47 ++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 40594d03e..a477ebf8a 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -7,6 +7,7 @@ 'use strict'; const assert = require('bsert'); +const blacklist = require('bsock/lib/blacklist'); const AsyncEmitter = require('bevent'); /** @@ -27,6 +28,7 @@ class NodeClient extends AsyncEmitter { this.network = node.network; this.filter = null; this.opened = false; + this.hooks = new Map(); this.init(); } @@ -98,13 +100,46 @@ class NodeClient extends AsyncEmitter { } /** - * Add a listener. - * @param {String} type + * Add a hook. + * @param {String} event * @param {Function} handler */ - hook(type, handler) { - return this.on(type, handler); + hook(event, handler) { + assert(typeof event === 'string', 'Event must be a string.'); + assert(typeof handler === 'function', 'Handler must be a function.'); + assert(!this.hooks.has(event), 'Hook already bound.'); + assert(!Object.prototype.hasOwnProperty.call(blacklist, event), + 'Blacklisted event.'); + this.hooks.set(event, handler); + } + + /** + * Remove a hook. + * @param {String} event + */ + + unhook(event) { + assert(typeof event === 'string', 'Event must be a string.'); + assert(!Object.prototype.hasOwnProperty.call(blacklist, event), + 'Blacklisted event.'); + this.hooks.delete(event); + } + + /** + * Call a hook. + * @param {String} event + * @param {...Object} args + * @returns {Promise} + */ + + handleCall(event, ...args) { + const hook = this.hooks.get(event); + + if (!hook) + throw new Error('No hook available.'); + + return hook(...args); } /** @@ -215,8 +250,6 @@ class NodeClient extends AsyncEmitter { /** * Rescan for any missed transactions. * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. * @returns {Promise} */ @@ -225,7 +258,7 @@ class NodeClient extends AsyncEmitter { return this.node.chain.reset(start); return this.node.chain.scan(start, this.filter, (entry, txs) => { - return this.emitAsync('block rescan', entry, txs); + return this.handleCall('block rescan', entry, txs); }); } From 57194f4cd6fe7897038a7108d656ecc59db02bc8 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 14 Feb 2024 18:30:02 +0400 Subject: [PATCH 11/13] http: interactive rescan will now throw on socket.call similar to rescan. --- lib/blockchain/chain.js | 13 +++++- lib/blockchain/chaindb.js | 8 +++- lib/blockchain/common.js | 36 +++++++++++++++++ lib/node/http.js | 5 +-- test/node-rescan-test.js | 85 ++++++++++++++++++++++++++++++++++----- 5 files changed, 132 insertions(+), 15 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index f537143e6..3389ce02d 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2266,13 +2266,22 @@ class Chain extends AsyncEmitter { } } + /** @typedef {import('./common').ScanAction} ScanAction */ + + /** + * @callback ScanInteractiveIterCB + * @param {ChainEntry} entry + * @param {TX[]} txs + * @returns {Promise} + */ + /** * Interactive scan the blockchain for transactions containing specified * address hashes. Allows repeat and abort. * @param {Hash|Number} start - Block hash or height to start at. * @param {BloomFilter} filter - Starting bloom filter containing tx, * address and name hashes. - * @param {Function} iter - Iterator. + * @param {ScanInteractiveIterCB} iter - Iterator. * @param {Boolean} [fullLock=false] * @returns {Promise} */ @@ -2300,7 +2309,7 @@ class Chain extends AsyncEmitter { * @param {Hash|Number} start - Block hash or height to start at. * @param {BloomFilter} filter - Starting bloom filter containing tx, * address and name hashes. - * @param {Function} iter - Iterator. + * @param {ScanInteractiveIterCB} iter - Iterator. * @param {Boolean} [lockPerScan=true] - if we should lock per block scan. * @returns {Promise} */ diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 7f075349b..1349c0ea7 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1612,12 +1612,18 @@ class ChainDB { this.logger.info('Finished scanning %d blocks.', total); } + /** + * @typedef {Object} ScanBlockResult + * @property {ChainEntry} entry + * @property {TX[]} txs + */ + /** * Interactive scans block checks. * @param {Hash|Number} blockID - Block hash or height to start at. * @param {BloomFilter} [filter] - Starting bloom filter containing tx, * address and name hashes. - * @returns {Promise} + * @returns {Promise} */ async scanBlock(blockID, filter) { diff --git a/lib/blockchain/common.js b/lib/blockchain/common.js index 2a45e2d45..6a9ab8339 100644 --- a/lib/blockchain/common.js +++ b/lib/blockchain/common.js @@ -84,3 +84,39 @@ exports.scanActions = { REPEAT_ADD: 4, REPEAT: 5 }; + +/** + * @typedef {Object} ActionAbort + * @property {exports.scanActions} type - ABORT + */ + +/** + * @typedef {Object} ActionNext + * @property {exports.scanActions} type - NEXT + */ + +/** + * @typedef {Object} ActionRepeat + * @property {exports.ScanAction} type - REPEAT + */ + +/** + * @typedef {Object} ActionRepeatAdd + * @property {exports.scanActions} type - REPEAT_ADD + * @property {Buffer[]} chunks + */ + +/** + * @typedef {Object} ActionRepeatSet + * @property {exports.scanActions} type - REPEAT_SET + * @property {BloomFilter} filter + */ + +/** + * @typedef {ActionAbort + * | ActionNext + * | ActionRepeat + * | ActionRepeatAdd + * | ActionRepeatSet + * } ScanAction + */ diff --git a/lib/node/http.js b/lib/node/http.js index 93277edb2..c30db1ff1 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -925,10 +925,9 @@ class HTTP extends Server { try { await this.node.scanInteractive(start, filter, iter, fullLock); } catch (err) { - return socket.call('block rescan interactive abort', err.message); + await socket.call('block rescan interactive abort', err.message); + throw err; } - - return null; } } diff --git a/test/node-rescan-test.js b/test/node-rescan-test.js index 86584b7b2..40cbc5069 100644 --- a/test/node-rescan-test.js +++ b/test/node-rescan-test.js @@ -543,7 +543,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -554,7 +561,15 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight, null); + err = null; + try { + await client.rescanInteractive(startHeight, null); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -596,7 +611,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -606,7 +628,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight); + err = null; + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -648,7 +677,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -658,7 +694,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight); + err = null; + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -705,7 +748,15 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, tests.length); assert.strictEqual(aborted, true); }); @@ -748,17 +799,33 @@ describe('Node Rescan Interactive API', function() { aborted = true; }); - await client.rescanInteractive(startHeight, filter.encode()); + let err; + try { + await client.rescanInteractive(startHeight, filter.encode()); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(aborted, true); // Now try using client.filter + err = null; aborted = false; filter = BloomFilter.fromRate(10000, 0.001); testTXs = allTXs[startHeight].slice(); expected = 0; await client.setFilter(filter.encode()); - await client.rescanInteractive(startHeight); + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(aborted, true); }); From 38efb06904437c7b6714955fc8f3725f9a9721c8 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 14 Feb 2024 20:57:07 +0400 Subject: [PATCH 12/13] wallet: Use `interactive scan` on initial sync and rescan. Check issue #872 --- CHANGELOG.md | 7 +- lib/wallet/client.js | 18 +++ lib/wallet/nodeclient.js | 28 ++++ lib/wallet/nullclient.js | 13 +- lib/wallet/txdb.js | 5 +- lib/wallet/wallet.js | 2 +- lib/wallet/walletdb.js | 100 ++++++++++++- test/wallet-rescan-test.js | 296 +++++++++++++++++++++++++------------ test/wallet-unit-test.js | 2 +- 9 files changed, 360 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5410c7541..fa7b70afe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,9 @@ process and allows parallel rescans. - Add `getFee`, an HTTP alternative to estimateFee socket call. ### Wallet Changes +- Add migration that recalculates txdb balances to fix any inconsistencies. +- Wallet will now use `interactive scan` for initial sync(on open) and rescan. + #### Configuration - Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you want to disable rescan when migration recommends it. It may result in the @@ -54,7 +57,6 @@ process and allows parallel rescans. #### Wallet API -- Add migration that recalculates txdb balances to fix any inconsistencies. - WalletNode now emits `open` and `close` events. - WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`. - WalletDB @@ -63,13 +65,14 @@ process and allows parallel rescans. sync to do the rescan. - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`. -### Wallet HTTP Client +#### Wallet HTTP - All transaction creating endpoints now accept `hardFee` for specifying the exact fee. - All transaction sending endpoints now fundlock/queue tx creation. (no more conflicting transactions) - Add options to `getNames` for passing `own`. + ## v6.0.0 ### Node and Wallet HTTP API diff --git a/lib/wallet/client.js b/lib/wallet/client.js index b67891ee5..0f7e3a729 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -17,6 +17,7 @@ const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), 'block disconnect': entry => [parseEntry(entry)], 'block rescan': (entry, txs) => parseBlock(entry, txs), + 'block rescan interactive': (entry, txs) => parseBlock(entry, txs), 'chain reset': entry => [parseEntry(entry)], 'tx': tx => [TX.decode(tx)] }; @@ -75,10 +76,27 @@ class WalletClient extends NodeClient { return super.setFilter(filter.encode()); } + /** + * Rescan for any missed transactions. + * @param {Number|Hash} start - Start block. + * @returns {Promise} + */ + async rescan(start) { return super.rescan(start); } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock) { + return super.rescanInteractive(start, null, fullLock); + } + async getNameStatus(nameHash) { const json = await super.getNameStatus(nameHash); return NameState.fromJSON(json); diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index a477ebf8a..5f181bcbc 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -262,6 +262,34 @@ class NodeClient extends AsyncEmitter { }); } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock = true) { + if (this.node.spv) + return this.node.chain.reset(start); + + const iter = async (entry, txs) => { + return await this.handleCall('block rescan interactive', entry, txs); + }; + + try { + return await this.node.scanInteractive( + start, + this.filter, + iter, + fullLock + ); + } catch (e) { + await this.handleCall('block rescan interactive abort', e.message); + throw e; + } + } + /** * Get name state. * @param {Buffer} nameHash diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index abc30f266..0dcdde0b7 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -165,8 +165,6 @@ class NullClient extends EventEmitter { /** * Rescan for any missed transactions. * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. * @returns {Promise} */ @@ -174,6 +172,17 @@ class NullClient extends EventEmitter { ; } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock) { + ; + } + /** * Get opening bid height. * @param {Buffer} nameHash diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index f92ef8103..2d92e6bd1 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -14,14 +14,13 @@ const Amount = require('../ui/amount'); const CoinView = require('../coins/coinview'); const Coin = require('../primitives/coin'); const Outpoint = require('../primitives/outpoint'); -const records = require('./records'); const layout = require('./layout').txdb; const consensus = require('../protocol/consensus'); const policy = require('../protocol/policy'); const rules = require('../covenants/rules'); const NameState = require('../covenants/namestate'); const NameUndo = require('../covenants/undo'); -const {TXRecord} = records; +const {TXRecord} = require('./records'); const {types} = rules; /* @@ -1486,7 +1485,7 @@ class TXDB { /** * Revert a block. * @param {Number} height - * @returns {Promise} - blocks length + * @returns {Promise} - number of txs removed. */ async revert(height) { diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 9c23aa88b..0423eeb4f 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -4785,7 +4785,7 @@ class Wallet extends EventEmitter { /** * Revert a block. * @param {Number} height - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async revert(height) { diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 9c8098fc2..202f50038 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -30,8 +30,10 @@ const layout = layouts.wdb; const tlayout = layouts.txdb; const {states} = require('../covenants/namestate'); const util = require('../utils/util'); +const {scanActions} = require('../blockchain/common'); /** @typedef {import('../primitives/tx')} TX */ +/** @typedef {import('../blockchain/common').ScanAction} ScanAction */ const { ChainState, @@ -190,6 +192,21 @@ class WalletDB extends EventEmitter { } }); + this.client.hook('block rescan interactive', async (entry, txs) => { + try { + return await this.rescanBlockInteractive(entry, txs); + } catch (e) { + this.emit('error', e); + return { + type: scanActions.ABORT + }; + } + }); + + this.client.hook('block rescan interactive abort', async (message) => { + this.emit('error', new Error(message)); + }); + this.client.bind('tx', async (tx) => { try { await this.addTX(tx); @@ -523,7 +540,7 @@ class WalletDB extends EventEmitter { } // syncNode sets the rescanning to true. - return this.scan(height); + return this.scanInteractive(height); } /** @@ -535,11 +552,17 @@ class WalletDB extends EventEmitter { */ async scan(height) { + assert(this.rescanning, 'WDB: Rescanning guard not set.'); + if (height == null) height = this.state.startHeight; assert((height >>> 0) === height, 'WDB: Must pass in a height.'); + this.logger.info( + 'Rolling back %d blocks.', + this.height - height + 1); + await this.rollback(height); this.logger.info( @@ -551,6 +574,38 @@ class WalletDB extends EventEmitter { return this.client.rescan(tip.hash); } + /** + * Interactive scan blockchain from a given height. + * Expect this.rescanning to be set to true. + * @private + * @param {Number} [height=this.state.startHeight] + * @param {Boolean} [fullLock=true] + * @returns {Promise} + */ + + async scanInteractive(height, fullLock = true) { + assert(this.rescanning, 'WDB: Rescanning guard not set.'); + + if (height == null) + height = this.state.startHeight; + + assert((height >>> 0) === height, 'WDB: Must pass in a height.'); + + this.logger.info( + 'Rolling back %d blocks.', + this.height - height + 1); + + await this.rollback(height); + + this.logger.info( + 'WalletDB is scanning %d blocks.', + this.state.height - height + 1); + + const tip = await this.getTip(); + + return this.client.rescanInteractive(tip.hash, fullLock); + } + /** * Deep Clean: * Keep all keys, account data, wallet maps (name and path). @@ -661,7 +716,7 @@ class WalletDB extends EventEmitter { this.rescanning = true; try { - return await this.scan(height); + return await this.scanInteractive(height); } finally { this.rescanning = false; } @@ -2133,7 +2188,7 @@ class WalletDB extends EventEmitter { */ async addOutpointMap(b, hash, index, wid) { - await this.addOutpoint(hash, index); + this.addOutpoint(hash, index); return this.addMap(b, layout.o.encode(hash, index), wid); } @@ -2432,7 +2487,7 @@ class WalletDB extends EventEmitter { * Unconfirm a block's transactions * and write the new best hash (SPV version). * @param {ChainEntry} entry - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async removeBlock(entry) { @@ -2448,7 +2503,7 @@ class WalletDB extends EventEmitter { * Unconfirm a block's transactions. * @private * @param {ChainEntry} entry - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async _removeBlock(entry) { @@ -2525,6 +2580,41 @@ class WalletDB extends EventEmitter { } } + /** + * Rescan a block interactively. + * @param {ChainEntry} entry + * @param {TX[]} txs + * @returns {Promise} - interactive action + */ + + async rescanBlockInteractive(entry, txs) { + if (!this.rescanning) + throw new Error(`WDB: Unsolicited rescan block: ${entry.height}.`); + + if (entry.height > this.state.height + 1) + throw new Error(`WDB: Rescan block too high: ${entry.height}.`); + + const blockAdded = await this._addBlock(entry, txs); + + if (!blockAdded) + throw new Error('WDB: Block not added.'); + + if (blockAdded.filterUpdated) { + // We remove block, because adding the same block twice, will ignore + // already indexed transactions. This handles the case where single + // transaction has undiscovered outputs. + await this._removeBlock(entry); + + return { + type: scanActions.REPEAT + }; + } + + return { + type: scanActions.NEXT + }; + } + /** * Add a transaction to the database, map addresses * to wallet IDs, potentially store orphans, resolve diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index 920e0ae37..b5b810b69 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -38,10 +38,9 @@ const combinations = [ ]; const noSPVcombinations = combinations.filter(c => !c.SPV); +const regtest = Network.get('regtest'); describe('Wallet rescan/addBlock', function() { - const network = Network.get('regtest'); - // TODO: Add SPV tests. for (const {SPV, STANDALONE, name} of noSPVcombinations) { describe(`rescan/addBlock gapped addresses (${name} Integration)`, function() { @@ -54,66 +53,16 @@ describe('Wallet rescan/addBlock', function() { const WALLET_NAME = 'test'; const ACCOUNT = 'default'; - const network = Network.get('regtest'); + const regtest = Network.get('regtest'); /** @type {NodesContext} */ let nodes; let minerWallet, minerAddress; let main, addBlock, rescan; - const deriveAddresses = async (wallet, depth) => { - const accInfo = await wallet.getAccount('default'); - let currentDepth = accInfo.receiveDepth; - - if (depth <= currentDepth) - return; - - while (currentDepth !== depth) { - const addr = await wallet.createAddress('default'); - currentDepth = addr.index; - } - }; - - const getAddress = async (wallet, depth = -1) => { - const accInfo = await wallet.getAccount('default'); - const {accountKey, lookahead} = accInfo; - - if (depth === -1) - depth = accInfo.receiveDepth; - - const XPUBKey = HDPublicKey.fromBase58(accountKey, network); - const key = XPUBKey.derive(0).derive(depth).publicKey; - const address = Address.fromPubkey(key); - - const gappedDepth = depth + lookahead + 1; - return {address, depth, gappedDepth}; - }; - - const generateGappedAddresses = async (wallet, count) => { - let depth = -1; - - const addresses = []; - - // generate gapped addresses. - for (let i = 0; i < count; i++) { - const addrInfo = await getAddress(wallet, depth); - - addresses.push({ - address: addrInfo.address, - depth: addrInfo.depth, - gappedDepth: addrInfo.gappedDepth - }); - - await deriveAddresses(wallet, depth); - depth = addrInfo.gappedDepth; - } - - return addresses; - }; - before(async () => { // Initial node is the one that progresses the network. - nodes = new NodesContext(network, 1); + nodes = new NodesContext(regtest, 1); // MAIN_WALLET = 0 nodes.init({ wallet: true, @@ -199,7 +148,8 @@ describe('Wallet rescan/addBlock', function() { // 1 address per block, all of them gapped. // Start after first gap, make sure rescan has no clue. - const all = await generateGappedAddresses(main.client, blocks + 1); + const all = await generateGappedAddresses(main.client, blocks + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); const addresses = all.slice(1); // give addBlock first address. await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); @@ -211,7 +161,7 @@ describe('Wallet rescan/addBlock', function() { for (let i = 0; i < blocks; i++) { await minerWallet.send({ outputs: [{ - address: addresses[i].address.toString(network), + address: addresses[i].address.toString(regtest), value: 1e6 }] }); @@ -259,7 +209,8 @@ describe('Wallet rescan/addBlock', function() { const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); const txCount = 5; - const all = await generateGappedAddresses(main.client, txCount + 1); + const all = await generateGappedAddresses(main.client, txCount + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); const addresses = all.slice(1); // give addBlock first address. @@ -272,7 +223,7 @@ describe('Wallet rescan/addBlock', function() { for (const {address} of addresses) { await minerWallet.send({ outputs: [{ - address: address.toString(network), + address: address.toString(regtest), value: 1e6 }] }); @@ -302,7 +253,7 @@ describe('Wallet rescan/addBlock', function() { assert.deepStrictEqual(addBlockInfo, mainInfo); }); - it.skip('should receive gapped txs in the same block (rescan)', async () => { + it('should receive gapped txs in the same block (rescan)', async () => { const expectedBalance = await getBalance(main.client, ACCOUNT); const expectedInfo = await main.client.getAccount(ACCOUNT); @@ -319,7 +270,8 @@ describe('Wallet rescan/addBlock', function() { const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); const outCount = 5; - const all = await generateGappedAddresses(main.client, outCount + 1); + const all = await generateGappedAddresses(main.client, outCount + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); const addresses = all.slice(1); // give addBlock first address. @@ -330,7 +282,7 @@ describe('Wallet rescan/addBlock', function() { const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect'); const outputs = addresses.map(({address}) => ({ - address: address.toString(network), + address: address.toString(regtest), value: 1e6 })); @@ -359,7 +311,7 @@ describe('Wallet rescan/addBlock', function() { assert.deepStrictEqual(addBlockInfo, mainInfo); }); - it.skip('should receive gapped outputs in the same tx (rescan)', async () => { + it('should receive gapped outputs in the same tx (rescan)', async () => { const expectedBalance = await getBalance(main.client, ACCOUNT); const expectedInfo = await main.client.getAccount(ACCOUNT); @@ -389,10 +341,10 @@ describe('Wallet rescan/addBlock', function() { let nodes; let wnodeCtx, noWnodeCtx; let minerWallet, minerAddress; - let testAddress; + let testWallet, testAddress; before(async () => { - nodes = new NodesContext(network, 1); + nodes = new NodesContext(regtest, 1); // MINER = 0 nodes.init({ @@ -432,7 +384,7 @@ describe('Wallet rescan/addBlock', function() { minerWallet = nodes.context(MINER).wclient.wallet('primary'); minerAddress = (await minerWallet.createAddress('default')).address; - const testWallet = wnodeCtx.wclient.wallet('primary'); + testWallet = wnodeCtx.wclient.wallet('primary'); testAddress = (await testWallet.createAddress('default')).address; await nodes.close(WALLET); @@ -501,29 +453,23 @@ describe('Wallet rescan/addBlock', function() { // Disable wallet await noWnodeCtx.close(); - // sync node. - let eventsToWait; - wnodeCtx.init(); + const eventsToWait = []; // For spv we don't wait for sync done, as it will do the full rescan // and reset the SPVNode as well. It does not depend on the accumulated // blocks. if (SPV) { - eventsToWait = [ - // This will happen right away, as scan will just call reset - forEvent(wnodeCtx.wdb, 'sync done'), - // This is what matters for the rescan. - forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { - return entry.height === nodes.height(MINER); - }), + // This will happen right away, as scan will just call reset + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + // This is what matters for the rescan. + eventsToWait.push(forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + })); // Make sure node gets resets. - forEvent(wnodeCtx.node, 'reset') - ]; + eventsToWait.push(forEvent(wnodeCtx.node, 'reset')); } else { - eventsToWait = [ - forEvent(wnodeCtx.wdb, 'sync done') - ]; + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); } await wnodeCtx.open(); @@ -584,24 +530,23 @@ describe('Wallet rescan/addBlock', function() { wnodeCtx.init(); // initial sync - let eventsToWait; + const eventsToWait = []; + if (SPV) { - eventsToWait = [ - // This will happen right away, as scan will just call reset - forEvent(wnodeCtx.wdb, 'sync done'), - // This is what matters for the rescan. - forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { - return entry.height === nodes.height(MINER); - }), - // Make sure node gets resets. - forEvent(wnodeCtx.node, 'reset'), - forEvent(wnodeCtx.wdb, 'unconfirmed') - ]; + // This will happen right away, as scan will just call reset + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + + // This is what matters for the rescan. + eventsToWait.push(forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + })); + + // Make sure node gets resets. + eventsToWait.push(forEvent(wnodeCtx.node, 'reset')); + eventsToWait.push(forEvent(wnodeCtx.wdb, 'unconfirmed')); } else { - eventsToWait = [ - forEvent(wnodeCtx.wdb, 'sync done'), - forEvent(wnodeCtx.wdb, 'unconfirmed') - ]; + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + eventsToWait.push(forEvent(wnodeCtx.wdb, 'unconfirmed')); } await wnodeCtx.open(); await Promise.all(eventsToWait); @@ -621,13 +566,111 @@ describe('Wallet rescan/addBlock', function() { await wnodeCtx.close(); }); + + it('should rescan/resync after wallet was off and received gapped txs in the same block', async () => { + if (SPV) + this.skip(); + + const txCount = 5; + await wnodeCtx.open(); + const startingBalance = await getBalance(testWallet, 'default'); + const all = await generateGappedAddresses(testWallet, txCount, regtest); + await wnodeCtx.close(); + + await noWnodeCtx.open(); + + for (const {address} of all) { + await minerWallet.send({ + outputs: [{ + address: address.toString(regtest), + value: 1e6 + }] + }); + } + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + + await nodeSync; + await noWnodeCtx.close(); + + wnodeCtx.init(); + + const syncDone = forEvent(wnodeCtx.wdb, 'sync done'); + await wnodeCtx.open(); + await syncDone; + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getBalance(testWallet, 'default'); + const diff = balance.diff(startingBalance); + assert.deepStrictEqual(diff, new Balance({ + tx: txCount, + coin: txCount, + confirmed: 1e6 * txCount, + unconfirmed: 1e6 * txCount + })); + + await wnodeCtx.close(); + }); + + it('should rescan/resync after wallet was off and received gapped coins in the same tx', async () => { + if (SPV) + this.skip(); + + const outCount = 5; + await wnodeCtx.open(); + const startingBalance = await getBalance(testWallet, 'default'); + const all = await generateGappedAddresses(testWallet, outCount, regtest); + await wnodeCtx.close(); + + await noWnodeCtx.open(); + + const outputs = all.map(({address}) => ({ + address: address.toString(regtest), + value: 1e6 + })); + + await minerWallet.send({outputs}); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + + await nodeSync; + await noWnodeCtx.close(); + + wnodeCtx.init(); + + const syncDone = forEvent(wnodeCtx.wdb, 'sync done'); + await wnodeCtx.open(); + await syncDone; + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getBalance(testWallet, 'default'); + const diff = balance.diff(startingBalance); + assert.deepStrictEqual(diff, new Balance({ + tx: 1, + coin: outCount, + confirmed: 1e6 * outCount, + unconfirmed: 1e6 * outCount + })); + + await wnodeCtx.close(); + }); }); } for (const {STANDALONE, name} of noSPVcombinations) { describe(`Deadlock (${name} Integration)`, function() { this.timeout(10000); - const nodes = new NodesContext(network, 1); + const nodes = new NodesContext(regtest, 1); let minerCtx; let nodeCtx, address, node, wdb; @@ -767,6 +810,10 @@ describe('Wallet rescan/addBlock', function() { // We start rescan only after first disconnect is detected to ensure // wallet guard is set. await forEvent(node.chain, 'disconnect'); + + // abort should also report reason as an error. + const errorEvents = forEvent(wdb, 'error', 1); + let err; try { // Because we are rescanning within the rescan blocks, @@ -780,8 +827,63 @@ describe('Wallet rescan/addBlock', function() { assert(err); assert.strictEqual(err.message, 'Cannot rescan an alternate chain.'); + const errors = await errorEvents; + assert.strictEqual(errors.length, 1); + const errEv = errors[0].values[0]; + assert(errEv); + assert.strictEqual(errEv.message, 'Cannot rescan an alternate chain.'); + await mineBlocks; }); }); } }); + +async function deriveAddresses(walletClient, depth) { + const accInfo = await walletClient.getAccount('default'); + let currentDepth = accInfo.receiveDepth; + + if (depth <= currentDepth) + return; + + while (currentDepth !== depth) { + const addr = await walletClient.createAddress('default'); + currentDepth = addr.index; + } +} + +async function getAddress(walletClient, depth = -1, network = regtest) { + const accInfo = await walletClient.getAccount('default'); + const {accountKey, lookahead} = accInfo; + + if (depth === -1) + depth = accInfo.receiveDepth; + + const XPUBKey = HDPublicKey.fromBase58(accountKey, network); + const key = XPUBKey.derive(0).derive(depth).publicKey; + const address = Address.fromPubkey(key); + + const gappedDepth = depth + lookahead + 1; + return {address, depth, gappedDepth}; +} + +async function generateGappedAddresses(walletClient, count, network = regtest) { + let depth = -1; + + const addresses = []; + + // generate gapped addresses. + for (let i = 0; i < count; i++) { + const addrInfo = await getAddress(walletClient, depth, network); + + addresses.push({ + address: addrInfo.address, + depth: addrInfo.depth, + gappedDepth: addrInfo.gappedDepth + }); + + depth = addrInfo.gappedDepth; + } + + return addresses; +} diff --git a/test/wallet-unit-test.js b/test/wallet-unit-test.js index d991085b8..1e6a55302 100644 --- a/test/wallet-unit-test.js +++ b/test/wallet-unit-test.js @@ -520,7 +520,7 @@ describe('Wallet Unit Tests', () => { let rescan = false; let rescanHash = null; - wdb.client.rescan = async (hash) => { + wdb.client.rescanInteractive = async (hash) => { rescan = true; rescanHash = hash; }; From f46192d3a6201f76cdcfa1f63c3b16e8ce3773e0 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 15 Feb 2024 21:50:38 +0400 Subject: [PATCH 13/13] test: Change nodes-context seed ports. Add stack traces to common test helper timeouts. Change default timeout to 2000. --- test/node-spv-sync-test.js | 2 +- test/util/common.js | 30 +++++++++++++++++++++++------- test/util/nodes-context.js | 22 +++++++++++++++------- test/wallet-rescan-test.js | 20 ++++++++++++-------- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/test/node-spv-sync-test.js b/test/node-spv-sync-test.js index 62134b2ec..7d411cbbc 100644 --- a/test/node-spv-sync-test.js +++ b/test/node-spv-sync-test.js @@ -158,7 +158,7 @@ describe('SPV Node Sync', function() { }); it('should send a tx from chain 1 to SPV node', async () => { - const balanceEvent = forEvent(spvwallet, 'balance'); + const balanceEvent = forEvent(spvwallet, 'balance', 1, 9000); await wallet.send({ outputs: [{ value: 1012345678, diff --git a/test/util/common.js b/test/util/common.js index c2a6e3f06..57b738558 100644 --- a/test/util/common.js +++ b/test/util/common.js @@ -103,13 +103,14 @@ common.rimraf = async function(p) { return await fs.rimraf(p); }; -common.forValue = async function forValue(obj, key, val, timeout = 5000) { +common.forValue = async function forValue(obj, key, val, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof key === 'string'); const ms = 10; let interval = null; let count = 0; + const stack = getStack(); return new Promise((resolve, reject) => { interval = setInterval(() => { @@ -118,14 +119,16 @@ common.forValue = async function forValue(obj, key, val, timeout = 5000) { resolve(); } else if (count * ms >= timeout) { clearInterval(interval); - reject(new Error('Timeout waiting for value.')); + const error = new Error('Timeout waiting for value.'); + error.stack = error.stack + '\n' + stack; + reject(error); } count += 1; }, ms); }); }; -common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) { +common.forEvent = async function forEvent(obj, name, count = 1, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof name === 'string'); assert(typeof count === 'number'); @@ -134,6 +137,8 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) let countdown = count; const events = []; + const stack = getStack(); + return new Promise((resolve, reject) => { let timeoutHandler, listener; @@ -159,9 +164,11 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) timeoutHandler = setTimeout(() => { cleanup(); const msg = `Timeout waiting for event ${name} ` - + `(received ${count - countdown}/${count})`; + + `(received ${count - countdown}/${count})\n${stack}`; - reject(new Error(msg)); + const error = new Error(msg); + error.stack = error.stack + '\n' + stack; + reject(error); return; }, timeout); @@ -169,12 +176,14 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) }); }; -common.forEventCondition = async function forEventCondition(obj, name, fn, timeout = 5000) { +common.forEventCondition = async function forEventCondition(obj, name, fn, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof name === 'string'); assert(typeof fn === 'function'); assert(typeof timeout === 'number'); + const stack = getStack(); + return new Promise((resolve, reject) => { let timeoutHandler, listener; @@ -190,6 +199,7 @@ common.forEventCondition = async function forEventCondition(obj, name, fn, timeo res = await fn(...args); } catch (e) { cleanup(); + e.stack = e.stack + '\n' + stack; reject(e); return; } @@ -203,7 +213,9 @@ common.forEventCondition = async function forEventCondition(obj, name, fn, timeo timeoutHandler = setTimeout(() => { cleanup(); const msg = `Timeout waiting for event ${name} with condition`; - reject(new Error(msg)); + const error = new Error(msg); + error.stack = error.stack + '\n' + stack; + reject(error); return; }, timeout); @@ -357,3 +369,7 @@ class TXContext { return [tx, view]; } } + +function getStack() { + return new Error().stack.split('\n').slice(2).join('\n'); +} diff --git a/test/util/nodes-context.js b/test/util/nodes-context.js index 221b88208..2989464df 100644 --- a/test/util/nodes-context.js +++ b/test/util/nodes-context.js @@ -19,12 +19,12 @@ class NodesContext { } addNode(options = {}) { - const index = this.nodeCtxs.length + 1; + const index = this.nodeCtxs.length; - let seedPort = this.network.port + index - 1; + let seedPort = getPort(this.network, index - 1); - if (seedPort < this.network.port) - seedPort = this.network.port; + if (options.seedNodeIndex != null) + seedPort = getPort(this.network, options.seedNodeIndex); const port = this.network.port + index; const brontidePort = this.network.brontidePort + index; @@ -33,6 +33,11 @@ class NodesContext { const nsPort = this.network.nsPort + index; const rsPort = this.network.rsPort + index + 100; + const seeds = []; + + if (options.seedNodeIndex != null || index > 0) + seeds.push(`127.0.0.1:${seedPort}`); + const nodeCtx = new NodeContext({ listen: true, @@ -47,9 +52,8 @@ class NodesContext { nsPort: nsPort, httpPort: httpPort, walletHttpPort: walletHttpPort, - seeds: [ - `127.0.0.1:${seedPort}` - ] + + seeds: seeds }); this.nodeCtxs.push(nodeCtx); @@ -191,4 +195,8 @@ class NodesContext { } } +function getPort(network, index) { + return Math.max(network.port + index, network.port); +} + module.exports = NodesContext; diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index b5b810b69..0048126e7 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -41,9 +41,9 @@ const noSPVcombinations = combinations.filter(c => !c.SPV); const regtest = Network.get('regtest'); describe('Wallet rescan/addBlock', function() { - // TODO: Add SPV tests. for (const {SPV, STANDALONE, name} of noSPVcombinations) { describe(`rescan/addBlock gapped addresses (${name} Integration)`, function() { + this.timeout(5000); const TEST_LOOKAHEAD = 20; const MAIN = 0; @@ -144,6 +144,7 @@ describe('Wallet rescan/addBlock', function() { // Prepare for the rescan and addBlock tests. it('should send gapped txs on each block', async () => { const expectedRescanBalance = await getBalance(main.client, ACCOUNT); + const height = nodes.height(MAIN); const blocks = 5; // 1 address per block, all of them gapped. @@ -154,9 +155,10 @@ describe('Wallet rescan/addBlock', function() { // give addBlock first address. await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); - const mainWalletBlocks = forEvent(main.wdb, 'block connect', blocks); - const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect', blocks); - const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect', blocks); + const condFn = entry => entry.height === blocks + height; + const mainWalletBlocks = forEventCondition(main.wdb, 'block connect', condFn); + const addBlockWalletBlocks = forEventCondition(addBlock.wdb, 'block connect', condFn); + const rescanWalletBlocks = forEventCondition(rescan.wdb, 'block connect', condFn); for (let i = 0; i < blocks; i++) { await minerWallet.send({ @@ -704,15 +706,17 @@ describe('Wallet rescan/addBlock', function() { const BLOCKS = 20; const chainBlocks = forEventCondition(node.chain, 'connect', (entry) => { return entry.height === BLOCKS; - }); + }, 5000); const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { return entry.height === BLOCKS; - }); + }, 5000); await minerCtx.mineBlocks(BLOCKS, address); - await chainBlocks; - await wdbBlocks; + await Promise.all([ + chainBlocks, + wdbBlocks + ]); }); it('should rescan when receiving a block', async () => {