From 2792d05304ed49e73cd9cb78c503f368de3ea934 Mon Sep 17 00:00:00 2001 From: Kenny Joseph Date: Wed, 1 Nov 2023 10:41:04 -0400 Subject: [PATCH 1/4] better handle descendant limit --- packages/bitcore-node/src/services/pruning.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/bitcore-node/src/services/pruning.ts b/packages/bitcore-node/src/services/pruning.ts index 3c130279f5a..facc43209f5 100644 --- a/packages/bitcore-node/src/services/pruning.ts +++ b/packages/bitcore-node/src/services/pruning.ts @@ -8,7 +8,7 @@ import parseArgv from '../utils/parseArgv'; import '../utils/polyfills'; import { Config } from './config'; -const { PRUNING_CHAIN, PRUNING_NETWORK, PRUNING_MEMPOOL_AGE, PRUNING_INTERVAL_HRS } = process.env; +const { PRUNING_CHAIN, PRUNING_NETWORK, PRUNING_MEMPOOL_AGE, PRUNING_INTERVAL_HRS, PRUNING_DESCENDANT_LIMIT } = process.env; const args = parseArgv([], [ { arg: 'CHAIN', type: 'string' }, { arg: 'NETWORK', type: 'string' }, @@ -17,7 +17,8 @@ const args = parseArgv([], [ { arg: 'EXIT', type: 'bool' }, { arg: 'DRY', type: 'bool' }, { arg: 'MEMPOOL_AGE', type: 'int' }, - { arg: 'INTERVAL_HRS', type: 'float' } + { arg: 'INTERVAL_HRS', type: 'float' }, + { arg: 'DESCENDANT_LIMIT', type: 'int' } ]); const ONE_MIN = 1000 * 60; @@ -28,6 +29,7 @@ const CHAIN = args.CHAIN || PRUNING_CHAIN; const NETWORK = args.NETWORK || PRUNING_NETWORK; const INTERVAL_HRS = args.INTERVAL_HRS || Number(PRUNING_INTERVAL_HRS) || 12; const MEMPOOL_AGE = args.MEMPOOL_AGE || Number(PRUNING_MEMPOOL_AGE) || 7; +const DESCENDANT_LIMIT = args.DESCENDANT_LIMIT || Number(PRUNING_DESCENDANT_LIMIT) || 50; // If --DRY was given w/o a follow arg (i.e. 'true', '0', etc) assume the user wants to run a dry run (safe) if (Object.keys(args).includes('DRY') && args.DRY === undefined) { @@ -126,8 +128,9 @@ export class PruningService { return cb(new Error(`Invalid coin! ${coin.mintTxid} `)); } count++; - if (count > 50) { - throw new Error(`${tx.txid} has too many decendents`); + if (count > DESCENDANT_LIMIT) { + logger.warn(`${tx.txid} has too many decendents`); + return cb(); } if (coin.spentTxid) { spentTxids.add(coin.spentTxid); @@ -137,7 +140,7 @@ export class PruningService { const uniqueTxids = Array.from(spentTxids); await this.removeOldMempool(chain, network, uniqueTxids); logger.info(`Removed ${tx.txid} transaction and ${count} dependent txs`); - cb(); + return cb(); } }) ) From 859b135e5368eca421822ab5231ba6506872b7dc Mon Sep 17 00:00:00 2001 From: Kenny Joseph Date: Wed, 1 Nov 2023 11:04:46 -0400 Subject: [PATCH 2/4] descendants != coins --- packages/bitcore-node/src/services/pruning.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/bitcore-node/src/services/pruning.ts b/packages/bitcore-node/src/services/pruning.ts index facc43209f5..21aacd621bc 100644 --- a/packages/bitcore-node/src/services/pruning.ts +++ b/packages/bitcore-node/src/services/pruning.ts @@ -29,7 +29,7 @@ const CHAIN = args.CHAIN || PRUNING_CHAIN; const NETWORK = args.NETWORK || PRUNING_NETWORK; const INTERVAL_HRS = args.INTERVAL_HRS || Number(PRUNING_INTERVAL_HRS) || 12; const MEMPOOL_AGE = args.MEMPOOL_AGE || Number(PRUNING_MEMPOOL_AGE) || 7; -const DESCENDANT_LIMIT = args.DESCENDANT_LIMIT || Number(PRUNING_DESCENDANT_LIMIT) || 50; +const DESCENDANT_LIMIT = args.DESCENDANT_LIMIT || Number(PRUNING_DESCENDANT_LIMIT) || 10; // If --DRY was given w/o a follow arg (i.e. 'true', '0', etc) assume the user wants to run a dry run (safe) if (Object.keys(args).includes('DRY') && args.DRY === undefined) { @@ -127,13 +127,12 @@ export class PruningService { if (coin.mintHeight >= 0 || coin.spentHeight >= 0) { return cb(new Error(`Invalid coin! ${coin.mintTxid} `)); } - count++; - if (count > DESCENDANT_LIMIT) { - logger.warn(`${tx.txid} has too many decendents`); - return cb(); - } if (coin.spentTxid) { spentTxids.add(coin.spentTxid); + if (spentTxids.size > DESCENDANT_LIMIT) { + logger.warn(`${tx.txid} has too many decendants`); + return cb(); + } } } spentTxids.add(tx.txid); From 5fdd345b7ef2ba8df48f65a9186ddbebe693ef0b Mon Sep 17 00:00:00 2001 From: Kenny Joseph Date: Wed, 1 Nov 2023 11:20:39 -0400 Subject: [PATCH 3/4] remove count --- packages/bitcore-node/src/services/pruning.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/bitcore-node/src/services/pruning.ts b/packages/bitcore-node/src/services/pruning.ts index 21aacd621bc..903678c6d5d 100644 --- a/packages/bitcore-node/src/services/pruning.ts +++ b/packages/bitcore-node/src/services/pruning.ts @@ -122,7 +122,6 @@ export class PruningService { logger.info(`Finding ${tx.txid} outputs and dependent outputs`); const outputGenerator = this.transactionModel.yieldRelatedOutputs(tx.txid); let spentTxids = new Set(); - let count = 0; for await (const coin of outputGenerator) { if (coin.mintHeight >= 0 || coin.spentHeight >= 0) { return cb(new Error(`Invalid coin! ${coin.mintTxid} `)); @@ -138,7 +137,7 @@ export class PruningService { spentTxids.add(tx.txid); const uniqueTxids = Array.from(spentTxids); await this.removeOldMempool(chain, network, uniqueTxids); - logger.info(`Removed ${tx.txid} transaction and ${count} dependent txs`); + logger.info(`Removed ${tx.txid} transaction and ${spentTxids.size - 1} dependent txs`); return cb(); } }) From 38d9f3c49d218eb21a6cf7cd2adb5ea659ee0ca6 Mon Sep 17 00:00:00 2001 From: Kenny Joseph Date: Wed, 1 Nov 2023 11:52:01 -0400 Subject: [PATCH 4/4] apply logic to invalid tx; log total count at the end --- packages/bitcore-node/src/services/pruning.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/bitcore-node/src/services/pruning.ts b/packages/bitcore-node/src/services/pruning.ts index 903678c6d5d..e90c858c529 100644 --- a/packages/bitcore-node/src/services/pruning.ts +++ b/packages/bitcore-node/src/services/pruning.ts @@ -108,6 +108,7 @@ export class PruningService { blockTimeNormalized: { $lt: oldTime } }); logger.info(`Found ${count} outdated ${chain} ${network} mempool txs`); + let rmCount = 0; await new Promise((resolve, reject) => { this.transactionModel.collection .find({ chain, network, blockHeight: -1, blockTimeNormalized: { $lt: oldTime } }) @@ -135,9 +136,10 @@ export class PruningService { } } spentTxids.add(tx.txid); + rmCount += spentTxids.size; const uniqueTxids = Array.from(spentTxids); await this.removeOldMempool(chain, network, uniqueTxids); - logger.info(`Removed ${tx.txid} transaction and ${spentTxids.size - 1} dependent txs`); + logger.info(`Removed tx ${tx.txid} and ${spentTxids.size - 1} dependent txs`); return cb(); } }) @@ -145,7 +147,7 @@ export class PruningService { .on('finish', resolve) .on('error', reject); }); - logger.info(`Removed all old mempool txs within the last ${days} days`); + logger.info(`Removed all pending txs older than ${days} days: ${rmCount}`); } async processAllInvalidTxs(chain, network) { @@ -169,23 +171,22 @@ export class PruningService { logger.info(`Invalidating ${tx.txid} outputs and dependent outputs`); const outputGenerator = this.transactionModel.yieldRelatedOutputs(tx.txid); let spentTxids = new Set(); - let count = 0; for await (const coin of outputGenerator) { if (coin.mintHeight >= 0 || coin.spentHeight >= 0) { return cb(new Error(`Invalid coin! ${coin.mintTxid} `)); } - count++; - if (count > 50) { - throw new Error(`${tx.txid} has too many decendents`); - } if (coin.spentTxid) { spentTxids.add(coin.spentTxid); + if (spentTxids.size > DESCENDANT_LIMIT) { + logger.warn(`${tx.txid} has too many decendants`); + return cb(); + } } } spentTxids.add(tx.txid); const uniqueTxids = Array.from(spentTxids); await this.clearInvalid(uniqueTxids); - logger.info(`Invalidated ${tx.txid} and ${count} dependent txs`); + logger.info(`Invalidated tx ${tx.txid} and ${spentTxids.size - 1} dependent txs`); cb(); } })