Add CheckConflictTopology to check conflicting tx topology

We only want to support package RBF for limited cases we can
reason about more precisely.

In this case, we check if each direct conflict is in a mempool
cluster of size 2. Note that two direct conflicts could be in the
same mempool cluster.

src/policy/rbf.cpp

@@ -182,6 +182,38 @@ std::optional<std::string> PaysForRBF(CAmount original_fees,
     return std::nullopt;
 }
 
+std::optional<std::string> CheckConflictTopology(const CTxMemPool::setEntries& direct_conflicts)
+{
+    for (const auto& direct_conflict : direct_conflicts) {
+        const auto desc_count = direct_conflict->GetCountWithDescendants() - 1;
+        if (desc_count == 0) {
+            // It should be by itself, or be a child in a parent-child cluster
+            if (direct_conflict->GetCountWithAncestors() > 2) {
+                return strprintf("Child transaction has too many ancestors");
+            }
+            if (direct_conflict->GetCountWithDescendants() == 2) {
+                const auto& parent = direct_conflict->GetMemPoolParentsConst().begin();
+                if (parent->get().GetCountWithDescendants() > 1) {
+                    return strprintf("Parent in package rbf should not have multiple descendants");
+                }
+            }
+        } else if (desc_count == 1) {
+            // It should be the parent in a parent-child cluster
+            const auto& parent = direct_conflict;
+            if (parent->GetCountWithAncestors() > 1) {
+                return strprintf("Parent in package rbf should not have any ancestors");
+            }
+            const auto& child = parent->GetMemPoolChildrenConst().begin();
+            if (child->get().GetCountWithAncestors() != 2) {
+                return strprintf("Child in package RBF has too many ancestors");
+            }
+        } else {
+            return strprintf("Too many descendants of direct conflict: %zu", desc_count);
+        }
+    }
+    return std::nullopt;
+}
+
 std::optional<std::string> CheckMinerScores(CAmount replacement_fees,
                                             int64_t replacement_vsize,
                                             const CTxMemPool::setEntries& direct_conflicts,

src/policy/rbf.h

@@ -106,6 +106,11 @@ std::optional<std::string> PaysForRBF(CAmount original_fees,
                                       CFeeRate relay_fee,
                                       const uint256& txid);
 
+/** Ensure we are only attempting RBF against connected components we can easily
+ * compute mining scores from.
+ */
+std::optional<std::string> CheckConflictTopology(const CTxMemPool::setEntries& direct_conflicts);
+
 /** Attempt to ensure replacement transactions are more incentive compatible to mine than the
  * transactions they are replacing. Currently, this function requires that feerate
  * of the replacement transaction(s) be higher than the individual feerates of