fix: findDiffDepthi to support more than 31 bytes (#371)

* fix: findDiffDepthi to support more than 31 bytes

* chore: simplify get num bits

* chore: more comments

* fix: use NUMBER_2_POW_32 to get high bits

packages/persistent-merkle-tree/src/tree.ts

@@ -692,6 +692,52 @@ export function treeZeroAfterIndex(rootNode: Node, nodesDepth: number, index: nu
   return node;
 }
 
+const NUMBER_32_MAX = 0xffffffff;
+const NUMBER_2_POW_32 = 2 ** 32;
+
+/**
+ * depth depthi   gindexes   indexes
+ * 0     1           1          0
+ * 1     0         2   3      0   1
+ * 2     -        4 5 6 7    0 1 2 3
+ *
+ * **Conditions**:
+ * - `from` and `to` must not be equal
+ *
+ * @param from Index
+ * @param to Index
+ */
+export function findDiffDepthi(from: number, to: number): number {
+  if (from === to || from < 0 || to < 0) {
+    throw Error(`Expect different positive inputs, from=${from} to=${to}`);
+  }
+  // 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 2, 4 -> 3
+  const numBits0 = Math.ceil(Math.log2(from + 1));
+  const numBits1 = Math.ceil(Math.log2(to + 1));
+
+  // these indexes stay in 2 sides of a merkle tree
+  if (numBits0 !== numBits1) {
+    // must offset by one to match the depthi scale
+    return Math.max(numBits0, numBits1) - 1;
+  }
+
+  // same number of bits and > 32
+  if (numBits0 > 32) {
+    const highBits0 = Math.floor(from / NUMBER_2_POW_32) & NUMBER_32_MAX;
+    const highBits1 = Math.floor(to / NUMBER_2_POW_32) & NUMBER_32_MAX;
+    if (highBits0 === highBits1) {
+      // different part is just low bits
+      return findDiffDepthi32Bits(from & NUMBER_32_MAX, to & NUMBER_32_MAX);
+    }
+
+    // highBits are different, no need to compare low bits
+    return 32 + findDiffDepthi32Bits(highBits0, highBits1);
+  }
+
+  // same number of bits and <= 32
+  return findDiffDepthi32Bits(from, to);
+}
+
 /**
  * Returns true if the `index` at `depth` is a left node, false if it is a right node.
  *
@@ -713,22 +759,19 @@ function isLeftNode(depthi: number, index: number): boolean {
 }
 
 /**
- * depth depthi   gindexes   indexes
- * 0     1           1          0
- * 1     0         2   3      0   1
- * 2     -        4 5 6 7    0 1 2 3
- *
- * **Conditions**:
- * - `from` and `to` must not be equal
- *
- * @param from Index
- * @param to Index
+ * Similar to findDiffDepthi() but for 32-bit numbers only
  */
-function findDiffDepthi(from: number, to: number): number {
-  return (
-    // (0,0) -> 0 | (0,1) -> 1 | (0,2) -> 2
-    Math.ceil(Math.log2(-~(from ^ to))) -
-    // Must offset by one to match the depthi scale
-    1
-  );
+function findDiffDepthi32Bits(from: number, to: number): number {
+  const xor = from ^ to;
+  if (xor === 0) {
+    // this should not happen as checked in `findDiffDepthi`
+    // otherwise this function return -1 which is weird for diffi
+    throw Error(`Do not support equal value from=${from} to=${to}`);
+  }
+
+  // (0,0) -> 0 | (0,1) -> 1 | (0,2) -> 2
+  // xor < 0 means the 1st bit of `from` and `to` is diffent, which mean num bits diff 32
+  const numBitsDiff = xor < 0 ? 32 : Math.ceil(Math.log2(xor + 1));
+  // must offset by one to match the depthi scale
+  return numBitsDiff - 1;
 }

packages/persistent-merkle-tree/test/unit/tree.test.ts

@@ -9,6 +9,7 @@ import {
   subtreeFillToContents,
   uint8ArrayToHashObject,
   setNodesAtDepth,
+  findDiffDepthi,
 } from "../../src";
 
 describe("fixed-depth tree iteration", () => {
@@ -186,6 +187,54 @@ describe("Tree batch setNodes", () => {
   }
 });
 
+/**
+ * These tests passed/fixed the old version of findDiffDepthi
+ * To validate it manually, use `Number(index).toString(2)` and compare bits
+ */
+describe("findDiffDepthi", () => {
+  it("validate inputs", () => {
+    expect(() => findDiffDepthi(-1, 100)).to.throw("Expect different positive inputs, from=-1 to=100");
+    expect(() => findDiffDepthi(101, 101)).to.throw("Expect different positive inputs, from=101 to=101");
+  });
+
+  const testCases: {index0: number; index1: number; result: number}[] = [
+    {index0: 0, index1: 1, result: 0},
+    // 2 sides of a 4-width tree
+    {index0: 1, index1: 3, result: 1},
+    // 2 sides of a 8-width tree
+    {index0: 3, index1: 4, result: 2},
+    // 16 bits
+    {index0: 0, index1: 0xffff, result: 15},
+    // 31 bits, different number of bits
+    {index0: 5, index1: (0xffffffff >>> 1) - 5, result: 30},
+    // 31 bits, same number of bits
+    {index0: 0x7fffffff, index1: 0x70000000, result: 27},
+    // 32 bits tree, different number of bits
+    {index0: 0, index1: 0xffffffff, result: 31},
+    {index0: 0, index1: (0xffffffff >>> 1) + 1, result: 31},
+    {index0: 0xffffffff >>> 1, index1: (0xffffffff >>> 1) + 1, result: 31},
+    // 32 bits tree, same number of bits
+    {index0: 0xf0000000, index1: 0xffffffff, result: 27},
+    // below tests are same to first tests but go from right to left
+    // similar to {0, 1}
+    {index0: 0xffffffff - 1, index1: 0xffffffff, result: 0},
+    // similar to {1, 3}
+    {index0: 0xffffffff - 3, index1: 0xffffffff - 1, result: 1},
+    // similar to {3, 4}
+    {index0: 0xffffffff - 4, index1: 0xffffffff - 3, result: 2},
+    // more than 32 bits, same number of bits
+    {index0: 1153210973487, index1: 1344787435182, result: 37},
+    // more than 32 bits, different number of bits
+    {index0: 1153210973487, index1: 1344787435182 >>> 2, result: 40},
+  ];
+
+  for (const {index0, index1, result} of testCases) {
+    it(`expect diffi between ${index0} and ${index1} to be ${result}`, () => {
+      expect(findDiffDepthi(index0, index1)).to.be.equal(result);
+    });
+  }
+});
+
 function getTreeRoots(tree: Tree, maxGindex: number): string[] {
   const roots = new Array<string>(maxGindex);
   for (let i = 1; i < maxGindex; i++) {