Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat!: implement hashInto() api for as-sha256 #382

Merged
merged 3 commits into from
Jul 16, 2024
Merged

feat!: implement hashInto() api for as-sha256 #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c1b9b20
feat: implement hashInto() api
twoeths Jul 14, 2024
73b4566
fix: uint8ArrayToHashObject compilation error
twoeths Jul 14, 2024
c7dc5c2
fix: tree.test.ts compilation error
twoeths Jul 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 60 additions & 52 deletions packages/as-sha256/src/hashObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,95 +98,103 @@ export function hashObjectToByteArray(obj: HashObject, byteArr: Uint8Array, offs
* This function contains multiple same procedures but we intentionally
* do it step by step to improve performance a bit.
**/
export function byteArrayToHashObject(byteArr: Uint8Array): HashObject {
export function byteArrayToHashObject(byteArr: Uint8Array, offset: number): HashObject {
const result: HashObject = {
h0: 0,
h1: 0,
h2: 0,
h3: 0,
h4: 0,
h5: 0,
h6: 0,
h7: 0,
};

byteArrayIntoHashObject(byteArr, offset, result);
return result;
}

/**
* Same to above but this set result to the output param to save memory.
*/
export function byteArrayIntoHashObject(byteArr: Uint8Array, offset: number, output: HashObject): void {
let tmp = 0;
tmp |= byteArr[3] & 0xff;
tmp |= byteArr[3 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[2] & 0xff;
tmp |= byteArr[2 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[1] & 0xff;
tmp |= byteArr[1 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[0] & 0xff;
const h0 = tmp;
tmp |= byteArr[0 + offset] & 0xff;
output.h0 = tmp;

tmp = 0;
tmp |= byteArr[7] & 0xff;
tmp |= byteArr[7 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[6] & 0xff;
tmp |= byteArr[6 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[5] & 0xff;
tmp |= byteArr[5 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[4] & 0xff;
const h1 = tmp;
tmp |= byteArr[4 + offset] & 0xff;
output.h1 = tmp;

tmp = 0;
tmp |= byteArr[11] & 0xff;
tmp |= byteArr[11 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[10] & 0xff;
tmp |= byteArr[10 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[9] & 0xff;
tmp |= byteArr[9 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[8] & 0xff;
const h2 = tmp;
tmp |= byteArr[8 + offset] & 0xff;
output.h2 = tmp;

tmp = 0;
tmp |= byteArr[15] & 0xff;
tmp |= byteArr[15 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[14] & 0xff;
tmp |= byteArr[14 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[13] & 0xff;
tmp |= byteArr[13 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[12] & 0xff;
const h3 = tmp;
tmp |= byteArr[12 + offset] & 0xff;
output.h3 = tmp;

tmp = 0;
tmp |= byteArr[19] & 0xff;
tmp |= byteArr[19 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[18] & 0xff;
tmp |= byteArr[18 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[17] & 0xff;
tmp |= byteArr[17 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[16] & 0xff;
const h4 = tmp;
tmp |= byteArr[16 + offset] & 0xff;
output.h4 = tmp;

tmp = 0;
tmp |= byteArr[23] & 0xff;
tmp |= byteArr[23 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[22] & 0xff;
tmp |= byteArr[22 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[21] & 0xff;
tmp |= byteArr[21 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[20] & 0xff;
const h5 = tmp;
tmp |= byteArr[20 + offset] & 0xff;
output.h5 = tmp;

tmp = 0;
tmp |= byteArr[27] & 0xff;
tmp |= byteArr[27 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[26] & 0xff;
tmp |= byteArr[26 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[25] & 0xff;
tmp |= byteArr[25 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[24] & 0xff;
const h6 = tmp;
tmp |= byteArr[24 + offset] & 0xff;
output.h6 = tmp;

tmp = 0;
tmp |= byteArr[31] & 0xff;
tmp |= byteArr[31 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[30] & 0xff;
tmp |= byteArr[30 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[29] & 0xff;
tmp |= byteArr[29 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[28] & 0xff;
const h7 = tmp;

return {
h0,
h1,
h2,
h3,
h4,
h5,
h6,
h7,
};
tmp |= byteArr[28 + offset] & 0xff;
output.h7 = tmp;
}
62 changes: 55 additions & 7 deletions packages/as-sha256/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {newInstance} from "./wasm";
import {HashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject";
import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject";
import SHA256 from "./sha256";
export {HashObject, byteArrayToHashObject, hashObjectToByteArray, SHA256};
export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256};

const ctx = newInstance();
const wasmInputValue = ctx.input.value;
Expand Down Expand Up @@ -52,6 +52,24 @@ export function digest2Bytes32(bytes1: Uint8Array, bytes2: Uint8Array): Uint8Arr
* @returns
*/
export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObject {
const result: HashObject = {
h0: 0,
h1: 0,
h2: 0,
h3: 0,
h4: 0,
h5: 0,
h6: 0,
h7: 0,
};
digest64HashObjectsInto(obj1, obj2, result);
return result;
}

/**
* Same to above but this set result to the output param to save memory.
*/
export function digest64HashObjectsInto(obj1: HashObject, obj2: HashObject, output: HashObject): void {
// TODO: expect obj1 and obj2 as HashObject
inputUint32Array[0] = obj1.h0;
inputUint32Array[1] = obj1.h1;
Expand All @@ -73,7 +91,7 @@ export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObj
ctx.digest64(wasmInputValue, wasmOutputValue);

// extracting numbers from Uint32Array causes more memory
return byteArrayToHashObject(outputUint8Array);
byteArrayIntoHashObject(outputUint8Array, 0, output);
}

/**
Expand Down Expand Up @@ -121,6 +139,7 @@ export function batchHash4UintArray64s(inputs: Uint8Array[]): Uint8Array[] {
* Inputs i0 i1 i2 i3 i4 i5 i6 i7
* \ / \ / \ / \ /
* Outputs o0 o1 o2 o3
* // TODO - batch: support equivalent method to hash into
*/
export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] {
if (inputs.length !== 8) {
Expand Down Expand Up @@ -227,14 +246,43 @@ export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] {

ctx.batchHash4HashObjectInputs(wasmOutputValue);

const output0 = byteArrayToHashObject(outputUint8Array.subarray(0, 32));
const output1 = byteArrayToHashObject(outputUint8Array.subarray(32, 64));
const output2 = byteArrayToHashObject(outputUint8Array.subarray(64, 96));
const output3 = byteArrayToHashObject(outputUint8Array.subarray(96, 128));
const output0 = byteArrayToHashObject(outputUint8Array, 0);
const output1 = byteArrayToHashObject(outputUint8Array, 32);
const output2 = byteArrayToHashObject(outputUint8Array, 64);
const output3 = byteArrayToHashObject(outputUint8Array, 96);

return [output0, output1, output2, output3];
}

/**
* Hash an input into preallocated input using batch if possible.
*/
export function hashInto(input: Uint8Array, output: Uint8Array): void {
if (input.length % 64 !== 0) {
throw new Error(`Invalid input length ${input.length}`);
}
if (input.length !== output.length * 2) {
throw new Error(`Invalid output length ${output.length}`);
}
// for every 64 x 4 = 256 bytes, do the batch hash
const endBatch = Math.floor(input.length / 256);
for (let i = 0; i < endBatch; i++) {
inputUint8Array.set(input.subarray(i * 256, (i + 1) * 256), 0);
ctx.batchHash4UintArray64s(wasmOutputValue);
output.set(outputUint8Array.subarray(0, 128), i * 128);
}

const numHashed = endBatch * 4;
const remainingHash = Math.floor((input.length % 256) / 64);
const inputOffset = numHashed * 64;
const outputOffset = numHashed * 32;
for (let i = 0; i < remainingHash; i++) {
inputUint8Array.set(input.subarray(inputOffset + i * 64, inputOffset + (i + 1) * 64), 0);
ctx.digest64(wasmInputValue, wasmOutputValue);
output.set(outputUint8Array.subarray(0, 32), outputOffset + i * 32);
}
}

function update(data: Uint8Array): void {
const INPUT_LENGTH = ctx.INPUT_LENGTH;
if (data.length > INPUT_LENGTH) {
Expand Down
8 changes: 4 additions & 4 deletions packages/as-sha256/test/perf/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ describe("digestTwoHashObjects vs digest64 vs digest", () => {
const input2 = "gajindergajindergajindergajinder";
const buffer1 = Buffer.from(input1, "utf-8");
const buffer2 = Buffer.from(input2, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj2 = sha256.byteArrayToHashObject(buffer2);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);
const obj2 = sha256.byteArrayToHashObject(buffer2, 0);
// total number of time running hash for 200000 balances
const iterations = 50023;
itBench(`digestTwoHashObjects ${iterations} times`, () => {
Expand Down Expand Up @@ -71,7 +71,7 @@ describe("hash - compare to java", () => {
describe("utils", () => {
const input1 = "gajindergajindergajindergajinder";
const buffer1 = Buffer.from(input1, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);

// total number of time running hash for 200000 balances
const iterations = 50023;
Expand All @@ -82,6 +82,6 @@ describe("utils", () => {
});

itBench(`byteArrayToHashObject ${iterations} times`, () => {
for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1);
for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1, 0);
});
});
17 changes: 11 additions & 6 deletions packages/as-sha256/test/perf/simd.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import {byteArrayToHashObject} from "../../src/hashObject";
/**
* This really depends on cpu, on a test ubuntu node, batch*() is 2x faster than digest64
* On Mac M1 May 2025:
* digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs
✓ digest64 200092 times 6.850496 ops/s 145.9748 ms/op - 66 runs 10.2 s
✓ hash 200092 times using batchHash4UintArray64s 8.454788 ops/s 118.2762 ms/op - 82 runs 10.2 s
✓ hash 200092 times using batchHash4HashObjectInputs 8.454464 ops/s 118.2807 ms/op - 82 runs 10.2 s
* digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs
✓ digest64 200092 times 6.648102 ops/s 150.4189 ms/op - 64 runs 10.2 s
✓ hash 200092 times using batchHash4UintArray64s 9.120131 ops/s 109.6476 ms/op - 88 runs 10.2 s
✓ digest64HashObjects 200092 times 7.095494 ops/s 140.9345 ms/op - 68 runs 10.2 s
✓ hash 200092 times using batchHash4HashObjectInputs 9.211751 ops/s 108.5570 ms/op - 88 runs 10.1 s
*/
describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", function () {
describe("digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs", function () {
this.timeout(0);

setBenchOpts({
Expand All @@ -31,7 +32,11 @@ describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", fun
}
});

const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8"));
const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8"), 0);
itBench(`digest64HashObjects ${iterations * 4} times`, () => {
for (let j = 0; j < iterations * 4; j++) sha256.digest64HashObjects(hashObject, hashObject);
});

const hashInputs = Array.from({length: 8}, () => hashObject);
// batchHash4HashObjectInputs do 4 sha256 in parallel
itBench(`hash ${iterations * 4} times using batchHash4HashObjectInputs`, () => {
Expand Down
6 changes: 3 additions & 3 deletions packages/as-sha256/test/unit/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ describe("sha256", function () {
// digestTwoHashObjects should be the same to digest64
const buffer1 = Buffer.from(input1, "utf-8");
const buffer2 = Buffer.from(input2, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj2 = sha256.byteArrayToHashObject(buffer2);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);
const obj2 = sha256.byteArrayToHashObject(buffer2, 0);
const obj = sha256.digest64HashObjects(obj1, obj2);
const output2 = new Uint8Array(32);
sha256.hashObjectToByteArray(obj, output2, 0);
Expand Down Expand Up @@ -97,7 +97,7 @@ describe("sha256.hashObjectToByteArray and sha256.byteArrayToHashObject", functi
];
for (const [i, byteArr] of tcs.entries()) {
it("test case " + i, function () {
const obj = sha256.byteArrayToHashObject(byteArr);
const obj = sha256.byteArrayToHashObject(byteArr, 0);
const newByteArr = new Uint8Array(32);
sha256.hashObjectToByteArray(obj, newByteArr, 0);
expect(newByteArr).to.be.deep.equal(byteArr, "failed test case" + i);
Expand Down
21 changes: 20 additions & 1 deletion packages/as-sha256/test/unit/simd.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe("Test SIMD implementation of as-sha256", () => {

it("testHash4HashObjectInputs", () => {
const input1 = "gajindergajindergajindergajinder";
const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8"));
const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8"), 0);
const outputs = sha256.batchHash4HashObjectInputs(Array.from({length: 8}, () => inputHashObject));
const expectedOutput = new Uint8Array([
190, 57, 56, 15, 241, 208, 38, 30, 111, 55, 218, 254, 66, 120, 182, 98, 239, 97, 31, 28, 178, 247, 192, 161,
Expand All @@ -44,4 +44,23 @@ describe("Test SIMD implementation of as-sha256", () => {
expect(output).to.be.deep.equal(expectedOutput, "incorrect batchHash4UintArray64s result " + i);
}
});

const numHashes = [4, 5, 6, 7];
for (const numHash of numHashes) {
it(`hashInto ${numHash} hashes`, () => {
const inputs = Array.from({length: numHash}, () => crypto.randomBytes(64));
const input = new Uint8Array(numHash * 64);
for (let i = 0; i < numHash; i++) {
input.set(inputs[i], i * 64);
}
const output = new Uint8Array(numHash * 32);

sha256.hashInto(input, output);

const expectedOutputs = Array.from({length: numHash}, (_, i) => sha256.digest64(inputs[i]));
for (let i = 0; i < numHash; i++) {
expect(output.subarray(i * 32, (i + 1) * 32)).to.be.deep.equal(expectedOutputs[i]);
}
});
}
});
2 changes: 1 addition & 1 deletion packages/persistent-merkle-tree/src/hasher/util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ export function hashObjectToUint8Array(obj: HashObject): Uint8Array {
}

export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject {
return byteArrayToHashObject(byteArr);
return byteArrayToHashObject(byteArr, 0);
}
4 changes: 2 additions & 2 deletions packages/persistent-merkle-tree/test/unit/tree.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => {
tree.setNode(BigInt(18), LeafNode.fromRoot(Buffer.alloc(32, 2)));
expect(toHex(tree.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2");
// setHashObjectFn
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2)));
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0));
const tree2 = new Tree(zeroNode(depth));
tree2.setNodeWithFn(BigInt(18), getNewNodeFn);
expect(toHex(tree2.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2");
Expand All @@ -103,7 +103,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => {
tree.setNode(BigInt(60), LeafNode.fromRoot(Buffer.alloc(32, 2)));
expect(toHex(tree.root)).to.equal("02607e58782c912e2f96f4ff9daf494d0d115e7c37e8c2b7ddce17213591151b");
// setHashObjectFn
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2)));
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0));
const tree2 = new Tree(zeroNode(depth));
tree2.setNodeWithFn(BigInt(18), getNewNodeFn);
tree2.setNodeWithFn(BigInt(46), getNewNodeFn);
Expand Down
Loading