feat: ListUintNum64Type create ViewDU from existing tree

packages/persistent-merkle-tree/src/packedNode.ts

@@ -18,9 +18,16 @@ export function packedRootsBytesToNode(depth: number, dataView: DataView, start:
  *
  *    h0     h1     h2     h3     h4     h5     h6     h7
  * |------|------|------|------|------|------|------|------|
+ *
+ * @param values list of uint64 numbers
+ * @param leafNodes optional list of LeafNodes to reuse
  */
-export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] {
-  const leafNodes = new Array<LeafNode>(Math.ceil(values.length / 4));
+export function packedUintNum64sToLeafNodes(values: number[], leafNodes?: LeafNode[]): LeafNode[] {
+  const nodeCount = Math.ceil(values.length / 4);
+  if (leafNodes && leafNodes.length !== nodeCount) {
+    throw new Error(`Invalid leafNode length: ${leafNodes.length} !== ${nodeCount}`);
+  }
+  leafNodes = leafNodes ?? new Array<LeafNode>(Math.ceil(values.length / 4));
   for (let i = 0; i < values.length; i++) {
     const nodeIndex = Math.floor(i / 4);
     const leafNode = leafNodes[nodeIndex] ?? new LeafNode(0, 0, 0, 0, 0, 0, 0, 0);

packages/ssz/src/index.ts

@@ -29,6 +29,7 @@ export {TreeView} from "./view/abstract";
 export {ValueOfFields, ContainerTypeGeneric} from "./view/container";
 export {TreeViewDU} from "./viewDU/abstract";
 export {ListCompositeTreeViewDU} from "./viewDU/listComposite";
+export {ListBasicTreeViewDU} from "./viewDU/listBasic";
 export {ArrayCompositeTreeViewDUCache} from "./viewDU/arrayComposite";
 export {ContainerNodeStructTreeViewDU} from "./viewDU/containerNodeStruct";
 

packages/ssz/src/type/listUintNum64.ts

@@ -1,24 +1,53 @@
-import {LeafNode, Node, packedUintNum64sToLeafNodes, subtreeFillToContents} from "@chainsafe/persistent-merkle-tree";
+import {
+  HashComputationGroup,
+  HashComputationLevel,
+  LeafNode,
+  Node,
+  executeHashComputations,
+  getNodesAtDepth,
+  levelAtIndex,
+  packedUintNum64sToLeafNodes,
+  setNodesAtDepth,
+  subtreeFillToContents,
+  zeroNode,
+} from "@chainsafe/persistent-merkle-tree";
 
 import {ListBasicTreeViewDU} from "../viewDU/listBasic";
 import {ListBasicOpts, ListBasicType} from "./listBasic";
 import {UintNumberType} from "./uint";
-import {addLengthNode} from "./arrayBasic";
+import {addLengthNode, getLengthFromRootNode} from "./arrayBasic";
 
 /**
  * Specific implementation of ListBasicType for UintNumberType with some optimizations.
  */
 export class ListUintNum64Type extends ListBasicType<UintNumberType> {
+  private hcGroup: HashComputationGroup | undefined;
   constructor(limit: number, opts?: ListBasicOpts) {
     super(new UintNumberType(8), limit, opts);
   }
 
   /**
    * Return a ListBasicTreeViewDU with nodes populated
+   * @param unusedViewDU optional, if provided we'll create ViewDU using the provided rootNode. Need to rehash the whole
+   * tree in this case to make it clean for consumers.
    */
-  toViewDU(value: number[]): ListBasicTreeViewDU<UintNumberType> {
+  toViewDU(value: number[], unusedViewDU?: ListBasicTreeViewDU<UintNumberType>): ListBasicTreeViewDU<UintNumberType> {
     // no need to serialize and deserialize like in the abstract class
-    const {treeNode, leafNodes} = this.packedUintNum64sToNode(value);
+    const {treeNode, leafNodes} = this.packedUintNum64sToNode(value, unusedViewDU?.node);
+
+    if (unusedViewDU) {
+      const hcGroup = this.getHcGroup();
+      hcGroup.reset();
+      forceGetHashComputations(treeNode, this.chunkDepth + 1, 0, hcGroup.byLevel);
+      hcGroup.clean();
+
+      treeNode.h0 = null as unknown as number;
+      executeHashComputations(hcGroup.byLevel);
+      // This makes sure the root node is computed by batch
+      if (treeNode.h0 === null) {
+        throw Error("Root is not computed by batch");
+      }
+    }
     // cache leaf nodes in the ViewDU
     return this.getViewDU(treeNode, {
       nodes: leafNodes,
@@ -29,21 +58,86 @@ export class ListUintNum64Type extends ListBasicType<UintNumberType> {
 
   /**
    * No need to serialize and deserialize like in the abstract class
+   * This should be conformed to parent's signature so cannot provide an `unusedViewDU` parameter here
    */
   value_toTree(value: number[]): Node {
     const {treeNode} = this.packedUintNum64sToNode(value);
     return treeNode;
   }
 
-  private packedUintNum64sToNode(value: number[]): {treeNode: Node; leafNodes: LeafNode[]} {
+  private packedUintNum64sToNode(value: number[], unusedRootNode?: Node): {treeNode: Node; leafNodes: LeafNode[]} {
     if (value.length > this.limit) {
       throw new Error(`Exceeds limit: ${value.length} > ${this.limit}`);
     }
 
+    if (unusedRootNode) {
+      // create new tree from unusedRootNode
+      const oldLength = getLengthFromRootNode(unusedRootNode);
+      if (oldLength > value.length) {
+        throw new Error(`Cannot decrease length: ${oldLength} > ${value.length}`);
+      }
+
+      const oldNodeCount = Math.ceil(oldLength / 4);
+      const oldChunksNode = unusedRootNode.left;
+      const oldLeafNodes = getNodesAtDepth(oldChunksNode, this.chunkDepth, 0, oldNodeCount) as LeafNode[];
+      if (oldLeafNodes.length !== oldNodeCount) {
+        throw new Error(`oldLeafNodes.length ${oldLeafNodes.length} !== oldNodeCount ${oldNodeCount}`);
+      }
+
+      const newNodeCount = Math.ceil(value.length / 4);
+      const count = newNodeCount - oldNodeCount;
+      const newLeafNodes = Array.from({length: count}, () => new LeafNode(0, 0, 0, 0, 0, 0, 0, 0));
+      const leafNodes = [...oldLeafNodes, ...newLeafNodes];
+      packedUintNum64sToLeafNodes(value, leafNodes);
+
+      // middle nodes are not changed so consumer must recompute parent hashes
+      const newChunksNode = setNodesAtDepth(
+        oldChunksNode,
+        this.chunkDepth,
+        Array.from({length: count}, (_, i) => oldNodeCount + i),
+        newLeafNodes
+      );
+      const treeNode = addLengthNode(newChunksNode, value.length);
+
+      return {treeNode, leafNodes};
+    }
+
+    // create new tree from scratch
     const leafNodes = packedUintNum64sToLeafNodes(value);
     // subtreeFillToContents mutates the leafNodes array
-    const rootNode = subtreeFillToContents([...leafNodes], this.chunkDepth);
-    const treeNode = addLengthNode(rootNode, value.length);
+    const chunksNode = subtreeFillToContents([...leafNodes], this.chunkDepth);
+    const treeNode = addLengthNode(chunksNode, value.length);
     return {treeNode, leafNodes};
   }
+
+  private getHcGroup(): HashComputationGroup {
+    if (!this.hcGroup) {
+      this.hcGroup = new HashComputationGroup();
+    }
+    return this.hcGroup;
+  }
+}
+
+/**
+ * Consider moving this to persistent-merkle-tree.
+ * For now this is the only flow to force get hash computations.
+ */
+function forceGetHashComputations(
+  node: Node,
+  nodeDepth: number,
+  index: number,
+  hcByLevel: HashComputationLevel[]
+): void {
+  // very important: never mutate zeroNode
+  if (node === zeroNode(nodeDepth) || node.isLeaf()) {
+    return;
+  }
+
+  // if (node.h0 === null) {
+  const hashComputations = levelAtIndex(hcByLevel, index);
+  const {left, right} = node;
+  hashComputations.push(left, right, node);
+  // leaf nodes should have h0 to stop the recursion
+  forceGetHashComputations(left, nodeDepth - 1, index + 1, hcByLevel);
+  forceGetHashComputations(right, nodeDepth - 1, index + 1, hcByLevel);
 }

packages/ssz/test/perf/byType/listUintNum64.test.ts

@@ -0,0 +1,25 @@
+import {itBench} from "@dapplion/benchmark";
+import {ListUintNum64Type} from "../../../src/type/listUintNum64";
+
+describe("ListUintNum64Type.toViewDU", () => {
+  const balancesType = new ListUintNum64Type(1099511627776);
+  const seedLength = 1_900_000;
+  const seedViewDU = balancesType.toViewDU(Array.from({length: seedLength}, () => 0));
+
+  const vc = 2_000_000;
+  const value = Array.from({length: vc}, (_, i) => 32 * 1e9 + i);
+
+  itBench({
+    id: `ListUintNum64Type.toViewDU ${seedLength} -> ${vc}`,
+    fn: () => {
+      balancesType.toViewDU(value, seedViewDU);
+    },
+  });
+
+  itBench({
+    id: "ListUintNum64Type.toViewDU()",
+    fn: () => {
+      balancesType.toViewDU(value);
+    },
+  });
+});

packages/ssz/test/unit/byType/listBasic/listUintNum64.test.ts

@@ -0,0 +1,21 @@
+import {expect} from "chai";
+import {ListUintNum64Type} from "../../../../src/type/listUintNum64";
+
+describe("ListUintNum64Type.toViewDU", () => {
+  const type = new ListUintNum64Type(1024);
+  // seed ViewDU contains 16 leaf nodes = 64 uint64
+  // but we test all cases
+  for (const seedLength of [61, 62, 63, 64]) {
+    const value = Array.from({length: seedLength}, (_, i) => i);
+    const unusedViewDU = type.toViewDU(value);
+
+    it(`should create ViewDU from a seedViewDU with ${seedLength} uint64`, () => {
+      for (let i = seedLength; i < 1024; i++) {
+        const newValue = Array.from({length: i + 1}, (_, j) => j);
+        const expectedRoot = type.toViewDU(newValue).hashTreeRoot();
+        const viewDUFromExistingTree = type.toViewDU(newValue, unusedViewDU);
+        expect(viewDUFromExistingTree.hashTreeRoot()).to.deep.equal(expectedRoot);
+      }
+    });
+  }
+});