Add stateClonedCount metric

packages/beacon-node/src/chain/stateCache/stateContextCache.ts

@@ -38,7 +38,11 @@ export class StateContextCache {
     if (!item) {
       return null;
     }
+
     this.metrics?.hits.inc();
+    // clonedCount + 1 as there's a .clone() below
+    this.metrics?.stateClonedCount.observe(item.clonedCount + 1);
+
     return item.clone();
   }
 

packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts

@@ -36,13 +36,20 @@ export class CheckpointStateCache {
     this.metrics?.lookups.inc();
     const cpKey = toCheckpointKey(cp);
     const item = this.cache.get(cpKey);
-    if (item) {
-      this.metrics?.hits.inc();
-      if (cpKey === this.preComputedCheckpoint) {
-        this.preComputedCheckpointHits = (this.preComputedCheckpointHits ?? 0) + 1;
-      }
+
+    if (!item) {
+      return null;
+    }
+
+    this.metrics?.hits.inc();
+    // clonedCount + 1 as there's a .clone() below
+    this.metrics?.stateClonedCount.observe(item.clonedCount + 1);
+
+    if (cpKey === this.preComputedCheckpoint) {
+      this.preComputedCheckpointHits = (this.preComputedCheckpointHits ?? 0) + 1;
     }
-    return item ? item.clone() : null;
+
+    return item.clone();
   }
 
   add(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void {

packages/beacon-node/src/metrics/metrics/lodestar.ts

@@ -821,6 +821,11 @@ export function createLodestarMetrics(
         name: "lodestar_state_cache_seconds_since_last_read",
         help: "Avg min max of all state cache items seconds since last reads",
       }),
+      stateClonedCount: register.histogram({
+        name: "lodestar_state_cache_state_cloned_clount",
+        help: "Histogram of cloned count per state every time state.clone() is called",
+        buckets: [1, 2, 5, 10, 50, 250],
+      }),
     },
 
     cpStateCache: {
@@ -852,6 +857,11 @@ export function createLodestarMetrics(
         name: "lodestar_cp_state_epoch_seconds_since_last_read",
         help: "Avg min max of all state cache items seconds since last reads",
       }),
+      stateClonedCount: register.histogram({
+        name: "lodestar_cp_state_cache_state_cloned_clount",
+        help: "Histogram of cloned count per state every time state.clone() is called",
+        buckets: [1, 2, 5, 10, 50, 250],
+      }),
     },
 
     seenCache: {

packages/state-transition/src/cache/stateCache.ts

@@ -5,6 +5,8 @@ import {BeaconStatePhase0, BeaconStateAltair, BeaconStateBellatrix, BeaconStateA
 export type BeaconStateCache = {
   config: IBeaconConfig;
   epochCtx: EpochContext;
+  /** Count of clones created from this BeaconStateCache instance. readonly to prevent accidental usage downstream */
+  readonly clonedCount: number;
 };
 
 /**
@@ -119,6 +121,7 @@ export function createCachedBeaconState<T extends BeaconStateAllForks>(
   return getCachedBeaconState(state, {
     config: immutableData.config,
     epochCtx: EpochContext.createFromState(state, immutableData, opts),
+    clonedCount: 0,
   });
 }
 
@@ -132,16 +135,22 @@ export function getCachedBeaconState<T extends BeaconStateAllForks>(
   const cachedState = state as T & BeaconStateCache;
   cachedState.config = cache.config;
   cachedState.epochCtx = cache.epochCtx;
+  (cachedState as {clonedCount: number}).clonedCount = cache.clonedCount;
 
   // Overwrite .clone function to preserve cache
   // TreeViewDU.clone() creates a new object that does not have the attached cache
   const viewDUClone = cachedState.clone.bind(cachedState);
 
   function clone(this: T & BeaconStateCache): T & BeaconStateCache {
     const viewDUCloned = viewDUClone();
+
+    // Override `readonly` attribute in single place where `.clonedCount` is incremented
+    (this as {clonedCount: number}).clonedCount++;
+
     return getCachedBeaconState(viewDUCloned, {
       config: this.config,
       epochCtx: this.epochCtx.clone(),
+      clonedCount: 0,
     }) as T & BeaconStateCache;
   }