Drop flat caches

Simplify command

Fix PR issues

Use .rc

Add more accurate epoch perf tests

Re-arrange epoch tests with a real state

Fix perf type issues

Ensure state is valid

Review all spec tests

Fix hash computations

Fix processRewardsAndPenalties

Fix perf titles

Re-org imports

Fix get state script

Perf test hashing

Fix altair step

Review stfn caches and performance

Add SLOW CODE comments and whitespace

Cache effectiveBalances

Process altair attestations in batch

Don't keep withdrawalCredentials and pubkey in validators flat

Move flat arrays to epoch process only

Add exitQueue churn cache

Cache chrunLimit

Optimize beacon state transition

Move exitQueue cached values to epochCtx

Improve error message in processAttestation

Add comments to processBlock functions

Allow to set validators and balances

Convert from struct to tree properly

Silence effectiveBalances type warnings

Add data representation tradeoffs

Add more docs

Fix type errors

packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts

@@ -7,8 +7,11 @@ import {CachedBeaconState} from "../util";
  */
 export function initiateValidatorExit(state: CachedBeaconState<allForks.BeaconState>, index: ValidatorIndex): void {
   const {config, validators, epochCtx} = state;
+
+  const validator = validators[index];
+
   // return if validator already initiated exit
-  if (validators[index].exitEpoch !== FAR_FUTURE_EPOCH) {
+  if (validator.exitEpoch !== FAR_FUTURE_EPOCH) {
     return;
   }
 
@@ -24,8 +27,6 @@ export function initiateValidatorExit(state: CachedBeaconState<allForks.BeaconSt
   }
 
   // set validator exit epoch and withdrawable epoch
-  validators.update(index, {
-    exitEpoch: epochCtx.exitQueueEpoch,
-    withdrawableEpoch: epochCtx.exitQueueEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY,
-  });
+  validator.exitEpoch = epochCtx.exitQueueEpoch;
+  validator.withdrawableEpoch = epochCtx.exitQueueEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY;
 }

packages/beacon-state-transition/src/allForks/block/processDeposit.ts

@@ -67,17 +67,19 @@ export function processDeposit(
     }
 
     // add validator and balance entries
+    const effectiveBalance = bigIntMin(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE);
     validators.push({
       pubkey,
       withdrawalCredentials: deposit.data.withdrawalCredentials.valueOf() as Uint8Array, // Drop tree
       activationEligibilityEpoch: FAR_FUTURE_EPOCH,
       activationEpoch: FAR_FUTURE_EPOCH,
       exitEpoch: FAR_FUTURE_EPOCH,
       withdrawableEpoch: FAR_FUTURE_EPOCH,
-      effectiveBalance: bigIntMin(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE),
+      effectiveBalance: effectiveBalance,
       slashed: false,
     });
     state.balances.push(amount);
+    epochCtx.effectiveBalances.push(effectiveBalance);
 
     // add participation caches
     state.previousEpochParticipation.pushStatus({timelyHead: false, timelySource: false, timelyTarget: false});

packages/beacon-state-transition/src/allForks/block/slashValidator.ts

@@ -24,13 +24,16 @@ export function slashValidatorAllForks(
 ): void {
   const {validators, epochCtx} = state;
   const epoch = epochCtx.currentShuffling.epoch;
+
+  // TODO: Merge initiateValidatorExit validators.update() with the one below
   initiateValidatorExit(state as CachedBeaconState<allForks.BeaconState>, slashedIndex);
+
   const validator = validators[slashedIndex];
-  validators.update(slashedIndex, {
-    slashed: true,
-    withdrawableEpoch: Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR),
-  });
 
+  validator.slashed = true;
+  validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR);
+
+  // TODO: Use effectiveBalance array
   const {effectiveBalance} = validator;
   state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += effectiveBalance;
 
@@ -39,16 +42,18 @@ export function slashValidatorAllForks(
   decreaseBalance(state, slashedIndex, effectiveBalance / minSlashingPenaltyQuotient);
 
   // apply proposer and whistleblower rewards
-  const proposerIndex = epochCtx.getBeaconProposer(state.slot);
-  if (whistleblowerIndex === undefined || !Number.isSafeInteger(whistleblowerIndex)) {
-    whistleblowerIndex = proposerIndex;
-  }
   const whistleblowerReward = effectiveBalance / WHISTLEBLOWER_REWARD_QUOTIENT;
   const proposerReward =
     fork === ForkName.phase0
       ? whistleblowerReward / PROPOSER_REWARD_QUOTIENT
       : (whistleblowerReward * PROPOSER_WEIGHT) / WEIGHT_DENOMINATOR;
 
-  increaseBalance(state, proposerIndex, proposerReward);
-  increaseBalance(state, whistleblowerIndex, whistleblowerReward - proposerReward);
+  const proposerIndex = epochCtx.getBeaconProposer(state.slot);
+  if (whistleblowerIndex === undefined || !Number.isSafeInteger(whistleblowerIndex)) {
+    // Call increaseBalance() once with `(whistleblowerReward - proposerReward) + proposerReward`
+    increaseBalance(state, proposerIndex, whistleblowerReward);
+  } else {
+    increaseBalance(state, proposerIndex, proposerReward);
+    increaseBalance(state, whistleblowerIndex, whistleblowerReward - proposerReward);
+  }
 }

packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts

@@ -23,23 +23,32 @@ export function processEffectiveBalanceUpdates(
   state: CachedBeaconState<allForks.BeaconState>,
   epochProcess: IEpochProcess
 ): void {
-  const {validators} = state;
+  const {validators, epochCtx} = state;
   const HYSTERESIS_INCREMENT = EFFECTIVE_BALANCE_INCREMENT / BigInt(HYSTERESIS_QUOTIENT);
   const DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * BigInt(HYSTERESIS_DOWNWARD_MULTIPLIER);
   const UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * BigInt(HYSTERESIS_UPWARD_MULTIPLIER);
 
+  const effectiveBalancesArr = epochCtx.effectiveBalances.toArray();
+
   // update effective balances with hysteresis
-  (epochProcess.balances ?? state.balances).forEach((balance: bigint, i: number) => {
-    const effectiveBalance = epochProcess.validators[i].effectiveBalance;
+  for (let i = 0, len = epochProcess.balancesFlat.length; i < len; i++) {
+    const balance = epochProcess.balancesFlat[i];
+    const effectiveBalance = effectiveBalancesArr[i];
     if (
       // Too big
       effectiveBalance > balance + DOWNWARD_THRESHOLD ||
       // Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates
       (effectiveBalance < MAX_EFFECTIVE_BALANCE && effectiveBalance < balance - UPWARD_THRESHOLD)
     ) {
-      validators.update(i, {
-        effectiveBalance: bigIntMin(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE),
-      });
+      const newEffectiveBalance = bigIntMin(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE);
+      if (newEffectiveBalance !== effectiveBalance) {
+        // Update the state tree
+        validators[i].effectiveBalance = newEffectiveBalance;
+        // Also update the fast cached version
+        // Should happen rarely, so it's fine to update the tree
+        // TODO: Update all in batch after this loop
+        epochCtx.effectiveBalances.set(i, newEffectiveBalance);
+      }
     }
-  });
+  }
 }

packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts

@@ -23,6 +23,7 @@ export function processRegistryUpdates(
 ): void {
   const {validators, epochCtx} = state;
 
+  // TODO: Batch set this properties in the tree at once with setMany() or setNodes()
   // process ejections
   for (const index of epochProcess.indicesToEject) {
     // set validator exit epoch and withdrawable epoch
@@ -32,20 +33,29 @@ export function processRegistryUpdates(
 
   // set new activation eligibilities
   for (const index of epochProcess.indicesEligibleForActivationQueue) {
-    validators.update(index, {
-      activationEligibilityEpoch: epochCtx.currentShuffling.epoch + 1,
-    });
+    validators[index].activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1;
+  }
+
+  // process ejections
+  for (const index of epochProcess.indicesToEject) {
+    // set validator exit epoch and withdrawable epoch
+    // TODO: Figure out a way to quickly set properties on the validators tree
+    initiateValidatorExit(state, index);
   }
 
   const finalityEpoch = state.finalizedCheckpoint.epoch;
+
   // dequeue validators for activation up to churn limit
   for (const index of epochProcess.indicesEligibleForActivation.slice(0, epochCtx.churnLimit)) {
+    const validator = validators[index];
     // placement in queue is finalized
-    if (epochProcess.validators[index].activationEligibilityEpoch > finalityEpoch) {
-      break; // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early
+    if (validator.activationEligibilityEpoch > finalityEpoch) {
+      // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early
+      // activationEligibilityEpoch has been sorted in epoch process in ascending order.
+      // At that point the finalityEpoch was not known because processJustificationAndFinalization() wasn't called yet.
+      // So we need to filter by finalityEpoch here to comply with the spec.
+      break;
     }
-    validators.update(index, {
-      activationEpoch: computeActivationExitEpoch(epochProcess.currentEpoch),
-    });
+    validator.activationEpoch = computeActivationExitEpoch(epochProcess.currentEpoch);
   }
 }

packages/beacon-state-transition/src/allForks/epoch/processSlashings.ts

@@ -23,25 +23,33 @@ import {CachedBeaconState, IEpochProcess} from "../../allForks/util";
 export function processSlashingsAllForks(
   fork: ForkName,
   state: CachedBeaconState<allForks.BeaconState>,
-  process: IEpochProcess
+  epochProcess: IEpochProcess
 ): void {
   // No need to compute totalSlashings if there no index to slash
-  if (process.indicesToSlash.length === 0) {
+  if (epochProcess.indicesToSlash.length === 0) {
     return;
   }
 
-  const totalBalance = process.totalActiveStake;
-  const totalSlashings = Array.from(readonlyValues(state.slashings)).reduce((a, b) => a + b, BigInt(0));
+  const totalBalance = epochProcess.totalActiveStake;
+  // TODO: Use readonlyAllValues()
+  let totalSlashings = BigInt(0);
+  for (const slashing of readonlyValues(state.slashings)) {
+    totalSlashings += slashing;
+  }
 
   const proportionalSlashingMultiplier =
     fork === ForkName.phase0 ? PROPORTIONAL_SLASHING_MULTIPLIER : PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR;
 
   const adjustedTotalSlashingBalance = bigIntMin(totalSlashings * proportionalSlashingMultiplier, totalBalance);
   const increment = EFFECTIVE_BALANCE_INCREMENT;
-  for (const index of process.indicesToSlash) {
-    const effectiveBalance = process.validators[index].effectiveBalance;
+  for (const index of epochProcess.indicesToSlash) {
+    const effectiveBalance = state.epochCtx.effectiveBalances.get(index)!;
+
     const penaltyNumerator = (effectiveBalance / increment) * adjustedTotalSlashingBalance;
     const penalty = (penaltyNumerator / totalBalance) * increment;
+
+    // In all forks processSlashings() is called after processRewardsAndPenalties() but before processEffectiveBalanceUpdates()
+    // The changes done here must apply to both the state and balances array mutated in processRewardsAndPenalties()
     decreaseBalance(state, index, penalty);
   }
 }

packages/beacon-state-transition/src/allForks/stateTransition.ts

@@ -147,7 +147,7 @@ function processSlotsWithTransientCache(
       try {
         const epochProcess = beforeProcessEpoch(postState);
         processEpochByFork[fork](postState, epochProcess);
-        metrics?.registerValidatorStatuses(epochProcess.currentEpoch, epochProcess.statuses);
+        metrics?.registerValidatorStatuses(epochProcess.currentEpoch, epochProcess.statusesFlat);
 
         postState.slot++;
         afterProcessEpoch(postState, epochProcess);