From 5bfa88836a19201a9a98292259aa92c173b830cd Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:28:20 +0100 Subject: [PATCH 01/30] Integrate SSZ v2 This is a combination of 50 commits. Remove SSZ from params Update types in lodestar-api Replace .defaultValue() with .defaultValue Fix more types More type fixes Fix upgradeState functions Fix processDeposit pending logic Update test types f Update fork-choice types Update validator types WIP Lodestar changes WIP Lodestar Fix spec tests FIx more Lodestar types Fix proofs types Fix all type issues Use Uint64NumInf only where applicable Don't manipulate casing in loadYaml WIP Fix spec tests Fix build types Fix test types Update test types Fix getRandaoMix tests Review pickEth1Vote syncPubkeys in createFromState Fix unit tests Safe clone for CachedBeaconState Fix unit tests issues Fix genesis logic with ViewDU Fix all unit tests Fix processRewardsAndPenaltiesAllForks Commit state after processing Fix statusProcessEpoch Create valid pubkey cache in perf test Fix getDeposits for perf test Ensure CachedBeaconState clone is immutable Fix genesis spec tests Ensure stable commit in ViewDU Add advice on running spec tests Remove use of the type unsafe Number constructor Fix casing in spec test definitions Fix casing and bigint types in spec tests Advance epochCtx cached time units Add debug mode to fork-choice tests Fix spec test bigint type Fix typo in processAttestation flags Fix altair transition spec tests Fix altair transition tests Commit ViewDU changes in upgradeState Fix rebase issues --- package.json | 4 +- packages/api/package.json | 4 +- packages/api/src/client/utils/client.ts | 6 +- packages/api/src/routes/beacon/block.ts | 24 +- packages/api/src/routes/beacon/pool.ts | 11 +- packages/api/src/routes/beacon/state.ts | 49 +- packages/api/src/routes/config.ts | 20 +- packages/api/src/routes/debug.ts | 9 +- packages/api/src/routes/events.ts | 80 +-- packages/api/src/routes/lightclient.ts | 32 +- packages/api/src/routes/lodestar.ts | 32 +- packages/api/src/routes/node.ts | 25 +- packages/api/src/routes/validator.ts | 101 ++- packages/api/src/server/debug.ts | 5 +- packages/api/src/server/utils/server.ts | 15 +- packages/api/src/utils/serdes.ts | 12 +- packages/api/src/utils/types.ts | 52 +- packages/api/test/unit/beacon.test.ts | 40 +- packages/api/test/unit/config.test.ts | 2 +- packages/api/test/unit/debug.test.ts | 6 +- packages/api/test/unit/lightclient.test.ts | 6 +- packages/api/test/unit/node.test.ts | 2 +- packages/api/test/unit/validator.test.ts | 14 +- packages/beacon-state-transition/README.md | 10 +- packages/beacon-state-transition/package.json | 5 +- .../allForks/block/initiateValidatorExit.ts | 2 +- .../block/isValidIndexedAttestation.ts | 3 +- .../allForks/block/processAttesterSlashing.ts | 4 +- .../src/allForks/block/processBlockHeader.ts | 13 +- .../src/allForks/block/processDeposit.ts | 49 +- .../src/allForks/block/processEth1Data.ts | 29 +- .../allForks/block/processProposerSlashing.ts | 12 +- .../src/allForks/block/processRandao.ts | 11 +- .../allForks/block/processVoluntaryExit.ts | 10 +- .../src/allForks/block/slashValidator.ts | 7 +- .../epoch/processEffectiveBalanceUpdates.ts | 23 +- .../allForks/epoch/processEth1DataReset.ts | 5 +- .../epoch/processHistoricalRootsUpdate.ts | 8 +- .../processJustificationAndFinalization.ts | 30 +- .../allForks/epoch/processRandaoMixesReset.ts | 6 +- .../allForks/epoch/processRegistryUpdates.ts | 6 +- .../epoch/processRewardsAndPenalties.ts | 25 +- .../src/allForks/epoch/processSlashings.ts | 11 +- .../allForks/epoch/processSlashingsReset.ts | 2 +- .../signatureSets/attesterSlashings.ts | 15 +- .../signatureSets/indexedAttestation.ts | 15 +- .../src/allForks/signatureSets/proposer.ts | 2 +- .../signatureSets/proposerSlashings.ts | 17 +- .../src/allForks/signatureSets/randao.ts | 2 +- .../allForks/signatureSets/voluntaryExits.ts | 13 +- .../src/allForks/slot/index.ts | 17 +- .../src/allForks/stateTransition.ts | 23 +- .../src/altair/block/index.ts | 2 +- .../src/altair/block/processAttestation.ts | 77 +-- .../src/altair/block/processOperations.ts | 11 +- .../src/altair/block/processSyncCommittee.ts | 66 +- .../altair/epoch/getRewardsAndPenalties.ts | 24 +- .../altair/epoch/processInactivityUpdates.ts | 44 +- .../epoch/processParticipationFlagUpdates.ts | 13 +- .../epoch/processRewardsAndPenalties.ts | 8 +- .../epoch/processSyncCommitteeUpdates.ts | 14 +- .../src/altair/upgradeState.ts | 138 +++-- .../src/bellatrix/block/index.ts | 2 +- .../block/processExecutionPayload.ts | 10 +- .../src/bellatrix/block/processOperations.ts | 15 +- .../src/bellatrix/upgradeState.ts | 77 ++- .../src/bellatrix/utils.ts | 28 +- .../src/cache/balanceList.ts | 67 -- .../src/cache/cachedBeaconState.ts | 355 ----------- .../src/cache/cachedEpochParticipation.ts | 119 ---- .../src/cache/cachedInactivityScoreList.ts | 97 --- .../src/cache/effectiveBalanceIncrements.ts | 17 + .../src/cache/epochContext.ts | 138 +++-- .../src/cache/epochProcess.ts | 117 ++-- .../src/cache/pubkeyCache.ts | 12 +- .../src/cache/stateCache.ts | 151 +++++ .../src/cache/syncCommitteeCache.ts | 12 +- .../src/cache/types.ts | 13 + packages/beacon-state-transition/src/index.ts | 14 +- .../src/phase0/block/index.ts | 2 +- .../src/phase0/block/processAttestation.ts | 6 +- .../src/phase0/block/processOperations.ts | 33 +- .../src/phase0/epoch/getAttestationDeltas.ts | 8 +- .../src/phase0/epoch/index.ts | 2 +- .../processParticipationRecordUpdates.ts | 7 +- .../epoch/processPendingAttestations.ts | 35 +- .../epoch/processRewardsAndPenalties.ts | 7 +- packages/beacon-state-transition/src/types.ts | 20 +- .../src/util/aggregationBits.ts | 189 ------ .../src/util/aggregator.ts | 4 +- .../src/util/altair.ts | 13 + .../beacon-state-transition/src/util/array.ts | 21 + .../src/util/attesterStatus.ts | 6 +- .../src/util/balance.ts | 48 +- .../src/util/blockRoot.ts | 7 +- .../src/util/domain.ts | 4 +- .../beacon-state-transition/src/util/epoch.ts | 4 +- .../src/util/epochShuffling.ts | 16 +- .../src/util/finality.ts | 3 +- .../src/util/genesis.ts | 181 +++--- .../beacon-state-transition/src/util/index.ts | 2 - .../beacon-state-transition/src/util/seed.ts | 37 +- .../src/util/shuffle.ts | 14 +- .../src/util/shufflingDecisionRoot.ts | 9 +- .../src/util/signatureSets.ts | 4 +- .../beacon-state-transition/src/util/slot.ts | 12 +- .../src/util/syncCommittee.ts | 18 +- .../src/util/unsafeUint8ArrayToTree.ts | 26 - .../src/util/validator.ts | 17 +- .../src/util/weakSubjectivity.ts | 27 +- .../test/perf/allForks/hashing.test.ts | 23 +- .../perf/allForks/util/shufflings.test.ts | 8 +- .../altair/block/processAttestation.test.ts | 80 +-- .../perf/altair/block/processBlock.test.ts | 62 +- .../perf/altair/block/processEth1Data.test.ts | 54 ++ .../test/perf/altair/epoch/epoch.test.ts | 1 + .../epoch/processRewardsAndPenalties.test.ts | 3 +- .../epoch/processSyncCommitteeUpdates.test.ts | 20 +- .../test/perf/altair/epoch/util.ts | 2 +- .../test/perf/analyzeBlocks.ts | 3 +- .../test/perf/analyzeEpochs.ts | 14 +- .../test/perf/dataStructures/arrayish.test.ts | 4 +- .../test/perf/misc/aggregationBits.test.ts | 15 +- .../test/perf/misc/rootEquals.test.ts | 10 +- .../perf/phase0/block/processBlock.test.ts | 2 +- .../test/perf/phase0/block/util.ts | 68 +- .../processEffectiveBalanceUpdates.test.ts | 23 +- .../test/perf/sanityCheck.test.ts | 4 +- .../test/perf/types.ts | 6 +- .../beacon-state-transition/test/perf/util.ts | 442 +++++++------ .../block/isValidIndexedAttestation.test.ts | 27 +- .../test/unit/cachedBeaconState.test.ts | 57 ++ .../unit/signatureSets/signatureSets.test.ts | 44 +- .../test/unit/util/aggregationBits.test.ts | 169 ----- .../test/unit/util/balance.test.ts | 42 +- .../test/unit/util/cachedBeaconState.test.ts | 12 +- .../test/unit/util/epoch.test.ts | 4 +- .../test/unit/util/flags.test.ts | 40 ++ .../test/unit/util/interface.test.ts | 88 --- .../test/unit/util/seed.test.ts | 25 +- .../test/unit/util/validator.test.ts | 12 +- .../test/utils/attestation.ts | 4 +- .../test/utils/block.ts | 11 +- .../test/utils/specTestTypes/stateTestCase.ts | 6 - .../test/utils/state.ts | 60 +- .../test/utils/validator.ts | 6 +- packages/cli/package.json | 2 +- .../src/cmds/account/cmds/validator/create.ts | 2 +- .../cmds/account/cmds/validator/recover.ts | 2 +- .../validator/slashingProtection/export.ts | 4 +- .../cli/src/cmds/beacon/initBeaconState.ts | 23 +- packages/cli/src/cmds/dev/handler.ts | 2 +- packages/cli/src/config/beaconNodeOptions.ts | 5 +- packages/cli/src/config/peerId.ts | 3 +- .../cli/src/depositContract/depositData.ts | 7 +- packages/cli/src/networks/index.ts | 7 +- packages/cli/src/util/file.ts | 11 +- packages/config/package.json | 2 +- packages/config/src/chainConfig/sszTypes.ts | 58 -- packages/config/src/forkConfig/index.ts | 2 +- packages/config/src/genesisConfig/index.ts | 4 +- packages/db/package.json | 2 +- packages/db/src/abstractRepository.ts | 16 +- packages/fork-choice/package.json | 2 +- .../fork-choice/src/forkChoice/forkChoice.ts | 49 +- .../fork-choice/src/forkChoice/interface.ts | 3 +- packages/fork-choice/src/forkChoice/store.ts | 2 +- .../test/perf/forkChoice/forkChoice.test.ts | 8 +- .../perf/protoArray/computeDeltas.test.ts | 22 +- packages/light-client/package.json | 4 +- packages/light-client/src/index.ts | 4 +- packages/light-client/src/utils/domain.ts | 4 +- packages/light-client/src/utils/utils.ts | 36 +- packages/light-client/src/validation.ts | 12 +- packages/light-client/test/getGenesisData.ts | 2 +- .../light-client/test/lightclientApiServer.ts | 11 +- packages/light-client/test/naive/update.ts | 2 +- .../light-client/test/prepareUpdateNaive.ts | 22 +- packages/light-client/test/types.ts | 4 + packages/light-client/test/unit/sync.test.ts | 11 +- .../light-client/test/unit/syncNaive.test.ts | 21 +- .../light-client/test/unit/validation.test.ts | 23 +- packages/light-client/test/utils.ts | 16 +- packages/lodestar/package.json | 4 +- .../src/api/impl/beacon/blocks/index.ts | 2 +- .../lodestar/src/api/impl/beacon/index.ts | 2 +- .../src/api/impl/beacon/state/index.ts | 52 +- .../src/api/impl/beacon/state/utils.ts | 41 +- packages/lodestar/src/api/impl/debug/index.ts | 4 +- .../lodestar/src/api/impl/events/index.ts | 2 +- .../src/api/impl/lightclient/index.ts | 35 +- .../lodestar/src/api/impl/lodestar/index.ts | 4 +- .../lodestar/src/api/impl/validator/index.ts | 6 +- .../lodestar/src/api/impl/validator/utils.ts | 29 +- .../lodestar/src/chain/blocks/importBlock.ts | 4 +- .../src/chain/blocks/utils/checkpoint.ts | 3 +- .../lodestar/src/chain/blocks/verifyBlock.ts | 12 +- .../src/chain/bls/multithread/index.ts | 2 +- .../lodestar/src/chain/bls/singleThread.ts | 2 +- packages/lodestar/src/chain/chain.ts | 45 +- packages/lodestar/src/chain/eventHandlers.ts | 22 +- .../lodestar/src/chain/factory/block/body.ts | 15 +- .../lodestar/src/chain/factory/block/index.ts | 9 +- .../lodestar/src/chain/genesis/genesis.ts | 61 +- .../lodestar/src/chain/genesis/interface.ts | 9 +- packages/lodestar/src/chain/initState.ts | 72 +-- packages/lodestar/src/chain/interface.ts | 5 +- .../lodestar/src/chain/lightClient/index.ts | 25 +- .../lodestar/src/chain/lightClient/proofs.ts | 14 +- .../opPools/aggregatedAttestationPool.ts | 52 +- .../src/chain/opPools/attestationPool.ts | 28 +- packages/lodestar/src/chain/opPools/opPool.ts | 17 +- .../chain/opPools/syncCommitteeMessagePool.ts | 19 +- .../opPools/syncContributionAndProofPool.ts | 40 +- packages/lodestar/src/chain/regen/regen.ts | 8 +- .../src/chain/stateCache/stateContextCache.ts | 8 +- .../src/chain/validation/aggregateAndProof.ts | 8 +- .../src/chain/validation/attestation.ts | 29 +- .../signatureSets/aggregateAndProof.ts | 2 +- .../signatureSets/contributionAndProof.ts | 2 +- .../signatureSets/selectionProof.ts | 2 +- .../validation/signatureSets/syncCommittee.ts | 2 +- .../syncCommitteeContribution.ts | 32 +- .../syncCommitteeSelectionProof.ts | 2 +- .../syncCommitteeContributionAndProof.ts | 28 +- .../src/db/repositories/blockArchive.ts | 7 +- .../src/db/repositories/blockArchiveIndex.ts | 12 +- .../src/db/repositories/depositDataRoot.ts | 57 +- .../lightclientBestPartialUpdate.ts | 59 +- .../lightclientSyncCommitteeWitness.ts | 12 +- .../src/db/repositories/stateArchive.ts | 27 +- .../src/db/repositories/stateArchiveIndex.ts | 2 +- .../lodestar/src/db/single/preGenesisState.ts | 20 +- .../preGenesisStateLastProcessedBlock.ts | 12 +- .../src/eth1/eth1DepositDataTracker.ts | 13 +- packages/lodestar/src/eth1/index.ts | 5 +- packages/lodestar/src/eth1/provider/utils.ts | 8 +- .../src/eth1/utils/depositContract.ts | 9 +- packages/lodestar/src/eth1/utils/deposits.ts | 22 +- packages/lodestar/src/eth1/utils/eth1Data.ts | 12 +- packages/lodestar/src/eth1/utils/eth1Vote.ts | 18 +- .../lodestar/src/executionEngine/interface.ts | 5 +- packages/lodestar/src/executionEngine/mock.ts | 2 +- packages/lodestar/src/metrics/metrics.ts | 5 +- .../lodestar/src/metrics/metrics/lodestar.ts | 4 +- .../src/network/gossip/validation/index.ts | 11 +- packages/lodestar/src/network/metadata.ts | 12 +- packages/lodestar/src/network/network.ts | 2 +- .../lodestar/src/network/peers/metastore.ts | 2 +- .../lodestar/src/network/peers/peerManager.ts | 8 +- .../peers/utils/assertPeerRelevance.ts | 2 +- .../peers/utils/enrSubnetsDeserialize.ts | 3 +- .../network/peers/utils/prioritizePeers.ts | 11 +- .../reqresp/encoders/responseDecode.ts | 4 +- .../reqresp/encodingStrategies/index.ts | 7 +- .../encodingStrategies/sszSnappy/decode.ts | 27 +- .../reqresp/handlers/beaconBlocksByRoot.ts | 2 +- .../lodestar/src/network/reqresp/types.ts | 10 - .../src/network/subnets/attnetsService.ts | 4 +- .../src/network/subnets/syncnetsService.ts | 4 +- packages/lodestar/src/node/nodejs.ts | 6 +- packages/lodestar/src/node/notifier.ts | 6 +- .../src/node/utils/interop/deposits.ts | 15 +- .../lodestar/src/node/utils/interop/state.ts | 16 +- packages/lodestar/src/node/utils/state.ts | 27 +- .../lodestar/src/sync/backfill/backfill.ts | 9 +- packages/lodestar/src/sync/unknownBlock.ts | 4 +- packages/lodestar/src/util/multifork.ts | 5 +- packages/lodestar/src/util/tree.ts | 14 - .../test/e2e/chain/lightclient.test.ts | 5 +- .../e2e/eth1/eth1ForBlockProduction.test.ts | 12 +- .../test/e2e/interop/genesisState.test.ts | 92 +++ .../test/e2e/network/gossipsub.test.ts | 4 +- .../lodestar/test/e2e/network/network.test.ts | 2 +- .../e2e/network/peers/peerManager.test.ts | 3 +- .../lodestar/test/e2e/network/reqresp.test.ts | 5 +- .../perf/api/impl/validator/attester.test.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 46 +- .../stateContextCheckpointsCache.test.ts | 2 +- .../validation/aggregateAndProof.test.ts | 4 +- .../perf/chain/validation/attestation.test.ts | 4 +- .../test/perf/eth1/pickEth1Vote.test.ts | 29 +- packages/lodestar/test/sim/threaded/types.ts | 3 +- .../test/spec/allForks/epochProcessing.ts | 22 +- .../lodestar/test/spec/allForks/finality.ts | 42 +- packages/lodestar/test/spec/allForks/fork.ts | 18 +- .../lodestar/test/spec/allForks/forkChoice.ts | 33 +- .../lodestar/test/spec/allForks/genesis.ts | 59 +- .../lodestar/test/spec/allForks/merkle.ts | 24 +- .../lodestar/test/spec/allForks/operations.ts | 33 +- .../lodestar/test/spec/allForks/rewards.ts | 44 +- .../lodestar/test/spec/allForks/sanity.ts | 48 +- .../lodestar/test/spec/allForks/ssz_static.ts | 128 ++-- .../lodestar/test/spec/allForks/transition.ts | 43 +- .../test/spec/altair/epoch_processing.test.ts | 4 +- .../test/spec/altair/operations.test.ts | 15 +- .../test/spec/bellatrix/operations.test.ts | 8 +- .../test/spec/phase0/operations.test.ts | 9 +- .../test/spec/phase0/shuffling.test.ts | 12 +- .../test/spec/ssz/generic/index.test.ts | 90 +-- .../lodestar/test/spec/ssz/generic/types.ts | 102 ++- .../test/spec/ssz/util/specTypes/uint.ts | 8 - packages/lodestar/test/spec/util.ts | 18 +- .../replaceUintTypeWithUintBigintType.ts | 52 ++ .../test/spec/utils/runValidSszTest.ts | 188 ++++++ .../specTestTypes/beaconStateComparison.ts | 47 -- .../test/spec/utils/specTestTypes/genesis.ts | 13 - .../spec/utils/specTestTypes/shufflingCase.ts | 7 - .../spec/utils/specTestTypes/stateTestCase.ts | 7 - .../test/spec/utils/sszTestCaseParser.ts | 92 +++ .../api/impl/beacon/blocks/getBlock.test.ts | 2 - .../unit/api/impl/beacon/pool/pool.test.ts | 7 +- .../impl/beacon/state/stateValidators.test.ts | 181 ------ .../test/unit/api/impl/node/node.test.ts | 5 +- .../impl/validator/duties/proposer.test.ts | 9 +- .../unit/api/impl/validator/utils.test.ts | 22 +- .../unit/chain/blocks/verifyBlock.test.ts | 2 +- .../chain/factory/block/blockAssembly.test.ts | 2 +- .../unit/chain/factory/block/body.test.ts | 6 +- .../unit/chain/forkChoice/forkChoice.test.ts | 54 +- .../test/unit/chain/genesis/genesis.test.ts | 4 +- .../test/unit/chain/lightclient/proof.test.ts | 16 +- .../opPools/aggregatedAttestationPool.test.ts | 56 +- .../unit/chain/opPools/syncCommittee.test.ts | 8 +- .../opPools/syncCommitteeContribution.test.ts | 72 +-- .../stateCache/stateContextCache.test.ts | 5 +- .../validation/aggregateAndProof.test.ts | 8 +- .../unit/chain/validation/attestation.test.ts | 14 +- .../chain/validation/attesterSlashing.test.ts | 11 +- .../test/unit/chain/validation/block.test.ts | 2 +- .../validation/contributionAndProof.test.ts | 34 +- .../chain/validation/proposerSlashing.test.ts | 8 +- .../chain/validation/syncCommittee.test.ts | 1 - .../chain/validation/voluntaryExit.test.ts | 11 +- .../db/api/repositories/blockArchive.test.ts | 4 +- .../test/unit/db/api/repository.test.ts | 26 +- .../test/unit/eth1/utils/deposits.test.ts | 19 +- .../test/unit/eth1/utils/eth1Data.test.ts | 34 +- .../test/unit/eth1/utils/eth1Vote.test.ts | 19 +- packages/lodestar/test/unit/metrics/utils.ts | 2 +- .../unit/network/attestationService.test.ts | 8 +- .../test/unit/network/peers/metastore.test.ts | 8 +- .../unit/network/peers/priorization.test.ts | 29 +- .../network/peers/utils/enrSubnets.test.ts | 6 +- .../sszSnappy/decode.test.ts | 4 +- .../encodingStrategies/sszSnappy/testData.ts | 14 +- .../test/unit/network/reqresp/utils.ts | 6 +- .../test/unit/sync/backfill/verify.test.ts | 7 +- .../test/unit/sync/unknownBlock.test.ts | 6 +- .../lodestar/test/utils/aggregationBits.ts | 9 - packages/lodestar/test/utils/attestation.ts | 4 +- packages/lodestar/test/utils/balances.ts | 9 - packages/lodestar/test/utils/block.ts | 11 +- .../lodestar/test/utils/cachedBeaconState.ts | 14 + .../test/utils/contributionAndProof.ts | 4 +- packages/lodestar/test/utils/errors.ts | 3 +- .../lodestar/test/utils/mocks/chain/chain.ts | 29 +- packages/lodestar/test/utils/network.ts | 15 +- packages/lodestar/test/utils/node/beacon.ts | 6 +- packages/lodestar/test/utils/node/simTest.ts | 6 +- packages/lodestar/test/utils/render.ts | 5 + packages/lodestar/test/utils/state.ts | 78 +-- packages/lodestar/test/utils/stub/index.ts | 11 +- .../test/utils/validationData/attestation.ts | 11 +- packages/lodestar/test/utils/validator.ts | 6 +- packages/spec-test-util/package.json | 1 - packages/spec-test-util/src/index.ts | 3 +- packages/spec-test-util/src/multi.ts | 88 --- packages/spec-test-util/src/single.ts | 51 +- packages/spec-test-util/src/sszGeneric.ts | 72 ++- packages/spec-test-util/src/transform.ts | 35 -- .../test/e2e/_test_files/multi/bulk.yml | 14 - .../test/e2e/multi/index.test.ts | 16 - .../test/e2e/single/index.test.ts | 33 +- packages/types/README.md | 10 +- packages/types/package.json | 2 +- packages/types/src/allForks/sszTypes.ts | 34 +- packages/types/src/allForks/types.ts | 52 +- packages/types/src/altair/sszTypes.ts | 258 +++----- packages/types/src/altair/types.ts | 145 +---- packages/types/src/bellatrix/sszTypes.ts | 199 ++---- packages/types/src/bellatrix/types.ts | 63 +- packages/types/src/phase0/sszTypes.ts | 582 +++++++----------- packages/types/src/phase0/types.ts | 376 ++--------- packages/types/src/primitive/sszTypes.ts | 63 +- packages/types/src/primitive/types.ts | 46 +- packages/types/src/utils/StringType.ts | 56 +- packages/types/src/utils/lazyVar.ts | 12 - packages/types/test/unit/constants.test.ts | 9 +- packages/types/test/unit/ssz.test.ts | 38 +- packages/utils/src/bytes.ts | 2 +- packages/utils/test/unit/json.test.ts | 4 +- packages/validator/package.json | 2 +- .../src/repositories/metaDataRepository.ts | 15 +- packages/validator/src/services/utils.ts | 4 - .../validator/src/services/validatorStore.ts | 7 +- .../attestationByTargetRepository.ts | 18 +- .../attestationLowerBoundRepository.ts | 15 +- .../block/blockBySlotRepository.ts | 15 +- .../validator/src/slashingProtection/utils.ts | 6 +- packages/validator/src/util/clock.ts | 4 +- packages/validator/src/validator.ts | 2 +- .../unit/services/attestationDuties.test.ts | 2 +- .../unit/services/syncCommitteDuties.test.ts | 2 +- .../test/unit/services/syncCommittee.test.ts | 6 +- .../test/unit/services/utils.test.ts | 9 - .../interchange/index.test.ts | 2 +- .../minMaxSurround/utils.ts | 2 +- 408 files changed, 4821 insertions(+), 6378 deletions(-) delete mode 100644 packages/beacon-state-transition/src/cache/balanceList.ts delete mode 100644 packages/beacon-state-transition/src/cache/cachedBeaconState.ts delete mode 100644 packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts delete mode 100644 packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts create mode 100644 packages/beacon-state-transition/src/cache/stateCache.ts create mode 100644 packages/beacon-state-transition/src/cache/types.ts delete mode 100644 packages/beacon-state-transition/src/util/aggregationBits.ts create mode 100644 packages/beacon-state-transition/src/util/altair.ts delete mode 100644 packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts create mode 100644 packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts create mode 100644 packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts delete mode 100644 packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts create mode 100644 packages/beacon-state-transition/test/unit/util/flags.test.ts delete mode 100644 packages/beacon-state-transition/test/unit/util/interface.test.ts delete mode 100644 packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts delete mode 100644 packages/config/src/chainConfig/sszTypes.ts create mode 100644 packages/light-client/test/types.ts delete mode 100644 packages/lodestar/src/util/tree.ts create mode 100644 packages/lodestar/test/e2e/interop/genesisState.test.ts delete mode 100644 packages/lodestar/test/spec/ssz/util/specTypes/uint.ts create mode 100644 packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts create mode 100644 packages/lodestar/test/spec/utils/runValidSszTest.ts delete mode 100644 packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts delete mode 100644 packages/lodestar/test/spec/utils/specTestTypes/genesis.ts delete mode 100644 packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts delete mode 100644 packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts create mode 100644 packages/lodestar/test/spec/utils/sszTestCaseParser.ts delete mode 100644 packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts delete mode 100644 packages/lodestar/test/utils/aggregationBits.ts delete mode 100644 packages/lodestar/test/utils/balances.ts create mode 100644 packages/lodestar/test/utils/cachedBeaconState.ts create mode 100644 packages/lodestar/test/utils/render.ts delete mode 100644 packages/spec-test-util/src/multi.ts delete mode 100644 packages/spec-test-util/src/transform.ts delete mode 100644 packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml delete mode 100644 packages/spec-test-util/test/e2e/multi/index.test.ts delete mode 100644 packages/types/src/utils/lazyVar.ts delete mode 100644 packages/validator/test/unit/services/utils.test.ts diff --git a/package.json b/package.json index 4ee9466385d9..d39617f3397d 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,8 @@ "lint": "lerna run lint --no-bail", "check-types": "lerna run check-types --no-bail", "coverage": "lerna run coverage --no-bail", - "test:unit": "lerna run test:unit --no-bail", - "test:e2e": "lerna run test:e2e --no-bail", + "test:unit": "lerna run test:unit --no-bail --concurrency 1", + "test:e2e": "lerna run test:e2e --no-bail --concurrency 1", "test:e2e:sim": "lerna run test:e2e:sim --no-bail", "test:spec-min": "lerna run test:spec-min --no-bail", "test:spec-fast": "lerna run test:spec-fast --no-bail", diff --git a/packages/api/package.json b/packages/api/package.json index 19a7d6aa4fe1..99cb5f75f9bd 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,8 +40,8 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "cross-fetch": "^3.1.4", "eventsource": "^1.1.0", "qs": "^6.10.1" diff --git a/packages/api/src/client/utils/client.ts b/packages/api/src/client/utils/client.ts index f3d2efbc081f..3069547bd03e 100644 --- a/packages/api/src/client/utils/client.ts +++ b/packages/api/src/client/utils/client.ts @@ -1,4 +1,3 @@ -import {Json} from "@chainsafe/ssz"; import {mapValues} from "@chainsafe/lodestar-utils"; import {FetchOpts, IHttpClient} from "./httpClient"; import {compileRouteUrlFormater} from "../../utils/urlFormat"; @@ -8,7 +7,6 @@ import { RouteGeneric, ReturnTypes, TypeJson, - jsonOpts, ReqSerializer, ReqSerializers, RoutesData, @@ -69,10 +67,10 @@ export function generateGenericJsonClient< const returnType = returnTypes[routeKey as keyof ReturnTypes] as TypeJson | null; return async function request(...args: Parameters): Promise { - const res = await fetchFn.json(fetchOptsSerializer(...args)); + const res = await fetchFn.json(fetchOptsSerializer(...args)); if (returnType) { // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return returnType.fromJson(res, jsonOpts) as ReturnType; + return returnType.fromJson(res) as ReturnType; } }; }) as Api; diff --git a/packages/api/src/routes/beacon/block.ts b/packages/api/src/routes/beacon/block.ts index 48f0bf738733..b71afda8a87f 100644 --- a/packages/api/src/routes/beacon/block.ts +++ b/packages/api/src/routes/beacon/block.ts @@ -1,4 +1,4 @@ -import {ContainerType, Json} from "@chainsafe/ssz"; +import {ContainerType} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {phase0, allForks, Slot, Root, ssz} from "@chainsafe/lodestar-types"; @@ -114,7 +114,7 @@ export type ReqTypes = { getBlockHeader: BlockIdOnlyReq; getBlockHeaders: {query: {slot?: number; parent_root?: string}}; getBlockRoot: BlockIdOnlyReq; - publishBlock: {body: Json}; + publishBlock: {body: unknown}; }; export function getReqSerializers(config: IChainForkConfig): ReqSerializers { @@ -125,12 +125,12 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers => + const getSignedBeaconBlockType = (data: allForks.SignedBeaconBlock): allForks.AllForksSSZTypes["SignedBeaconBlock"] => config.getForkTypes(data.message.slot).SignedBeaconBlock; + const AllForksSignedBeaconBlock: TypeJson = { - toJson: (data, opts) => getSignedBeaconBlockType(data).toJson(data, opts), - fromJson: (data, opts) => - getSignedBeaconBlockType((data as unknown) as allForks.SignedBeaconBlock).fromJson(data, opts), + toJson: (data) => getSignedBeaconBlockType(data).toJson(data), + fromJson: (data) => getSignedBeaconBlockType((data as unknown) as allForks.SignedBeaconBlock).fromJson(data), }; return { @@ -149,14 +149,10 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers { - const BeaconHeaderResType = new ContainerType({ - fields: { - root: ssz.Root, - canonical: ssz.Boolean, - header: ssz.phase0.SignedBeaconBlockHeader, - }, - //from beacon apis - expectedCase: "notransform", + const BeaconHeaderResType = new ContainerType({ + root: ssz.Root, + canonical: ssz.Boolean, + header: ssz.phase0.SignedBeaconBlockHeader, }); return { diff --git a/packages/api/src/routes/beacon/pool.ts b/packages/api/src/routes/beacon/pool.ts index fbe8e16cccfd..3cf8c787b495 100644 --- a/packages/api/src/routes/beacon/pool.ts +++ b/packages/api/src/routes/beacon/pool.ts @@ -1,5 +1,4 @@ import {phase0, altair, CommitteeIndex, Slot, ssz} from "@chainsafe/lodestar-types"; -import {Json} from "@chainsafe/ssz"; import { RoutesData, ReturnTypes, @@ -122,11 +121,11 @@ export type ReqTypes = { getPoolAttesterSlashings: ReqEmpty; getPoolProposerSlashings: ReqEmpty; getPoolVoluntaryExits: ReqEmpty; - submitPoolAttestations: {body: Json}; - submitPoolAttesterSlashing: {body: Json}; - submitPoolProposerSlashing: {body: Json}; - submitPoolVoluntaryExit: {body: Json}; - submitPoolSyncCommitteeSignatures: {body: Json}; + submitPoolAttestations: {body: unknown}; + submitPoolAttesterSlashing: {body: unknown}; + submitPoolProposerSlashing: {body: unknown}; + submitPoolVoluntaryExit: {body: unknown}; + submitPoolSyncCommitteeSignatures: {body: unknown}; }; export function getReqSerializers(): ReqSerializers { diff --git a/packages/api/src/routes/beacon/state.ts b/packages/api/src/routes/beacon/state.ts index af376b0b68fb..6f26435231b3 100644 --- a/packages/api/src/routes/beacon/state.ts +++ b/packages/api/src/routes/beacon/state.ts @@ -217,54 +217,49 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { - const FinalityCheckpoints = new ContainerType({ - fields: { + const FinalityCheckpoints = new ContainerType( + { previousJustified: ssz.phase0.Checkpoint, currentJustified: ssz.phase0.Checkpoint, finalized: ssz.phase0.Checkpoint, }, - // From beacon apis - casingMap: { - previousJustified: "previous_justified", - currentJustified: "current_justified", - finalized: "finalized", - }, - }); + {jsonCase: "eth2"} + ); - const ValidatorResponse = new ContainerType({ - fields: { + const ValidatorResponse = new ContainerType( + { index: ssz.ValidatorIndex, - balance: ssz.Number64, + balance: ssz.UintNum64, status: new StringType(), validator: ssz.phase0.Validator, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); - const ValidatorBalance = new ContainerType({ - fields: { + const ValidatorBalance = new ContainerType( + { index: ssz.ValidatorIndex, - balance: ssz.Number64, + balance: ssz.UintNum64, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); - const EpochCommitteeResponse = new ContainerType({ - fields: { + const EpochCommitteeResponse = new ContainerType( + { index: ssz.CommitteeIndex, slot: ssz.Slot, validators: ssz.phase0.CommitteeIndices, }, - }); + {jsonCase: "eth2"} + ); - const EpochSyncCommitteesResponse = new ContainerType({ - fields: { + const EpochSyncCommitteesResponse = new ContainerType( + { validators: ArrayOf(ssz.ValidatorIndex), validatorAggregates: ArrayOf(ssz.ValidatorIndex), }, - }); + {jsonCase: "eth2"} + ); return { getStateRoot: ContainerData(ssz.Root), diff --git a/packages/api/src/routes/config.ts b/packages/api/src/routes/config.ts index 6285073621f7..2d5c58d0dbdc 100644 --- a/packages/api/src/routes/config.ts +++ b/packages/api/src/routes/config.ts @@ -1,6 +1,6 @@ import {BeaconPreset} from "@chainsafe/lodestar-params"; import {IChainConfig} from "@chainsafe/lodestar-config"; -import {Bytes32, Number64, phase0, ssz} from "@chainsafe/lodestar-types"; +import {Bytes32, UintNum64, phase0, ssz} from "@chainsafe/lodestar-types"; import {mapValues} from "@chainsafe/lodestar-utils"; import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; import {ArrayOf, ContainerData, ReqEmpty, reqEmpty, ReturnTypes, ReqSerializers, RoutesData, sameType} from "../utils"; @@ -8,7 +8,7 @@ import {ArrayOf, ContainerData, ReqEmpty, reqEmpty, ReturnTypes, ReqSerializers, // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes export type DepositContract = { - chainId: Number64; + chainId: UintNum64; address: Bytes32; }; @@ -57,17 +57,13 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { - const DepositContract = new ContainerType({ - fields: { - chainId: ssz.Number64, - address: new ByteVectorType({length: 20}), + const DepositContract = new ContainerType( + { + chainId: ssz.UintNum64, + address: new ByteVectorType(20), }, - // From beacon apis - casingMap: { - chainId: "chain_id", - address: "address", - }, - }); + {jsonCase: "eth2"} + ); return { getDepositContract: ContainerData(DepositContract), diff --git a/packages/api/src/routes/debug.ts b/packages/api/src/routes/debug.ts index ae9783329dab..e4d8748e9b20 100644 --- a/packages/api/src/routes/debug.ts +++ b/packages/api/src/routes/debug.ts @@ -115,14 +115,13 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { const stringType = new StringType(); - const SlotRoot = new ContainerType({ - fields: { + const SlotRoot = new ContainerType( + { slot: ssz.Slot, root: stringType, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); return { getHeads: ContainerData(ArrayOf(SlotRoot)), diff --git a/packages/api/src/routes/events.ts b/packages/api/src/routes/events.ts index ac4e58511d31..07b7ba02b4b9 100644 --- a/packages/api/src/routes/events.ts +++ b/packages/api/src/routes/events.ts @@ -1,6 +1,6 @@ -import {Epoch, Number64, phase0, Slot, ssz, StringType, RootHex, altair} from "@chainsafe/lodestar-types"; -import {ContainerType, Json, Type} from "@chainsafe/ssz"; -import {jsonOpts, RouteDef, TypeJson} from "../utils"; +import {Epoch, phase0, Slot, ssz, StringType, RootHex, altair, UintNum64} from "@chainsafe/lodestar-types"; +import {ContainerType, Type} from "@chainsafe/ssz"; +import {RouteDef, TypeJson} from "../utils"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -46,7 +46,7 @@ export type EventData = { [EventType.finalizedCheckpoint]: {block: RootHex; state: RootHex; epoch: Epoch}; [EventType.chainReorg]: { slot: Slot; - depth: Number64; + depth: UintNum64; oldHeadBlock: RootHex; newHeadBlock: RootHex; oldHeadState: RootHex; @@ -87,8 +87,8 @@ export type ReqTypes = { export function getTypeByEvent(): {[K in EventType]: Type} { const stringType = new StringType(); return { - [EventType.head]: new ContainerType({ - fields: { + [EventType.head]: new ContainerType( + { slot: ssz.Slot, block: stringType, state: stringType, @@ -96,71 +96,49 @@ export function getTypeByEvent(): {[K in EventType]: Type} { previousDutyDependentRoot: stringType, currentDutyDependentRoot: stringType, }, - // From beacon apis eventstream - casingMap: { - slot: "slot", - block: "block", - state: "state", - epochTransition: "epoch_transition", - previousDutyDependentRoot: "previous_duty_dependent_root", - currentDutyDependentRoot: "current_duty_dependent_root", - }, - }), + {jsonCase: "eth2"} + ), - [EventType.block]: new ContainerType({ - fields: { + [EventType.block]: new ContainerType( + { slot: ssz.Slot, block: stringType, }, - // From beacon apis eventstream - expectedCase: "notransform", - }), + {jsonCase: "eth2"} + ), [EventType.attestation]: ssz.phase0.Attestation, [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, - [EventType.finalizedCheckpoint]: new ContainerType({ - fields: { + [EventType.finalizedCheckpoint]: new ContainerType( + { block: stringType, state: stringType, epoch: ssz.Epoch, }, - // From beacon apis eventstream - expectedCase: "notransform", - }), + {jsonCase: "eth2"} + ), - [EventType.chainReorg]: new ContainerType({ - fields: { + [EventType.chainReorg]: new ContainerType( + { slot: ssz.Slot, - depth: ssz.Number64, + depth: ssz.UintNum64, oldHeadBlock: stringType, newHeadBlock: stringType, oldHeadState: stringType, newHeadState: stringType, epoch: ssz.Epoch, }, - // From beacon apis eventstream - casingMap: { - slot: "slot", - depth: "depth", - oldHeadBlock: "old_head_block", - newHeadBlock: "new_head_block", - oldHeadState: "old_head_state", - newHeadState: "new_head_state", - epoch: "epoch", - }, - }), + {jsonCase: "eth2"} + ), - [EventType.lightclientHeaderUpdate]: new ContainerType({ - fields: { + [EventType.lightclientHeaderUpdate]: new ContainerType( + { syncAggregate: ssz.altair.SyncAggregate, attestedHeader: ssz.phase0.BeaconBlockHeader, }, - casingMap: { - syncAggregate: "sync_aggregate", - attestedHeader: "attested_header", - }, - }), + {jsonCase: "eth2"} + ), }; } @@ -169,13 +147,13 @@ export function getEventSerdes() { const typeByEvent = getTypeByEvent(); return { - toJson: (event: BeaconEvent): Json => { + toJson: (event: BeaconEvent): unknown => { const eventType = typeByEvent[event.type] as TypeJson; - return eventType.toJson(event.message, jsonOpts); + return eventType.toJson(event.message); }, - fromJson: (type: EventType, data: Json): BeaconEvent["message"] => { + fromJson: (type: EventType, data: unknown): BeaconEvent["message"] => { const eventType = typeByEvent[type] as TypeJson; - return eventType.fromJson(data, jsonOpts); + return eventType.fromJson(data); }, }; } diff --git a/packages/api/src/routes/lightclient.ts b/packages/api/src/routes/lightclient.ts index 39efffb7f907..52d292f3c644 100644 --- a/packages/api/src/routes/lightclient.ts +++ b/packages/api/src/routes/lightclient.ts @@ -1,4 +1,4 @@ -import {ContainerType, Path, VectorType} from "@chainsafe/ssz"; +import {ContainerType, JsonPath, VectorCompositeType} from "@chainsafe/ssz"; import {Proof} from "@chainsafe/persistent-merkle-tree"; import {altair, phase0, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; import { @@ -29,10 +29,10 @@ export type LightclientSnapshotWithProof = { export type Api = { /** - * Returns a multiproof of `paths` at the requested `stateId`. + * Returns a multiproof of `jsonPaths` at the requested `stateId`. * The requested `stateId` may not be available. Regular nodes only keep recent states in memory. */ - getStateProof(stateId: string, paths: Path[]): Promise<{data: Proof}>; + getStateProof(stateId: string, jsonPaths: JsonPath[]): Promise<{data: Proof}>; /** * Returns an array of best updates in the requested periods within the inclusive range `from` - `to`. * Best is defined by (in order of priority): @@ -96,30 +96,22 @@ export function getReqSerializers(): ReqSerializers { } export function getReturnTypes(): ReturnTypes { - const lightclientSnapshotWithProofType = new ContainerType({ - fields: { + const lightclientSnapshotWithProofType = new ContainerType( + { header: ssz.phase0.BeaconBlockHeader, currentSyncCommittee: ssz.altair.SyncCommittee, - currentSyncCommitteeBranch: new VectorType({elementType: ssz.Root, length: 5}), + currentSyncCommitteeBranch: new VectorCompositeType(ssz.Root, 5), }, - // Custom type, not in the consensus specs - casingMap: { - header: "header", - currentSyncCommittee: "current_sync_committee", - currentSyncCommitteeBranch: "current_sync_committee_branch", - }, - }); + {jsonCase: "eth2"} + ); - const lightclientHeaderUpdate = new ContainerType({ - fields: { + const lightclientHeaderUpdate = new ContainerType( + { syncAggregate: ssz.altair.SyncAggregate, attestedHeader: ssz.phase0.BeaconBlockHeader, }, - casingMap: { - syncAggregate: "sync_aggregate", - attestedHeader: "attested_header", - }, - }); + {jsonCase: "eth2"} + ); return { // Just sent the proof JSON as-is diff --git a/packages/api/src/routes/lodestar.ts b/packages/api/src/routes/lodestar.ts index 8e49cf72a4fb..d0673a6b2af6 100644 --- a/packages/api/src/routes/lodestar.ts +++ b/packages/api/src/routes/lodestar.ts @@ -1,5 +1,5 @@ import {Epoch, RootHex, Slot, ssz, StringType} from "@chainsafe/lodestar-types"; -import {ByteVectorType, ContainerType, Json} from "@chainsafe/ssz"; +import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; import { jsonType, ReqEmpty, @@ -34,7 +34,7 @@ export type GossipQueueItem = { export type RegenQueueItem = { key: string; - args: Json; + args: unknown; addedTimeMs: number; }; @@ -161,32 +161,26 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { const stringType = new StringType(); - const GossipQueueItem = new ContainerType({ - fields: { + const GossipQueueItem = new ContainerType( + { topic: stringType, receivedFrom: stringType, - data: new ByteVectorType({length: 256}), + data: new ByteVectorType(256), addedTimeMs: ssz.Slot, }, - // Custom type, not in the consensus specs - casingMap: { - topic: "topic", - receivedFrom: "received_from", - data: "data", - addedTimeMs: "added_time_ms", - }, - }); + {jsonCase: "eth2"} + ); return { getWtfNode: sameType(), writeHeapdump: sameType(), getLatestWeakSubjectivityCheckpointEpoch: sameType(), - getSyncChainsDebugState: jsonType(), + getSyncChainsDebugState: jsonType("camel"), getGossipQueueItems: ArrayOf(GossipQueueItem), - getRegenQueueItems: jsonType(), - getBlockProcessorQueueItems: jsonType(), - getStateCacheItems: jsonType(), - getCheckpointStateCacheItems: jsonType(), - discv5GetKadValues: jsonType(), + getRegenQueueItems: jsonType("camel"), + getBlockProcessorQueueItems: jsonType("camel"), + getStateCacheItems: jsonType("camel"), + getCheckpointStateCacheItems: jsonType("camel"), + discv5GetKadValues: jsonType("camel"), }; } diff --git a/packages/api/src/routes/node.ts b/packages/api/src/routes/node.ts index 68eb53b26ea3..47c719fc45af 100644 --- a/packages/api/src/routes/node.ts +++ b/packages/api/src/routes/node.ts @@ -162,23 +162,16 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { const stringType = new StringType(); - const NetworkIdentity = new ContainerType({ - fields: { + const NetworkIdentity = new ContainerType( + { peerId: stringType, enr: stringType, p2pAddresses: ArrayOf(stringType), discoveryAddresses: ArrayOf(stringType), metadata: ssz.altair.Metadata, }, - // From beacon apis - casingMap: { - peerId: "peer_id", - enr: "enr", - p2pAddresses: "p2p_addresses", - discoveryAddresses: "discovery_addresses", - metadata: "metadata", - }, - }); + {jsonCase: "eth2"} + ); return { // @@ -187,11 +180,11 @@ export function getReturnTypes(): ReturnTypes { getNetworkIdentity: ContainerData(NetworkIdentity), // All these types don't contain any BigInt nor Buffer instances. // Use jsonType() to translate the casing in a generic way. - getPeers: jsonType(), - getPeer: jsonType(), - getPeerCount: jsonType(), - getNodeVersion: jsonType(), - getSyncingStatus: jsonType(), + getPeers: jsonType("camel"), + getPeer: jsonType("camel"), + getPeerCount: jsonType("camel"), + getNodeVersion: jsonType("camel"), + getSyncingStatus: jsonType("camel"), getHealth: sameType(), }; } diff --git a/packages/api/src/routes/validator.ts b/packages/api/src/routes/validator.ts index 73f7b81ec1e0..c5ef25d3518e 100644 --- a/packages/api/src/routes/validator.ts +++ b/packages/api/src/routes/validator.ts @@ -1,4 +1,4 @@ -import {ContainerType, fromHexString, Json, toHexString, Type} from "@chainsafe/ssz"; +import {ContainerType, fromHexString, toHexString, Type} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import { allForks, @@ -7,11 +7,11 @@ import { BLSSignature, CommitteeIndex, Epoch, - Number64, phase0, Root, Slot, ssz, + UintNum64, ValidatorIndex, } from "@chainsafe/lodestar-types"; import { @@ -57,11 +57,11 @@ export type AttesterDuty = { validatorIndex: ValidatorIndex; committeeIndex: CommitteeIndex; // Number of validators in committee - committeeLength: Number64; + committeeLength: UintNum64; // Number of committees at the provided slot - committeesAtSlot: Number64; + committeesAtSlot: UintNum64; // Index of validator in committee - validatorCommitteeIndex: Number64; + validatorCommitteeIndex: UintNum64; // The slot at which the validator must attest. slot: Slot; }; @@ -227,44 +227,32 @@ export type ReqTypes = { produceAttestationData: {query: {slot: number; committee_index: number}}; produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}}; getAggregatedAttestation: {query: {attestation_data_root: string; slot: number}}; - publishAggregateAndProofs: {body: Json}; - publishContributionAndProofs: {body: Json}; - prepareBeaconCommitteeSubnet: {body: Json}; - prepareSyncCommitteeSubnets: {body: Json}; + publishAggregateAndProofs: {body: unknown}; + publishContributionAndProofs: {body: unknown}; + prepareBeaconCommitteeSubnet: {body: unknown}; + prepareSyncCommitteeSubnets: {body: unknown}; }; export function getReqSerializers(): ReqSerializers { - const BeaconCommitteeSubscription = new ContainerType({ - fields: { + const BeaconCommitteeSubscription = new ContainerType( + { validatorIndex: ssz.ValidatorIndex, committeeIndex: ssz.CommitteeIndex, committeesAtSlot: ssz.Slot, slot: ssz.Slot, isAggregator: ssz.Boolean, }, - // From beacon apis - casingMap: { - validatorIndex: "validator_index", - committeeIndex: "committee_index", - committeesAtSlot: "committees_at_slot", - slot: "slot", - isAggregator: "is_aggregator", - }, - }); + {jsonCase: "eth2"} + ); - const SyncCommitteeSubscription = new ContainerType({ - fields: { + const SyncCommitteeSubscription = new ContainerType( + { validatorIndex: ssz.ValidatorIndex, syncCommitteeIndices: ArrayOf(ssz.CommitteeIndex), untilEpoch: ssz.Epoch, }, - // From beacon apis - casingMap: { - validatorIndex: "validator_index", - syncCommitteeIndices: "sync_committee_indices", - untilEpoch: "until_epoch", - }, - }); + {jsonCase: "eth2"} + ); const produceBlock: ReqSerializers["produceBlock"] = { writeReq: (slot, randaoReveal, grafitti) => ({ @@ -346,58 +334,39 @@ export function getReqSerializers(): ReqSerializers { } export function getReturnTypes(): ReturnTypes { - const WithDependentRoot = (dataType: Type): ContainerType<{data: T; dependentRoot: Root}> => - new ContainerType({fields: {data: dataType, dependentRoot: ssz.Root}}); + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + const WithDependentRoot = (dataType: Type) => new ContainerType({data: dataType, dependentRoot: ssz.Root}); - const AttesterDuty = new ContainerType({ - fields: { + const AttesterDuty = new ContainerType( + { pubkey: ssz.BLSPubkey, validatorIndex: ssz.ValidatorIndex, committeeIndex: ssz.CommitteeIndex, - committeeLength: ssz.Number64, - committeesAtSlot: ssz.Number64, - validatorCommitteeIndex: ssz.Number64, + committeeLength: ssz.UintNum64, + committeesAtSlot: ssz.UintNum64, + validatorCommitteeIndex: ssz.UintNum64, slot: ssz.Slot, }, - // From beacon apis - casingMap: { - pubkey: "pubkey", - validatorIndex: "validator_index", - committeeIndex: "committee_index", - committeeLength: "committee_length", - committeesAtSlot: "committees_at_slot", - validatorCommitteeIndex: "validator_committee_index", - slot: "slot", - }, - }); + {jsonCase: "eth2"} + ); - const ProposerDuty = new ContainerType({ - fields: { + const ProposerDuty = new ContainerType( + { slot: ssz.Slot, validatorIndex: ssz.ValidatorIndex, pubkey: ssz.BLSPubkey, }, - // From beacon apis - casingMap: { - slot: "slot", - validatorIndex: "validator_index", - pubkey: "pubkey", - }, - }); + {jsonCase: "eth2"} + ); - const SyncDuty = new ContainerType({ - fields: { + const SyncDuty = new ContainerType( + { pubkey: ssz.BLSPubkey, validatorIndex: ssz.ValidatorIndex, - validatorSyncCommitteeIndices: ArrayOf(ssz.Number64), - }, - // From beacon apis - casingMap: { - pubkey: "pubkey", - validatorIndex: "validator_index", - validatorSyncCommitteeIndices: "validator_sync_committee_indices", + validatorSyncCommitteeIndices: ArrayOf(ssz.UintNum64), }, - }); + {jsonCase: "eth2"} + ); return { getAttesterDuties: WithDependentRoot(ArrayOf(AttesterDuty)), diff --git a/packages/api/src/server/debug.ts b/packages/api/src/server/debug.ts index 619d07741cc4..2388ea557103 100644 --- a/packages/api/src/server/debug.ts +++ b/packages/api/src/server/debug.ts @@ -1,5 +1,4 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {jsonOpts} from "../utils"; import {ServerRoutes, getGenericJsonServer} from "./utils"; import {Api, ReqTypes, routesData, getReturnTypes, getReqSerializers} from "../routes/debug"; @@ -25,7 +24,7 @@ export function getRoutes(config: IChainForkConfig, api: Api): ServerRoutes { + handler: async function handler(req: ReqGeneric): Promise { const args: any[] = routeSerdes.parseReq(req as ReqTypes[keyof Api]); const data = (await api[routeKey](...args)) as Resolves; if (returnType) { - return returnType.toJson(data, jsonOpts); + return returnType.toJson(data); } else { return {}; } diff --git a/packages/api/src/utils/serdes.ts b/packages/api/src/utils/serdes.ts index 0f35f11feca6..c7c419eac4c6 100644 --- a/packages/api/src/utils/serdes.ts +++ b/packages/api/src/utils/serdes.ts @@ -1,11 +1,11 @@ -import {Path} from "@chainsafe/ssz"; +import {JsonPath} from "@chainsafe/ssz"; /** * Serialize proof path to JSON. * @param paths `[["finalized_checkpoint", 0, "root", 12000]]` * @returns `['["finalized_checkpoint",0,"root",12000]']` */ -export function querySerializeProofPathsArr(paths: Path[]): string[] { +export function querySerializeProofPathsArr(paths: JsonPath[]): string[] { return paths.map((path) => JSON.stringify(path)); } @@ -14,11 +14,11 @@ export function querySerializeProofPathsArr(paths: Path[]): string[] { * @param pathStrs `['["finalized_checkpoint",0,"root",12000]']` * @returns `[["finalized_checkpoint", 0, "root", 12000]]` */ -export function queryParseProofPathsArr(pathStrs: string | string[]): Path[] { +export function queryParseProofPathsArr(pathStrs: string | string[]): JsonPath[] { if (Array.isArray(pathStrs)) { return pathStrs.map((pathStr) => queryParseProofPaths(pathStr)); } else { - return [queryParseProofPaths(pathStrs) as Path]; + return [queryParseProofPaths(pathStrs) as JsonPath]; } } @@ -27,8 +27,8 @@ export function queryParseProofPathsArr(pathStrs: string | string[]): Path[] { * @param pathStr `'["finalized_checkpoint",0,"root",12000]'` * @returns `["finalized_checkpoint", 0, "root", 12000]` */ -export function queryParseProofPaths(pathStr: string): Path { - const path = JSON.parse(pathStr) as Path; +export function queryParseProofPaths(pathStr: string): JsonPath { + const path = JSON.parse(pathStr) as JsonPath; if (!Array.isArray(path)) { throw Error("Proof pathStr is not an array"); diff --git a/packages/api/src/utils/types.ts b/packages/api/src/utils/types.ts index 0a54fca39c9b..3e91c8037933 100644 --- a/packages/api/src/utils/types.ts +++ b/packages/api/src/utils/types.ts @@ -1,4 +1,4 @@ -import {IJsonOptions, Json, ListType, Type} from "@chainsafe/ssz"; +import {isBasicType, ListBasicType, Type, isCompositeType, ListCompositeType, ArrayType} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {objectToExpectedCase} from "@chainsafe/lodestar-utils"; @@ -8,10 +8,8 @@ import {Schema, SchemaDefinition} from "./schema"; /* eslint-disable @typescript-eslint/naming-convention, @typescript-eslint/no-explicit-any */ -/** All JSON must be sent in snake case */ -export const jsonOpts = {case: "snake" as const}; /** All JSON inside the JS code must be camel case */ -export const codeCase = "camel" as const; +const codeCase = "camel" as const; export type RouteGroupDefinition< Api extends Record, @@ -42,8 +40,8 @@ type ThenArg = T extends PromiseLike ? U : T; export type Resolves any> = ThenArg>; export type TypeJson = { - toJson(val: T, opts?: IJsonOptions): Json; - fromJson(json: Json, opts?: IJsonOptions): T; + toJson(val: T): unknown; + fromJson(json: unknown): T; }; // @@ -94,15 +92,21 @@ export const reqEmpty: ReqSerializer<() => void, ReqEmpty> = { export const reqOnlyBody = ( type: TypeJson, bodySchema: Schema -): ReqGenArg<(arg: T) => Promise, {body: Json}> => ({ - writeReq: (items) => ({body: type.toJson(items, jsonOpts)}), - parseReq: ({body}) => [type.fromJson(body, jsonOpts)], +): ReqGenArg<(arg: T) => Promise, {body: unknown}> => ({ + writeReq: (items) => ({body: type.toJson(items)}), + parseReq: ({body}) => [type.fromJson(body)], schema: {body: bodySchema}, }); /** SSZ factory helper + typed. limit = 1e6 as a big enough random number */ -export function ArrayOf(elementType: Type, limit = 1e6): ListType { - return new ListType({elementType, limit}); +export function ArrayOf(elementType: Type): ArrayType, unknown, unknown> { + if (isCompositeType(elementType)) { + return (new ListCompositeType(elementType, Infinity) as unknown) as ArrayType, unknown, unknown>; + } else if (isBasicType(elementType)) { + return (new ListBasicType(elementType, Infinity) as unknown) as ArrayType, unknown, unknown>; + } else { + throw Error(`Unknown type ${elementType.typeName}`); + } } /** @@ -113,12 +117,12 @@ export function ArrayOf(elementType: Type, limit = 1e6): ListType { */ export function ContainerData(dataType: TypeJson): TypeJson<{data: T}> { return { - toJson: ({data}, opts) => ({ - data: dataType.toJson(data, opts), + toJson: ({data}) => ({ + data: dataType.toJson(data), }), - fromJson: ({data}: {data: Json}, opts) => { + fromJson: ({data}: {data: unknown}) => { return { - data: dataType.fromJson(data, opts), + data: dataType.fromJson(data), }; }, }; @@ -133,26 +137,30 @@ export function ContainerData(dataType: TypeJson): TypeJson<{data: T}> { */ export function WithVersion(getType: (fork: ForkName) => TypeJson): TypeJson<{data: T; version: ForkName}> { return { - toJson: ({data, version}, opts) => ({ - data: getType(version || ForkName.phase0).toJson(data, opts), + toJson: ({data, version}) => ({ + data: getType(version || ForkName.phase0).toJson(data), version, }), - fromJson: ({data, version}: {data: Json; version: string}, opts) => { + fromJson: ({data, version}: {data: unknown; version: string}) => { // Un-safe external data, validate version is known ForkName value if (!ForkName[version as ForkName]) throw Error(`Invalid version ${version}`); return { - data: getType(version as ForkName).fromJson(data, opts), + data: getType(version as ForkName).fromJson(data), version: version as ForkName, }; }, }; } +type JsonCase = "snake" | "constant" | "camel" | "param" | "header" | "pascal" | "dot" | "notransform"; + /** Helper to only translate casing */ -export function jsonType | Record[]>(): TypeJson { +export function jsonType | Record[]>( + jsonCase: JsonCase +): TypeJson { return { - toJson: (val, opts) => objectToExpectedCase(val, opts?.case) as Json, + toJson: (val) => objectToExpectedCase(val, jsonCase) as unknown, fromJson: (json) => objectToExpectedCase(json as Record, codeCase) as T, }; } @@ -160,7 +168,7 @@ export function jsonType | Record(): TypeJson { return { - toJson: (val) => (val as unknown) as Json, + toJson: (val) => val as unknown, fromJson: (json) => (json as unknown) as T, }; } diff --git a/packages/api/test/unit/beacon.test.ts b/packages/api/test/unit/beacon.test.ts index 81f1bf4e76fc..78658f27253e 100644 --- a/packages/api/test/unit/beacon.test.ts +++ b/packages/api/test/unit/beacon.test.ts @@ -15,14 +15,14 @@ describe("beacon", () => { const blockHeaderResponse: BlockHeaderResponse = { root, canonical: true, - header: ssz.phase0.SignedBeaconBlockHeader.defaultValue(), + header: ssz.phase0.SignedBeaconBlockHeader.defaultValue, }; const validatorResponse: ValidatorResponse = { index: 1, balance, status: "active_ongoing", - validator: ssz.phase0.Validator.defaultValue(), + validator: ssz.phase0.Validator.defaultValue, }; runGenericServerTest(config, getClient, getRoutes, { @@ -30,15 +30,15 @@ describe("beacon", () => { getBlock: { args: ["head"], - res: {data: ssz.phase0.SignedBeaconBlock.defaultValue()}, + res: {data: ssz.phase0.SignedBeaconBlock.defaultValue}, }, getBlockV2: { args: ["head"], - res: {data: ssz.altair.SignedBeaconBlock.defaultValue(), version: ForkName.altair}, + res: {data: ssz.altair.SignedBeaconBlock.defaultValue, version: ForkName.altair}, }, getBlockAttestations: { args: ["head"], - res: {data: [ssz.phase0.Attestation.defaultValue()]}, + res: {data: [ssz.phase0.Attestation.defaultValue]}, }, getBlockHeader: { args: ["head"], @@ -53,7 +53,7 @@ describe("beacon", () => { res: {data: root}, }, publishBlock: { - args: [ssz.phase0.SignedBeaconBlock.defaultValue()], + args: [ssz.phase0.SignedBeaconBlock.defaultValue], res: undefined, }, @@ -61,38 +61,38 @@ describe("beacon", () => { getPoolAttestations: { args: [{slot: 1, committeeIndex: 2}], - res: {data: [ssz.phase0.Attestation.defaultValue()]}, + res: {data: [ssz.phase0.Attestation.defaultValue]}, }, getPoolAttesterSlashings: { args: [], - res: {data: [ssz.phase0.AttesterSlashing.defaultValue()]}, + res: {data: [ssz.phase0.AttesterSlashing.defaultValue]}, }, getPoolProposerSlashings: { args: [], - res: {data: [ssz.phase0.ProposerSlashing.defaultValue()]}, + res: {data: [ssz.phase0.ProposerSlashing.defaultValue]}, }, getPoolVoluntaryExits: { args: [], - res: {data: [ssz.phase0.SignedVoluntaryExit.defaultValue()]}, + res: {data: [ssz.phase0.SignedVoluntaryExit.defaultValue]}, }, submitPoolAttestations: { - args: [[ssz.phase0.Attestation.defaultValue()]], + args: [[ssz.phase0.Attestation.defaultValue]], res: undefined, }, submitPoolAttesterSlashing: { - args: [ssz.phase0.AttesterSlashing.defaultValue()], + args: [ssz.phase0.AttesterSlashing.defaultValue], res: undefined, }, submitPoolProposerSlashing: { - args: [ssz.phase0.ProposerSlashing.defaultValue()], + args: [ssz.phase0.ProposerSlashing.defaultValue], res: undefined, }, submitPoolVoluntaryExit: { - args: [ssz.phase0.SignedVoluntaryExit.defaultValue()], + args: [ssz.phase0.SignedVoluntaryExit.defaultValue], res: undefined, }, submitPoolSyncCommitteeSignatures: { - args: [[ssz.altair.SyncCommitteeMessage.defaultValue()]], + args: [[ssz.altair.SyncCommitteeMessage.defaultValue]], res: undefined, }, @@ -104,15 +104,15 @@ describe("beacon", () => { }, getStateFork: { args: ["head"], - res: {data: ssz.phase0.Fork.defaultValue()}, + res: {data: ssz.phase0.Fork.defaultValue}, }, getStateFinalityCheckpoints: { args: ["head"], res: { data: { - previousJustified: ssz.phase0.Checkpoint.defaultValue(), - currentJustified: ssz.phase0.Checkpoint.defaultValue(), - finalized: ssz.phase0.Checkpoint.defaultValue(), + previousJustified: ssz.phase0.Checkpoint.defaultValue, + currentJustified: ssz.phase0.Checkpoint.defaultValue, + finalized: ssz.phase0.Checkpoint.defaultValue, }, }, }, @@ -141,7 +141,7 @@ describe("beacon", () => { getGenesis: { args: [], - res: {data: ssz.phase0.Genesis.defaultValue()}, + res: {data: ssz.phase0.Genesis.defaultValue}, }, }); diff --git a/packages/api/test/unit/config.test.ts b/packages/api/test/unit/config.test.ts index b77c8ffbdce1..bc876b077270 100644 --- a/packages/api/test/unit/config.test.ts +++ b/packages/api/test/unit/config.test.ts @@ -27,7 +27,7 @@ describe("config", () => { }, getForkSchedule: { args: [], - res: {data: [ssz.phase0.Fork.defaultValue()]}, + res: {data: [ssz.phase0.Fork.defaultValue]}, }, getSpec: { args: [], diff --git a/packages/api/test/unit/debug.test.ts b/packages/api/test/unit/debug.test.ts index e70071edf233..86b3682f2024 100644 --- a/packages/api/test/unit/debug.test.ts +++ b/packages/api/test/unit/debug.test.ts @@ -23,11 +23,11 @@ describe("debug", function () { }, getState: { args: ["head", "json"], - res: {data: ssz.phase0.BeaconState.defaultValue()}, + res: {data: ssz.phase0.BeaconState.defaultValue}, }, getStateV2: { args: ["head", "json"], - res: {data: ssz.altair.BeaconState.defaultValue(), version: ForkName.altair}, + res: {data: ssz.altair.BeaconState.defaultValue, version: ForkName.altair}, }, connectToPeer: { args: ["peerId", ["multiaddr1", "multiaddr2"]], @@ -49,7 +49,7 @@ describe("debug", function () { for (const method of ["getState" as const, "getStateV2" as const]) { it(method, async () => { - const state = ssz.phase0.BeaconState.defaultValue(); + const state = ssz.phase0.BeaconState.defaultValue; const stateSerialized = ssz.phase0.BeaconState.serialize(state); mockApi[method].resolves(stateSerialized); diff --git a/packages/api/test/unit/lightclient.test.ts b/packages/api/test/unit/lightclient.test.ts index 8730b463f88a..fae705b0571f 100644 --- a/packages/api/test/unit/lightclient.test.ts +++ b/packages/api/test/unit/lightclient.test.ts @@ -10,9 +10,9 @@ import {toHexString} from "@chainsafe/ssz"; const root = Uint8Array.from(Buffer.alloc(32, 1)); describe("lightclient", () => { - const lightClientUpdate = ssz.altair.LightClientUpdate.defaultValue(); - const syncAggregate = ssz.altair.SyncAggregate.defaultValue(); - const header = ssz.phase0.BeaconBlockHeader.defaultValue(); + const lightClientUpdate = ssz.altair.LightClientUpdate.defaultValue; + const syncAggregate = ssz.altair.SyncAggregate.defaultValue; + const header = ssz.phase0.BeaconBlockHeader.defaultValue; runGenericServerTest(config, getClient, getRoutes, { getStateProof: { diff --git a/packages/api/test/unit/node.test.ts b/packages/api/test/unit/node.test.ts index 1a810265b7cf..65d02968a063 100644 --- a/packages/api/test/unit/node.test.ts +++ b/packages/api/test/unit/node.test.ts @@ -24,7 +24,7 @@ describe("node", () => { enr: "enr", p2pAddresses: ["p2pAddresses"], discoveryAddresses: ["discoveryAddresses"], - metadata: ssz.altair.Metadata.defaultValue(), + metadata: ssz.altair.Metadata.defaultValue, }, }, }, diff --git a/packages/api/test/unit/validator.test.ts b/packages/api/test/unit/validator.test.ts index 532a90c19550..48e3adf9f640 100644 --- a/packages/api/test/unit/validator.test.ts +++ b/packages/api/test/unit/validator.test.ts @@ -40,30 +40,30 @@ describe("validator", () => { }, produceBlock: { args: [32000, Buffer.alloc(96, 1), "graffiti"], - res: {data: ssz.phase0.BeaconBlock.defaultValue()}, + res: {data: ssz.phase0.BeaconBlock.defaultValue}, }, produceBlockV2: { args: [32000, Buffer.alloc(96, 1), "graffiti"], - res: {data: ssz.altair.BeaconBlock.defaultValue(), version: ForkName.altair}, + res: {data: ssz.altair.BeaconBlock.defaultValue, version: ForkName.altair}, }, produceAttestationData: { args: [2, 32000], - res: {data: ssz.phase0.AttestationData.defaultValue()}, + res: {data: ssz.phase0.AttestationData.defaultValue}, }, produceSyncCommitteeContribution: { args: [32000, 2, ZERO_HASH], - res: {data: ssz.altair.SyncCommitteeContribution.defaultValue()}, + res: {data: ssz.altair.SyncCommitteeContribution.defaultValue}, }, getAggregatedAttestation: { args: [ZERO_HASH, 32000], - res: {data: ssz.phase0.Attestation.defaultValue()}, + res: {data: ssz.phase0.Attestation.defaultValue}, }, publishAggregateAndProofs: { - args: [[ssz.phase0.SignedAggregateAndProof.defaultValue()]], + args: [[ssz.phase0.SignedAggregateAndProof.defaultValue]], res: undefined, }, publishContributionAndProofs: { - args: [[ssz.altair.SignedContributionAndProof.defaultValue()]], + args: [[ssz.altair.SignedContributionAndProof.defaultValue]], res: undefined, }, prepareBeaconCommitteeSubnet: { diff --git a/packages/beacon-state-transition/README.md b/packages/beacon-state-transition/README.md index a4a011fc4995..72e4f1e4e69b 100644 --- a/packages/beacon-state-transition/README.md +++ b/packages/beacon-state-transition/README.md @@ -18,17 +18,13 @@ import {generateEmptySignedBlock} from "../test/utils/block"; import {generateState} from "../test/utils/state"; // dummy test state -const state: CachedBeaconStateAllForks = generateState() as CachedBeaconStateAllForks; +const preState: CachedBeaconStateAllForks = generateState() as CachedBeaconStateAllForks; // dummy test block const block: allForks.SignedBeaconBlock = generateEmptySignedBlock(); -let postStateContext: allForks.BeaconState; -try { - postStateContext = allForks.stateTransition(state, block); -} catch (e) { - console.log(e); -} +// Run state transition on block +const postState = allForks.stateTransition(preState, block); ``` ## License diff --git a/packages/beacon-state-transition/package.json b/packages/beacon-state-transition/package.json index 19fd21dcf94d..60af4fd6a5fd 100644 --- a/packages/beacon-state-transition/package.json +++ b/packages/beacon-state-transition/package.json @@ -35,14 +35,15 @@ }, "types": "lib/index.d.ts", "dependencies": { + "@chainsafe/as-sha256": "^0.2.4", "@chainsafe/bls": "6.0.3", "@chainsafe/lodestar-config": "^0.35.0", "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", + "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts b/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts index b4ddc9ff8e32..e2a1411e3034 100644 --- a/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts +++ b/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts @@ -6,7 +6,7 @@ import {CachedBeaconStateAllForks} from "../../types"; * Initiate the exit of the validator with index ``index``. * * NOTE: This function takes a `validator` as argument instead of the validator index. - * SSZ TreeBacked have a dangerous edge case that may break the code here in a non-obvious way. + * SSZ TreeViews have a dangerous edge case that may break the code here in a non-obvious way. * When running `state.validators[i]` you get a SubTree of that validator with a hook to the state. * Then, when a property of `validator` is set it propagates the changes upwards to the parent tree up to the state. * This means that `validator` will propagate its new state along with the current state of its parent tree up to diff --git a/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts b/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts index 0a75ee82401a..9a6eeb7d8917 100644 --- a/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts +++ b/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "../../types"; @@ -12,7 +11,7 @@ export function isValidIndexedAttestation( indexedAttestation: phase0.IndexedAttestation, verifySignature = true ): boolean { - const indices = Array.from(readonlyValues(indexedAttestation.attestingIndices)); + const indices = indexedAttestation.attestingIndices; // verify max number of indices if (!(indices.length > 0 && indices.length <= MAX_VALIDATORS_PER_COMMITTEE)) { diff --git a/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts index 981a1d5744ae..cf513a60ab47 100644 --- a/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts @@ -18,7 +18,7 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - assertValidAttesterSlashing(state as CachedBeaconStateAllForks, attesterSlashing, verifySignatures); + assertValidAttesterSlashing(state, attesterSlashing, verifySignatures); const intersectingIndices = getAttesterSlashableIndices(attesterSlashing); @@ -26,7 +26,7 @@ export function processAttesterSlashing( const validators = state.validators; // Get the validators sub tree once for all indices // Spec requires to sort indexes beforehand for (const index of intersectingIndices.sort((a, b) => a - b)) { - if (isSlashableValidator(validators[index], state.epochCtx.currentShuffling.epoch)) { + if (isSlashableValidator(validators.get(index), state.epochCtx.currentShuffling.epoch)) { slashValidatorAllForks(fork, state, index); slashedAny = true; } diff --git a/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts b/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts index 3e9ef9156aef..cb0297290820 100644 --- a/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts +++ b/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts @@ -1,4 +1,4 @@ -import {toHexString} from "@chainsafe/ssz"; +import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {allForks, ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "../../types"; import {ZERO_HASH} from "../../constants"; @@ -22,7 +22,7 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: allF ); } // verify that proposer index is the correct index - const proposerIndex = state.getBeaconProposer(slot); + const proposerIndex = state.epochCtx.getBeaconProposer(slot); if (block.proposerIndex !== proposerIndex) { throw new Error( `Block proposer index does not match state proposer index blockProposerIndex=${block.proposerIndex} stateProposerIndex=${proposerIndex}` @@ -31,22 +31,23 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: allF const types = state.config.getForkTypes(slot); // verify that the parent matches - if (!ssz.Root.equals(block.parentRoot, ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader))) { + if (!byteArrayEquals(block.parentRoot, ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader))) { throw new Error( `Block parent root ${toHexString(block.parentRoot)} does not match state latest block, block slot=${slot}` ); } + // cache current block as the new latest block - state.latestBlockHeader = { + state.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU({ slot: slot, proposerIndex: block.proposerIndex, parentRoot: block.parentRoot, stateRoot: ZERO_HASH, bodyRoot: types.BeaconBlockBody.hashTreeRoot(block.body), - }; + }); // verify proposer is not slashed. Only once per block, may use the slower read from tree - if (state.validators[proposerIndex].slashed) { + if (state.validators.get(proposerIndex).slashed) { throw new Error("Block proposer is slashed"); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processDeposit.ts b/packages/beacon-state-transition/src/allForks/block/processDeposit.ts index fcbd8f051b4f..0046c83029eb 100644 --- a/packages/beacon-state-transition/src/allForks/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/allForks/block/processDeposit.ts @@ -22,14 +22,15 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../types"; */ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, deposit: phase0.Deposit): void { const {config, validators, epochCtx} = state; + // verify the merkle branch if ( !verifyMerkleBranch( ssz.phase0.DepositData.hashTreeRoot(deposit.data), - Array.from(deposit.proof).map((p) => p.valueOf() as Uint8Array), + deposit.proof, DEPOSIT_CONTRACT_TREE_DEPTH + 1, state.eth1DepositIndex, - state.eth1Data.depositRoot.valueOf() as Uint8Array + state.eth1Data.depositRoot ) ) { throw new Error("Deposit has invalid merkle proof"); @@ -38,7 +39,7 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // deposits must be processed in order state.eth1DepositIndex += 1; - const pubkey = deposit.data.pubkey.valueOf() as Uint8Array; // Drop tree + const pubkey = deposit.data.pubkey; // Drop tree const amount = deposit.data.amount; const cachedIndex = epochCtx.pubkey2index.get(pubkey); if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { @@ -54,7 +55,7 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, try { // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed const publicKey = bls.PublicKey.fromBytes(pubkey, CoordType.affine, true); - const signature = bls.Signature.fromBytes(deposit.data.signature.valueOf() as Uint8Array, CoordType.affine, true); + const signature = bls.Signature.fromBytes(deposit.data.signature, CoordType.affine, true); if (!signature.verify(publicKey, signingRoot)) { return; } @@ -64,17 +65,19 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // add validator and balance entries const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); - validators.push({ - pubkey, - withdrawalCredentials: deposit.data.withdrawalCredentials.valueOf() as Uint8Array, // Drop tree - activationEligibilityEpoch: FAR_FUTURE_EPOCH, - activationEpoch: FAR_FUTURE_EPOCH, - exitEpoch: FAR_FUTURE_EPOCH, - withdrawableEpoch: FAR_FUTURE_EPOCH, - effectiveBalance, - slashed: false, - }); - state.balanceList.push(Number(amount)); + validators.push( + ssz.phase0.Validator.toViewDU({ + pubkey, + withdrawalCredentials: deposit.data.withdrawalCredentials, + activationEligibilityEpoch: FAR_FUTURE_EPOCH, + activationEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + withdrawableEpoch: FAR_FUTURE_EPOCH, + effectiveBalance, + slashed: false, + }) + ); + state.balances.push(amount); // Updating here is better than updating at once on epoch transition // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately @@ -82,19 +85,21 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // - Should have equal performance since it sets a value in a flat array epochCtx.effectiveBalanceIncrementsSet(validators.length, effectiveBalance); - // add participation caches - state.previousEpochParticipation.push(0); - state.currentEpochParticipation.push(0); - - // Forks: altair, bellatrix, and future + // Only after altair: if (fork !== ForkName.phase0) { - (state as CachedBeaconStateAltair).inactivityScores.push(0); + const stateAltair = state as CachedBeaconStateAltair; + + stateAltair.inactivityScores.push(0); + + // add participation caches + stateAltair.previousEpochParticipation.push(0); + stateAltair.currentEpochParticipation.push(0); } // now that there is a new validator, update the epoch context with the new pubkey epochCtx.addPubkey(validators.length - 1, pubkey); } else { // increase balance by deposit amount - increaseBalance(state, cachedIndex, Number(amount)); + increaseBalance(state, cachedIndex, amount); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts b/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts index da55c6ae8ee4..dd6d0be50cee 100644 --- a/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts +++ b/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts @@ -1,8 +1,8 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {Node} from "@chainsafe/persistent-merkle-tree"; -import {readonlyValues, TreeBacked} from "@chainsafe/ssz"; -import {CachedBeaconStateAllForks} from "../../types"; +import {CompositeViewDU} from "@chainsafe/ssz"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../../types"; /** * Store vote counts for every eth1 block that has votes; if any eth1 block wins majority support within a 1024-slot @@ -12,24 +12,24 @@ import {CachedBeaconStateAllForks} from "../../types"; * - Best case: Vote is already decided, zero work. See becomesNewEth1Data conditions * - Worst case: 1023 votes and no majority vote yet. */ -export function processEth1Data(state: CachedBeaconStateAllForks, body: allForks.BeaconBlockBody): void { +export function processEth1Data(state: CachedBeaconStateAllForks, eth1Data: phase0.Eth1Data): void { // Convert to view first to hash once and compare hashes - const eth1DataView = ssz.phase0.Eth1Data.createTreeBackedFromStruct(body.eth1Data); + const eth1DataView = ssz.phase0.Eth1Data.toViewDU(eth1Data); if (becomesNewEth1Data(state, eth1DataView)) { state.eth1Data = eth1DataView; } - state.eth1DataVotes.push(body.eth1Data); + state.eth1DataVotes.push(eth1DataView); } /** - * Returns `newEth1Data` if adding the given `eth1Data` to `state.eth1DataVotes` would + * Returns true if adding the given `eth1Data` to `state.eth1DataVotes` would * result in a change to `state.eth1Data`. */ export function becomesNewEth1Data( - state: CachedBeaconStateAllForks, - newEth1Data: TreeBacked + state: BeaconStateAllForks, + newEth1Data: CompositeViewDU ): boolean { const SLOTS_PER_ETH1_VOTING_PERIOD = EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH; @@ -39,7 +39,7 @@ export function becomesNewEth1Data( } // Nothing to do if the state already has this as eth1data (happens a lot after majority vote is in) - if (ssz.phase0.Eth1Data.equals(state.eth1Data, newEth1Data)) { + if (isEqualEth1DataView(state.eth1Data, newEth1Data)) { return false; } @@ -48,7 +48,7 @@ export function becomesNewEth1Data( // Then isEqualEth1DataView compares cached roots (HashObject as of Jan 2022) which is much cheaper // than doing structural equality, which requires tree -> value conversions let sameVotesCount = 0; - const eth1DataVotes = Array.from(readonlyValues(state.eth1DataVotes)) as TreeBacked[]; + const eth1DataVotes = state.eth1DataVotes.getAllReadonly(); for (let i = 0; i < eth1DataVotes.length; i++) { if (isEqualEth1DataView(eth1DataVotes[i], newEth1Data)) { sameVotesCount++; @@ -63,8 +63,11 @@ export function becomesNewEth1Data( } } -function isEqualEth1DataView(eth1DataA: TreeBacked, eth1DataB: TreeBacked): boolean { - return isEqualNode(eth1DataA.tree.rootNode, eth1DataB.tree.rootNode); +function isEqualEth1DataView( + eth1DataA: CompositeViewDU, + eth1DataB: CompositeViewDU +): boolean { + return isEqualNode(eth1DataA.node, eth1DataB.node); } // TODO: Upstream to persistent-merkle-tree diff --git a/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts index d9fa8d5f98df..4698b1c3780d 100644 --- a/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts @@ -18,7 +18,7 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - assertValidProposerSlashing(state as CachedBeaconStateAllForks, proposerSlashing, verifySignatures); + assertValidProposerSlashing(state, proposerSlashing, verifySignatures); slashValidatorAllForks(fork, state, proposerSlashing.signedHeader1.message.proposerIndex); } @@ -51,18 +51,16 @@ export function assertValidProposerSlashing( } // verify the proposer is slashable - const proposer = state.validators[header1.proposerIndex]; + const proposer = state.validators.get(header1.proposerIndex); if (!isSlashableValidator(proposer, epochCtx.currentShuffling.epoch)) { throw new Error("ProposerSlashing proposer is not slashable"); } // verify signatures if (verifySignatures) { - for (const [i, signatureSet] of getProposerSlashingSignatureSets( - state as CachedBeaconStateAllForks, - proposerSlashing - ).entries()) { - if (!verifySignatureSet(signatureSet)) { + const signatureSets = getProposerSlashingSignatureSets(state, proposerSlashing); + for (let i = 0; i < signatureSets.length; i++) { + if (!verifySignatureSet(signatureSets[i])) { throw new Error(`ProposerSlashing header${i + 1} signature invalid`); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processRandao.ts b/packages/beacon-state-transition/src/allForks/block/processRandao.ts index f1c57ea65042..8287b9abcd6c 100644 --- a/packages/beacon-state-transition/src/allForks/block/processRandao.ts +++ b/packages/beacon-state-transition/src/allForks/block/processRandao.ts @@ -1,5 +1,5 @@ import xor from "buffer-xor"; -import {hash} from "@chainsafe/ssz"; +import SHA256 from "@chainsafe/as-sha256"; import {allForks} from "@chainsafe/lodestar-types"; import {getRandaoMix} from "../../util"; import {verifyRandaoSignature} from "../signatureSets"; @@ -18,18 +18,19 @@ export function processRandao( ): void { const {epochCtx} = state; const epoch = epochCtx.currentShuffling.epoch; - const randaoReveal = block.body.randaoReveal.valueOf() as Uint8Array; + const randaoReveal = block.body.randaoReveal; // verify RANDAO reveal if (verifySignature) { - if (!verifyRandaoSignature(state as CachedBeaconStateAllForks, block)) { + if (!verifyRandaoSignature(state, block)) { throw new Error("RANDAO reveal is an invalid signature"); } } // mix in RANDAO reveal - state.randaoMixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = xor( + const randaoMix = xor( Buffer.from(getRandaoMix(state, epoch) as Uint8Array), - Buffer.from(hash(randaoReveal)) + Buffer.from(SHA256.digest(randaoReveal)) ); + state.randaoMixes.set(epoch % EPOCHS_PER_HISTORICAL_VECTOR, randaoMix); } diff --git a/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts index cab6625c8f7f..5b52a7ed36b3 100644 --- a/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts @@ -15,12 +15,12 @@ export function processVoluntaryExitAllForks( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - if (!isValidVoluntaryExit(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature)) { + if (!isValidVoluntaryExit(state, signedVoluntaryExit, verifySignature)) { throw Error("Invalid voluntary exit"); } - const validator = state.validators[signedVoluntaryExit.message.validatorIndex]; - initiateValidatorExit(state as CachedBeaconStateAllForks, validator); + const validator = state.validators.get(signedVoluntaryExit.message.validatorIndex); + initiateValidatorExit(state, validator); } export function isValidVoluntaryExit( @@ -30,7 +30,7 @@ export function isValidVoluntaryExit( ): boolean { const {config, epochCtx} = state; const voluntaryExit = signedVoluntaryExit.message; - const validator = state.validators[voluntaryExit.validatorIndex]; + const validator = state.validators.get(voluntaryExit.validatorIndex); const currentEpoch = epochCtx.currentShuffling.epoch; return ( @@ -43,6 +43,6 @@ export function isValidVoluntaryExit( // verify the validator had been active long enough currentEpoch >= validator.activationEpoch + config.SHARD_COMMITTEE_PERIOD && // verify signature - (!verifySignature || verifyVoluntaryExitSignature(state as CachedBeaconStateAllForks, signedVoluntaryExit)) + (!verifySignature || verifyVoluntaryExitSignature(state, signedVoluntaryExit)) ); } diff --git a/packages/beacon-state-transition/src/allForks/block/slashValidator.ts b/packages/beacon-state-transition/src/allForks/block/slashValidator.ts index 49043e585fc3..96379cde46e7 100644 --- a/packages/beacon-state-transition/src/allForks/block/slashValidator.ts +++ b/packages/beacon-state-transition/src/allForks/block/slashValidator.ts @@ -23,17 +23,18 @@ export function slashValidatorAllForks( ): void { const {epochCtx} = state; const epoch = epochCtx.currentShuffling.epoch; - const validator = state.validators[slashedIndex]; + const validator = state.validators.get(slashedIndex); // TODO: Bellatrix initiateValidatorExit validators.update() with the one below - initiateValidatorExit(state as CachedBeaconStateAllForks, validator); + initiateValidatorExit(state, validator); validator.slashed = true; validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR); const {effectiveBalance} = validator; // TODO: could state.slashings be number? - state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += BigInt(effectiveBalance); + const slashingIndex = epoch % EPOCHS_PER_SLASHINGS_VECTOR; + state.slashings.set(slashingIndex, state.slashings.get(slashingIndex) + BigInt(effectiveBalance)); const minSlashingPenaltyQuotient = fork === ForkName.phase0 diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts b/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts index adbf9e816d8e..910a172eba34 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts @@ -26,16 +26,18 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, let nextEpochTotalActiveBalanceByIncrement = 0; // update effective balances with hysteresis - if (!epochProcess.balances) { - // only do this for genesis epoch, or spec test - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - epochProcess.balances = Array.from({length: state.balanceList.length}, (_, i) => state.balanceList.get(i)!); - } - for (let i = 0, len = epochProcess.balances.length; i < len; i++) { - const balance = epochProcess.balances[i]; + // epochProcess.balances is set in processRewardsAndPenalties(), so it's recycled here for performance. + // It defaults to `state.balances.getAll()` to make Typescript happy and for spec tests + const balances = epochProcess.balances ?? state.balances.getAll(); + + for (let i = 0, len = balances.length; i < len; i++) { + const balance = balances[i]; + + // PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms) let effectiveBalanceIncrement = effectiveBalanceIncrements[i]; let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT; + if ( // Too big effectiveBalance > balance + DOWNWARD_THRESHOLD || @@ -44,13 +46,14 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, ) { effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); // Update the state tree - validators[i].effectiveBalance = effectiveBalance; - // Also update the fast cached version // Should happen rarely, so it's fine to update the tree - // TODO: Update all in batch after this loop + validators.get(i).effectiveBalance = effectiveBalance; + // Also update the fast cached version effectiveBalanceIncrement = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); effectiveBalanceIncrements[i] = effectiveBalanceIncrement; } + + // TODO: Do this in afterEpochProcess, looping a Uint8Array should be very cheap if (epochProcess.isActiveNextEpoch[i]) { // We track nextEpochTotalActiveBalanceByIncrement as ETH to fit total network balance in a JS number (53 bits) nextEpochTotalActiveBalanceByIncrement += effectiveBalanceIncrement; diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts index fd8d11a945df..1a2293c292e5 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts @@ -1,6 +1,5 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD} from "@chainsafe/lodestar-params"; -import {phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {EpochProcess, CachedBeaconStateAllForks} from "../../types"; /** @@ -13,6 +12,6 @@ export function processEth1DataReset(state: CachedBeaconStateAllForks, epochProc // reset eth1 data votes if (nextEpoch % EPOCHS_PER_ETH1_VOTING_PERIOD === 0) { - state.eth1DataVotes = ([] as phase0.Eth1Data[]) as List; + state.eth1DataVotes = ssz.phase0.Eth1DataVotes.defaultViewDU; } } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts b/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts index 1c32db992727..459ebc49f804 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts @@ -14,9 +14,11 @@ export function processHistoricalRootsUpdate(state: CachedBeaconStateAllForks, e // set historical root accumulator if (nextEpoch % intDiv(SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH) === 0) { state.historicalRoots.push( - ssz.phase0.HistoricalBatch.hashTreeRoot({ - blockRoots: state.blockRoots, - stateRoots: state.stateRoots, + // HistoricalBatchRoots = Non-spec'ed helper type to allow efficient hashing in epoch transition. + // This type is like a 'Header' of HistoricalBatch where its fields are hashed. + ssz.phase0.HistoricalBatchRoots.hashTreeRoot({ + blockRoots: state.blockRoots.hashTreeRoot(), + stateRoots: state.stateRoots.hashTreeRoot(), }) ); } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts index e2b8721e49ec..dcda127dcde8 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts @@ -1,5 +1,5 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; - +import {ssz} from "@chainsafe/lodestar-types"; import {getBlockRoot} from "../../util"; import {CachedBeaconStateAllForks, EpochProcess} from "../../types"; @@ -27,42 +27,44 @@ export function processJustificationAndFinalization( // Process justifications state.previousJustifiedCheckpoint = state.currentJustifiedCheckpoint; const bits = state.justificationBits; - for (let i = bits.length - 1; i >= 1; i--) { - bits[i] = bits[i - 1]; + for (let i = bits.bitLen - 1; i >= 1; i--) { + bits.set(i, bits.get(i - 1)); } - bits[0] = false; + bits.set(0, false); if (epochProcess.prevEpochUnslashedStake.targetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { - state.currentJustifiedCheckpoint = { + state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: previousEpoch, root: getBlockRoot(state, previousEpoch), - }; - bits[1] = true; + }); + bits.set(1, true); } if (epochProcess.currEpochUnslashedTargetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { - state.currentJustifiedCheckpoint = { + state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: currentEpoch, root: getBlockRoot(state, currentEpoch), - }; - bits[0] = true; + }); + bits.set(0, true); } state.justificationBits = bits; + // TODO: Consider rendering bits as array of boolean for faster repeated access here + // Process finalizations // The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source - if (bits[1] && bits[2] && bits[3] && oldPreviousJustifiedCheckpoint.epoch + 3 === currentEpoch) { + if (bits.get(1) && bits.get(2) && bits.get(3) && oldPreviousJustifiedCheckpoint.epoch + 3 === currentEpoch) { state.finalizedCheckpoint = oldPreviousJustifiedCheckpoint; } // The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source - if (bits[1] && bits[2] && oldPreviousJustifiedCheckpoint.epoch + 2 === currentEpoch) { + if (bits.get(1) && bits.get(2) && oldPreviousJustifiedCheckpoint.epoch + 2 === currentEpoch) { state.finalizedCheckpoint = oldPreviousJustifiedCheckpoint; } // The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source - if (bits[0] && bits[1] && bits[2] && oldCurrentJustifiedCheckpoint.epoch + 2 === currentEpoch) { + if (bits.get(0) && bits.get(1) && bits.get(2) && oldCurrentJustifiedCheckpoint.epoch + 2 === currentEpoch) { state.finalizedCheckpoint = oldCurrentJustifiedCheckpoint; } // The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source - if (bits[0] && bits[1] && oldCurrentJustifiedCheckpoint.epoch + 1 === currentEpoch) { + if (bits.get(0) && bits.get(1) && oldCurrentJustifiedCheckpoint.epoch + 1 === currentEpoch) { state.finalizedCheckpoint = oldCurrentJustifiedCheckpoint; } } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts index 5ea10de02418..40f47f0437eb 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts @@ -1,5 +1,4 @@ import {EPOCHS_PER_HISTORICAL_VECTOR} from "@chainsafe/lodestar-params"; -import {getRandaoMix} from "../../util"; import {EpochProcess, CachedBeaconStateAllForks} from "../../types"; /** @@ -12,5 +11,8 @@ export function processRandaoMixesReset(state: CachedBeaconStateAllForks, epochP const nextEpoch = currentEpoch + 1; // set randao mix - state.randaoMixes[nextEpoch % EPOCHS_PER_HISTORICAL_VECTOR] = getRandaoMix(state, currentEpoch); + state.randaoMixes.set( + nextEpoch % EPOCHS_PER_HISTORICAL_VECTOR, + state.randaoMixes.get(currentEpoch % EPOCHS_PER_HISTORICAL_VECTOR) + ); } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts b/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts index 5da49036252f..4825bf133a14 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts @@ -28,18 +28,18 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, epochPr for (const index of epochProcess.indicesToEject) { // set validator exit epoch and withdrawable epoch // TODO: Figure out a way to quickly set properties on the validators tree - initiateValidatorExit(state, validators[index]); + initiateValidatorExit(state, validators.get(index)); } // set new activation eligibilities for (const index of epochProcess.indicesEligibleForActivationQueue) { - validators[index].activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1; + validators.get(index).activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1; } const finalityEpoch = state.finalizedCheckpoint.epoch; // dequeue validators for activation up to churn limit for (const index of epochProcess.indicesEligibleForActivation.slice(0, epochCtx.churnLimit)) { - const validator = validators[index]; + const validator = validators.get(index); // placement in queue is finalized if (validator.activationEligibilityEpoch > finalityEpoch) { // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts index e4cc7076b8c3..2db619628aae 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts @@ -1,17 +1,18 @@ -import {CachedBeaconStateAltair, CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; import {ForkName, GENESIS_EPOCH} from "@chainsafe/lodestar-params"; +import {ssz} from "@chainsafe/lodestar-types"; +import {EpochProcess} from "../../cache/epochProcess"; import {getAttestationDeltas as getAttestationDeltasPhase0} from "../../phase0/epoch/getAttestationDeltas"; import {getRewardsAndPenalties as getRewardsPenaltiesAltair} from "../../altair/epoch/getRewardsAndPenalties"; +import {CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../cache/stateCache"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ -export function processRewardsAndPenaltiesAllForks( +export function processRewardsAndPenaltiesAllForks( fork: ForkName, - state: T, + state: CachedBeaconStateAllForks, epochProcess: EpochProcess ): void { // No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch @@ -24,13 +25,17 @@ export function processRewardsAndPenaltiesAllForks a + b, BigInt(0)); + + // TODO: Could totalSlashings be number? + // TODO: Could totalSlashing be cached? + let totalSlashings = BigInt(0); + const slashings = state.slashings.getAll(); + for (let i = 0; i < slashings.length; i++) { + totalSlashings += slashings[i]; + } const proportionalSlashingMultiplier = fork === ForkName.phase0 diff --git a/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts index 6bccd3cb87cc..c25f9a5f4a73 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts @@ -10,5 +10,5 @@ export function processSlashingsReset(state: CachedBeaconStateAllForks, epochPro const nextEpoch = epochProcess.currentEpoch + 1; // reset slashings - state.slashings[nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR] = BigInt(0); + state.slashings.set(nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR, BigInt(0)); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts index 877f0c449023..7680504f5572 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0} from "@chainsafe/lodestar-types"; import {ISignatureSet} from "../../util"; import {CachedBeaconStateAllForks} from "../../types"; @@ -19,7 +18,15 @@ export function getAttesterSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.attesterSlashings), (attesterSlashing) => - getAttesterSlashingSignatureSets(state, attesterSlashing) - ).flat(1); + const signatureSets: ISignatureSet[] = []; + + for (const attesterSlashing of signedBlock.message.body.attesterSlashings) { + const attesterSlashingSigSets = getAttesterSlashingSignatureSets(state, attesterSlashing); + + for (const signatureSet of attesterSlashingSigSets) { + signatureSets.push(signatureSet); + } + } + + return signatureSets; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts index e50fe82f8486..5dc29de7c085 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts @@ -1,6 +1,5 @@ import {DOMAIN_BEACON_ATTESTER} from "@chainsafe/lodestar-params"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; -import {readonlyValues} from "@chainsafe/ssz"; import { computeSigningRoot, computeStartSlotAtEpoch, @@ -31,7 +30,7 @@ export function getAttestationWithIndicesSignatureSet( type: SignatureSetType.aggregate, pubkeys: indices.map((i) => epochCtx.index2pubkey[i]), signingRoot: computeSigningRoot(ssz.phase0.AttestationData, attestation.data, domain), - signature: attestation.signature.valueOf() as Uint8Array, + signature: attestation.signature, }; } @@ -43,7 +42,7 @@ export function getIndexedAttestationSignatureSet( return getAttestationWithIndicesSignatureSet( state, indexedAttestation, - indices ?? Array.from(readonlyValues(indexedAttestation.attestingIndices)) + indices ?? indexedAttestation.attestingIndices ); } @@ -51,7 +50,11 @@ export function getAttestationsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.attestations), (attestation) => - getIndexedAttestationSignatureSet(state, state.getIndexedAttestation(attestation)) - ); + const signatureSets: ISignatureSet[] = []; + + for (const attestation of signedBlock.message.body.attestations) { + signatureSets.push(getIndexedAttestationSignatureSet(state, state.epochCtx.getIndexedAttestation(attestation))); + } + + return signatureSets; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts b/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts index 976fb3966c48..7a4d5b6edd31 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts @@ -27,6 +27,6 @@ export function getProposerSignatureSet( signedBlock.message, domain ), - signature: signedBlock.signature.valueOf() as Uint8Array, + signature: signedBlock.signature, }; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts index e2e2feac58d2..5d6c9a11af44 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts @@ -1,5 +1,4 @@ import {DOMAIN_BEACON_PROPOSER} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; import {computeSigningRoot, ISignatureSet, SignatureSetType} from "../../util"; import {CachedBeaconStateAllForks} from "../../types"; @@ -23,7 +22,7 @@ export function getProposerSlashingSignatureSets( type: SignatureSetType.single, pubkey, signingRoot: computeSigningRoot(beaconBlockHeaderType, signedHeader.message, domain), - signature: signedHeader.signature.valueOf() as Uint8Array, + signature: signedHeader.signature, }; } ); @@ -33,7 +32,15 @@ export function getProposerSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.proposerSlashings), (proposerSlashing) => - getProposerSlashingSignatureSets(state, proposerSlashing) - ).flat(1); + const signatureSets: ISignatureSet[] = []; + + for (const proposerSlashing of signedBlock.message.body.proposerSlashings) { + const proposerSlashingSigSets = getProposerSlashingSignatureSets(state, proposerSlashing); + + for (const signatureSet of proposerSlashingSigSets) { + signatureSets.push(signatureSet); + } + } + + return signatureSets; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts b/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts index 3432e1ac35c3..fd1f28ddbf73 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts @@ -23,6 +23,6 @@ export function getRandaoRevealSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[block.proposerIndex], signingRoot: computeSigningRoot(ssz.Epoch, epoch, domain), - signature: block.body.randaoReveal.valueOf() as Uint8Array, + signature: block.body.randaoReveal, }; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts index b4939f54f50c..bdbbb163efc3 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts @@ -1,5 +1,4 @@ import {DOMAIN_VOLUNTARY_EXIT} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; import { computeSigningRoot, @@ -32,7 +31,7 @@ export function getVoluntaryExitSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[signedVoluntaryExit.message.validatorIndex], signingRoot: computeSigningRoot(ssz.phase0.VoluntaryExit, signedVoluntaryExit.message, domain), - signature: signedVoluntaryExit.signature.valueOf() as Uint8Array, + signature: signedVoluntaryExit.signature, }; } @@ -40,7 +39,11 @@ export function getVoluntaryExitsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.voluntaryExits), (voluntaryExit) => - getVoluntaryExitSignatureSet(state, voluntaryExit) - ); + const signatureSets: ISignatureSet[] = []; + + for (const voluntaryExit of signedBlock.message.body.voluntaryExits) { + signatureSets.push(getVoluntaryExitSignatureSet(state, voluntaryExit)); + } + + return signatureSets; } diff --git a/packages/beacon-state-transition/src/allForks/slot/index.ts b/packages/beacon-state-transition/src/allForks/slot/index.ts index a30cad8e5ed0..3ea0ddbf2c52 100644 --- a/packages/beacon-state-transition/src/allForks/slot/index.ts +++ b/packages/beacon-state-transition/src/allForks/slot/index.ts @@ -1,5 +1,5 @@ -import {ssz} from "@chainsafe/lodestar-types"; import {SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "../../types"; import {ZERO_HASH} from "../../constants"; @@ -7,19 +7,18 @@ import {ZERO_HASH} from "../../constants"; * Dial state to next slot. Common for all forks */ export function processSlot(state: CachedBeaconStateAllForks): void { - const {config} = state; - const types = config.getForkTypes(state.slot); - // Cache state root - const previousStateRoot = types.BeaconState.hashTreeRoot(state); - state.stateRoots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previousStateRoot; + // Note: .hashTreeRoot() automatically commits() pending changes + const previousStateRoot = state.hashTreeRoot(); + state.stateRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousStateRoot); // Cache latest block header state root - if (ssz.Root.equals(state.latestBlockHeader.stateRoot, ZERO_HASH)) { + if (byteArrayEquals(state.latestBlockHeader.stateRoot, ZERO_HASH)) { state.latestBlockHeader.stateRoot = previousStateRoot; } // Cache block root - const previousBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader); - state.blockRoots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previousBlockRoot; + // Note: .hashTreeRoot() automatically commits() pending changes + const previousBlockRoot = state.latestBlockHeader.hashTreeRoot(); + state.blockRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousBlockRoot); } diff --git a/packages/beacon-state-transition/src/allForks/stateTransition.ts b/packages/beacon-state-transition/src/allForks/stateTransition.ts index 1fd120a075b8..ebcd769a9707 100644 --- a/packages/beacon-state-transition/src/allForks/stateTransition.ts +++ b/packages/beacon-state-transition/src/allForks/stateTransition.ts @@ -54,8 +54,8 @@ export function stateTransition( let postState = state.clone(); - // Turn caches into a data-structure optimized for fast writes - postState.setStateCachesAsTransient(); + // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() + // postState.setStateCachesAsTransient(); // Process slots (including those with no blocks) since block. // Includes state upgrades @@ -71,20 +71,21 @@ export function stateTransition( // Process block processBlock(postState, block, options, metrics); + // Apply changes to state, must do before hashing. Note: .hashTreeRoot() automatically commits() too + postState.commit(); + // Verify state root if (verifyStateRoot) { - if (!ssz.Root.equals(block.stateRoot, postState.tree.root)) { + const stateRoot = postState.hashTreeRoot(); + if (!ssz.Root.equals(block.stateRoot, stateRoot)) { throw new Error( `Invalid state root at slot ${block.slot}, expected=${toHexString(block.stateRoot)}, actual=${toHexString( - postState.tree.root + stateRoot )}` ); } } - // Turn caches into a data-structure optimized for hashing and structural sharing - postState.setStateCachesAsPersistent(); - return postState; } @@ -122,13 +123,13 @@ export function processSlots( ): CachedBeaconStateAllForks { let postState = state.clone(); - // Turn caches into a data-structure optimized for fast writes - postState.setStateCachesAsTransient(); + // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() + // postState.setStateCachesAsTransient(); postState = processSlotsWithTransientCache(postState, slot, metrics); - // Turn caches into a data-structure optimized for hashing and structural sharing - postState.setStateCachesAsPersistent(); + // Apply changes to state, must do before hashing + postState.commit(); return postState; } diff --git a/packages/beacon-state-transition/src/altair/block/index.ts b/packages/beacon-state-transition/src/altair/block/index.ts index b62d1d6260fd..610deee9d23d 100644 --- a/packages/beacon-state-transition/src/altair/block/index.ts +++ b/packages/beacon-state-transition/src/altair/block/index.ts @@ -24,7 +24,7 @@ export { export function processBlock(state: CachedBeaconStateAltair, block: altair.BeaconBlock, verifySignatures = true): void { processBlockHeader(state as CachedBeaconStateAllForks, block); processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); processOperations(state, block.body, verifySignatures); processSyncAggregate(state, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processAttestation.ts b/packages/beacon-state-transition/src/altair/block/processAttestation.ts index dd60b172004a..a773f6135797 100644 --- a/packages/beacon-state-transition/src/altair/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/altair/block/processAttestation.ts @@ -1,8 +1,9 @@ -import {Epoch, ParticipationFlags, phase0, Root, Slot, ssz} from "@chainsafe/lodestar-types"; +import {Epoch, phase0, Root, Slot} from "@chainsafe/lodestar-types"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {intSqrt} from "@chainsafe/lodestar-utils"; import {getBlockRoot, getBlockRootAtSlot, increaseBalance, verifySignatureSet} from "../../util"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochContext} from "../../types"; +import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; import { MIN_ATTESTATION_INCLUSION_DELAY, PROPOSER_WEIGHT, @@ -17,7 +18,6 @@ import { } from "@chainsafe/lodestar-params"; import {checkpointToStr, validateAttestation} from "../../phase0/block/processAttestation"; import {getAttestationWithIndicesSignatureSet} from "../../allForks"; -import {CachedEpochParticipation} from "../../cache/cachedEpochParticipation"; const PROPOSER_REWARD_DOMINATOR = ((WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR) / PROPOSER_WEIGHT; @@ -34,19 +34,18 @@ export function processAttestations( const {epochCtx} = state; const {effectiveBalanceIncrements} = epochCtx; const stateSlot = state.slot; - const rootCache = new RootCache(state as CachedBeaconStateAllForks); + const rootCache = new RootCache(state); // Process all attestations first and then increase the balance of the proposer once let proposerReward = 0; - const previousEpochStatuses = new Map(); - const currentEpochStatuses = new Map(); for (const attestation of attestations) { const data = attestation.data; validateAttestation(state as CachedBeaconStateAllForks, attestation); // Retrieve the validator indices from the attestation participation bitfield - const attestingIndices = epochCtx.getAttestingIndices(data, attestation.aggregationBits); + const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeIndices); // this check is done last because its the most expensive (if signature verification is toggled on) // TODO: Why should we verify an indexed attestation that we just created? If it's just for the signature @@ -66,23 +65,21 @@ export function processAttestations( data.target.epoch === epochCtx.currentShuffling.epoch ? state.currentEpochParticipation : state.previousEpochParticipation; - const epochStatuses = - data.target.epoch === epochCtx.currentShuffling.epoch ? currentEpochStatuses : previousEpochStatuses; - const flagsAttestation = getAttestationParticipationStatus(data, stateSlot - data.slot, rootCache, epochCtx); + const flagsAttestation = getAttestationParticipationStatus(data, stateSlot - data.slot, epochCtx.epoch, rootCache); // For each participant, update their participation // In epoch processing, this participation info is used to calculate balance updates let totalBalanceIncrementsWithWeight = 0; for (const index of attestingIndices) { - const flags = epochStatuses.get(index) ?? (epochParticipation.get(index) as ParticipationFlags); - // Merge (OR) `flagsAttestation` (new flags) with `flags` (current flags) - const newFlags = flagsAttestation | flags; + const flags = epochParticipation.get(index); // For normal block, > 90% of attestations belong to current epoch // At epoch boundary, 100% of attestations belong to previous epoch // so we want to update the participation flag tree in batch - epochStatuses.set(index, newFlags); + + // Note ParticipationFlags type uses option {setBitwiseOR: true}, .set() does a |= operation + epochParticipation.set(index, flagsAttestation); // epochParticipation.setStatus(index, newStatus); // Returns flags that are NOT set before (~ bitwise NOT) AND are set after @@ -103,19 +100,9 @@ export function processAttestations( // Do the discrete math inside the loop to ensure a deterministic result const totalIncrements = totalBalanceIncrementsWithWeight; - const proposerRewardNumerator = totalIncrements * state.baseRewardPerIncrement; + const proposerRewardNumerator = totalIncrements * state.epochCtx.baseRewardPerIncrement; proposerReward += Math.floor(proposerRewardNumerator / PROPOSER_REWARD_DOMINATOR); } - updateEpochParticipants( - previousEpochStatuses, - state.previousEpochParticipation, - epochCtx.previousShuffling.activeIndices.length - ); - updateEpochParticipants( - currentEpochStatuses, - state.currentEpochParticipation, - epochCtx.currentShuffling.activeIndices.length - ); increaseBalance(state, epochCtx.getBeaconProposer(state.slot), proposerReward); } @@ -126,13 +113,11 @@ export function processAttestations( export function getAttestationParticipationStatus( data: phase0.AttestationData, inclusionDelay: number, - rootCache: RootCache, - epochCtx: EpochContext -): ParticipationFlags { + currentEpoch: Epoch, + rootCache: RootCache +): number { const justifiedCheckpoint = - data.target.epoch === epochCtx.currentShuffling.epoch - ? rootCache.currentJustifiedCheckpoint - : rootCache.previousJustifiedCheckpoint; + data.target.epoch === currentEpoch ? rootCache.currentJustifiedCheckpoint : rootCache.previousJustifiedCheckpoint; // The source and target votes are part of the FFG vote, the head vote is part of the fork choice vote // Both are tracked to properly incentivise validators @@ -140,7 +125,7 @@ export function getAttestationParticipationStatus( // The source vote always matches the justified checkpoint (else its invalid) // The target vote should match the most recent checkpoint (eg: the first root of the epoch) // The head vote should match the root at the attestation slot (eg: the root at data.slot) - const isMatchingSource = ssz.phase0.Checkpoint.equals(data.source, justifiedCheckpoint); + const isMatchingSource = checkpointValueEquals(data.source, justifiedCheckpoint); if (!isMatchingSource) { throw new Error( `Attestation source does not equal justified checkpoint: source=${checkpointToStr( @@ -149,11 +134,11 @@ export function getAttestationParticipationStatus( ); } - const isMatchingTarget = ssz.Root.equals(data.target.root, rootCache.getBlockRoot(data.target.epoch)); + const isMatchingTarget = byteArrayEquals(data.target.root, rootCache.getBlockRoot(data.target.epoch)); // a timely head is only be set if the target is _also_ matching const isMatchingHead = - isMatchingTarget && ssz.Root.equals(data.beaconBlockRoot, rootCache.getBlockRootAtSlot(data.slot)); + isMatchingTarget && byteArrayEquals(data.beaconBlockRoot, rootCache.getBlockRootAtSlot(data.slot)); let flags = 0; if (isMatchingSource && inclusionDelay <= intSqrt(SLOTS_PER_EPOCH)) flags |= TIMELY_SOURCE; @@ -163,6 +148,10 @@ export function getAttestationParticipationStatus( return flags; } +export function checkpointValueEquals(cp1: phase0.Checkpoint, cp2: phase0.Checkpoint): boolean { + return cp1.epoch === cp2.epoch && byteArrayEquals(cp1.root, cp2.root); +} + /** * Cache to prevent accessing the state tree to fetch block roots repeteadly. * In normal network conditions the same root is read multiple times, specially the target. @@ -196,23 +185,3 @@ export class RootCache { return root; } } - -export function updateEpochParticipants( - epochStatuses: Map, - epochParticipation: CachedEpochParticipation, - numActiveValidators: number -): void { - // all active validators are attesters, there are 32 slots per epoch - // if 1/2 of that or more are updated flags, do that in batch, see the benchmark for more details - if (epochStatuses.size >= numActiveValidators / (2 * SLOTS_PER_EPOCH)) { - const transientVector = epochParticipation.persistent.asTransient(); - for (const [index, flags] of epochStatuses.entries()) { - transientVector.set(index, flags); - } - epochParticipation.updateAllStatus(transientVector.vector); - } else { - for (const [index, flags] of epochStatuses.entries()) { - epochParticipation.set(index, flags); - } - } -} diff --git a/packages/beacon-state-transition/src/altair/block/processOperations.ts b/packages/beacon-state-transition/src/altair/block/processOperations.ts index 37151f52f4db..6fe79034edfb 100644 --- a/packages/beacon-state-transition/src/altair/block/processOperations.ts +++ b/packages/beacon-state-transition/src/altair/block/processOperations.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {altair} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair} from "../../types"; @@ -22,19 +21,19 @@ export function processOperations( ); } - for (const proposerSlashing of readonlyValues(body.proposerSlashings)) { + for (const proposerSlashing of body.proposerSlashings) { processProposerSlashing(state, proposerSlashing, verifySignatures); } - for (const attesterSlashing of readonlyValues(body.attesterSlashings)) { + for (const attesterSlashing of body.attesterSlashings) { processAttesterSlashing(state, attesterSlashing, verifySignatures); } - processAttestations(state, Array.from(readonlyValues(body.attestations)), verifySignatures); + processAttestations(state, body.attestations, verifySignatures); - for (const deposit of readonlyValues(body.deposits)) { + for (const deposit of body.deposits) { processDeposit(state, deposit); } - for (const voluntaryExit of readonlyValues(body.voluntaryExits)) { + for (const voluntaryExit of body.voluntaryExits) { processVoluntaryExit(state, voluntaryExit, verifySignatures); } } diff --git a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts index 82f9df56a2d8..5c29b531dfe9 100644 --- a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts +++ b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts @@ -1,17 +1,11 @@ -import {altair, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {altair, ssz} from "@chainsafe/lodestar-types"; import {DOMAIN_SYNC_COMMITTEE} from "@chainsafe/lodestar-params"; +import {byteArrayEquals} from "@chainsafe/ssz"; -import { - computeSigningRoot, - getBlockRootAtSlot, - ISignatureSet, - SignatureSetType, - verifySignatureSet, - zipAllIndexesSyncCommitteeBits, - zipIndexesSyncCommitteeBits, -} from "../../util"; +import {computeSigningRoot, getBlockRootAtSlot, ISignatureSet, SignatureSetType, verifySignatureSet} from "../../util"; import {CachedBeaconStateAltair} from "../../types"; import {G2_POINT_AT_INFINITY} from "../../constants"; +import {getUnparticipantValues} from "../../util/array"; export function processSyncAggregate( state: CachedBeaconStateAltair, @@ -19,7 +13,9 @@ export function processSyncAggregate( verifySignatures = true ): void { const {syncParticipantReward, syncProposerReward} = state.epochCtx; - const [participantIndices, unparticipantIndices] = getParticipantInfo(state, block.body.syncAggregate); + const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; + const participantIndices = block.body.syncAggregate.syncCommitteeBits.intersectValues(committeeIndices); + const unparticipantIndices = getUnparticipantValues(participantIndices, committeeIndices); // different from the spec but not sure how to get through signature verification for default/empty SyncAggregate in the spec test if (verifySignatures) { @@ -30,16 +26,22 @@ export function processSyncAggregate( throw Error("Sync committee signature invalid"); } } - const deltaByIndex = new Map(); + + const balances = state.balances; + + // Proposer reward const proposerIndex = state.epochCtx.getBeaconProposer(state.slot); - for (const participantIndex of participantIndices) { - accumulateDelta(deltaByIndex, participantIndex, syncParticipantReward); + balances.set(proposerIndex, balances.get(proposerIndex) + syncProposerReward * participantIndices.length); + + // Positive rewards for participants + for (const index of participantIndices) { + balances.set(index, balances.get(index) + syncParticipantReward); } - accumulateDelta(deltaByIndex, proposerIndex, syncProposerReward * participantIndices.length); - for (const unparticipantIndex of unparticipantIndices) { - accumulateDelta(deltaByIndex, unparticipantIndex, -syncParticipantReward); + + // Negative rewards for non participants + for (const index of unparticipantIndices) { + balances.set(index, balances.get(index) - syncParticipantReward); } - state.balanceList.applyDeltaInBatch(deltaByIndex); } export function getSyncCommitteeSignatureSet( @@ -50,7 +52,7 @@ export function getSyncCommitteeSignatureSet( ): ISignatureSet | null { const {epochCtx} = state; const {syncAggregate} = block.body; - const signature = syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array; + const signature = syncAggregate.syncCommitteeSignature; // The spec uses the state to get the previous slot // ```python @@ -67,14 +69,15 @@ export function getSyncCommitteeSignatureSet( const rootSigned = getBlockRootAtSlot(state, previousSlot); if (!participantIndices) { - participantIndices = getParticipantIndices(state, syncAggregate); + const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; + participantIndices = syncAggregate.syncCommitteeBits.intersectValues(committeeIndices); } // When there's no participation we consider the signature valid and just ignore it if (participantIndices.length === 0) { // Must set signature as G2_POINT_AT_INFINITY when participating bits are empty // https://github.com/ethereum/eth2.0-specs/blob/30f2a076377264677e27324a8c3c78c590ae5e20/specs/altair/bls.md#eth2_fast_aggregate_verify - if (ssz.BLSSignature.equals(signature, G2_POINT_AT_INFINITY)) { + if (byteArrayEquals(signature, G2_POINT_AT_INFINITY)) { return null; } else { throw Error("Empty sync committee signature is not infinity"); @@ -90,24 +93,3 @@ export function getSyncCommitteeSignatureSet( signature, }; } - -/** Get participant indices for a sync committee. */ -function getParticipantIndices(state: CachedBeaconStateAltair, syncAggregate: altair.SyncAggregate): number[] { - const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; - return zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits); -} - -/** Return [0] as participant indices and [1] as unparticipant indices for a sync committee. */ -function getParticipantInfo(state: CachedBeaconStateAltair, syncAggregate: altair.SyncAggregate): [number[], number[]] { - const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; - return zipAllIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits); -} - -function accumulateDelta(deltaByIndex: Map, index: ValidatorIndex, delta: number): void { - const existingDelta = deltaByIndex.get(index); - if (existingDelta === undefined) { - deltaByIndex.set(index, delta); - } else { - deltaByIndex.set(index, delta + existingDelta); - } -} diff --git a/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts b/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts index b53379915798..9ec87ad3a1b8 100644 --- a/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts @@ -9,12 +9,12 @@ import { WEIGHT_DENOMINATOR, ForkName, } from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import { FLAG_ELIGIBLE_ATTESTER, - FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED, - FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED, - FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED, + FLAG_PREV_HEAD_ATTESTER_UNSLASHED, + FLAG_PREV_SOURCE_ATTESTER_UNSLASHED, + FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers, } from "../../util/attesterStatus"; import {isInInactivityLeak, newZeroedArray} from "../../util"; @@ -48,7 +48,7 @@ export function getRewardsAndPenalties(state: CachedBeaconStateAltair, process: const rewards = newZeroedArray(validatorCount); const penalties = newZeroedArray(validatorCount); - const isInInactivityLeakBn = isInInactivityLeak(state as CachedBeaconStateAllForks); + const isInInactivityLeakBn = isInInactivityLeak(state); // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE // so there are limited values of them like 32, 31, 30 const rewardPenaltyItemCache = new Map(); @@ -100,31 +100,35 @@ export function getRewardsAndPenalties(state: CachedBeaconStateAltair, process: } = rewardPenaltyItem; // same logic to getFlagIndexDeltas - if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED)) { + if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelySourceReward; } } else { penalties[i] += timelySourcePenalty; } - if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { + + if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelyTargetReward; } } else { penalties[i] += timelyTargetPenalty; } - if (hasMarkers(status.flags, FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED)) { + + if (hasMarkers(status.flags, FLAG_PREV_HEAD_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelyHeadReward; } } + // Same logic to getInactivityPenaltyDeltas // TODO: if we have limited value in inactivityScores we can provide a cache too - if (!hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { - const penaltyNumerator = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT * state.inactivityScores[i]; + if (!hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { + const penaltyNumerator = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT * state.inactivityScores.get(i); penalties[i] += Math.floor(penaltyNumerator / penaltyDenominator); } } + return [rewards, penalties]; } diff --git a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts index 7e55339e4bc9..2ac7cc2de243 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts @@ -1,5 +1,4 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; -import {Number64} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; import * as attesterStatusUtil from "../../util/attesterStatus"; import {isInInactivityLeak} from "../../util"; @@ -19,33 +18,36 @@ import {isInInactivityLeak} from "../../util"; * TODO: Compute from altair testnet inactivityScores updates on average */ export function processInactivityUpdates(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - if (state.currentShuffling.epoch === GENESIS_EPOCH) { + if (state.epochCtx.epoch === GENESIS_EPOCH) { return; } + const {config, inactivityScores} = state; const {INACTIVITY_SCORE_BIAS, INACTIVITY_SCORE_RECOVERY_RATE} = config; - const {statuses} = epochProcess; + const {statuses, eligibleValidatorIndices} = epochProcess; const inActivityLeak = isInInactivityLeak(state as CachedBeaconStateAllForks); // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code - const {FLAG_ELIGIBLE_ATTESTER, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED, hasMarkers} = attesterStatusUtil; - const newValues = new Map(); - inactivityScores.forEach(function processInactivityScore(inactivityScore, i) { + const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers} = attesterStatusUtil; + + const inactivityScoresArr = inactivityScores.getAll(); + + for (let j = 0; j < eligibleValidatorIndices.length; j++) { + const i = eligibleValidatorIndices[j]; const status = statuses[i]; - if (hasMarkers(status.flags, FLAG_ELIGIBLE_ATTESTER)) { - const prevInactivityScore = inactivityScore; - if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { - inactivityScore -= Math.min(1, inactivityScore); - } else { - inactivityScore += Number(INACTIVITY_SCORE_BIAS); - } - if (!inActivityLeak) { - inactivityScore -= Math.min(Number(INACTIVITY_SCORE_RECOVERY_RATE), inactivityScore); - } - if (inactivityScore !== prevInactivityScore) { - newValues.set(i, inactivityScore); - } + let inactivityScore = inactivityScoresArr[i]; + + const prevInactivityScore = inactivityScore; + if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { + inactivityScore -= Math.min(1, inactivityScore); + } else { + inactivityScore += INACTIVITY_SCORE_BIAS; } - }); - inactivityScores.setMultiple(newValues); + if (!inActivityLeak) { + inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore); + } + if (inactivityScore !== prevInactivityScore) { + inactivityScores.set(i, inactivityScore); + } + } } diff --git a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts index 9d5dd1dfbf63..30dd4967a689 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts @@ -1,5 +1,5 @@ -import {PersistentVector} from "@chainsafe/persistent-ts"; -import {newFilledArray} from "../../util/array"; +import {ssz} from "@chainsafe/lodestar-types"; +import {newZeroedArray} from "../../util"; import {CachedBeaconStateAltair} from "../../types"; /** @@ -10,6 +10,11 @@ import {CachedBeaconStateAltair} from "../../types"; * trees completely. */ export function processParticipationFlagUpdates(state: CachedBeaconStateAltair): void { - state.previousEpochParticipation.updateAllStatus(state.currentEpochParticipation.persistent.vector); - state.currentEpochParticipation.updateAllStatus(PersistentVector.from(newFilledArray(state.validators.length, 0))); + // Set view and tree from currentEpochParticipation to previousEpochParticipation + state.previousEpochParticipation = state.currentEpochParticipation; + + // Wipe currentEpochParticipation with an empty value + const currentEpochParticipationArr = newZeroedArray(state.currentEpochParticipation.length); + // TODO: Benchmark the cost of transforming to .toViewDU() + state.currentEpochParticipation = ssz.altair.EpochParticipation.toViewDU(currentEpochParticipationArr); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts index ee3c764955a8..f75ec2eab25a 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts @@ -1,13 +1,13 @@ -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; import {ForkName} from "@chainsafe/lodestar-params"; import {processRewardsAndPenaltiesAllForks} from "../../allForks/epoch/processRewardsAndPenalties"; +import {EpochProcess} from "../../cache/epochProcess"; +import {CachedBeaconStateAltair} from "../../types"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ export function processRewardsAndPenalties(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processRewardsAndPenaltiesAllForks(ForkName.altair, state as CachedBeaconStateAllForks, epochProcess); + processRewardsAndPenaltiesAllForks(ForkName.altair, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts index 4c8d8abe43d9..6f662900a744 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts @@ -1,7 +1,8 @@ -import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; import {aggregatePublicKeys} from "@chainsafe/bls"; -import {CachedBeaconStateAltair} from "../../types"; +import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; +import {ssz} from "@chainsafe/lodestar-types"; import {getNextSyncCommitteeIndices} from "../../util/seed"; +import {CachedBeaconStateAltair} from "../../types"; /** * Rotate nextSyncCommittee to currentSyncCommittee if sync committee period is over. @@ -23,17 +24,16 @@ export function processSyncCommitteeUpdates(state: CachedBeaconStateAltair): voi ); // Using the index2pubkey cache is slower because it needs the serialized pubkey. - const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map( - (index) => state.validators[index].pubkey.valueOf() as Uint8Array - ); + const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map((index) => state.validators.get(index).pubkey); // Rotate syncCommittee in state state.currentSyncCommittee = state.nextSyncCommittee; - state.nextSyncCommittee = { + state.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU({ pubkeys: nextSyncCommitteePubkeys, aggregatePubkey: aggregatePublicKeys(nextSyncCommitteePubkeys), - }; + }); + // Rotate syncCommittee cache state.epochCtx.rotateSyncCommitteeIndexed(nextSyncCommitteeIndices); } } diff --git a/packages/beacon-state-transition/src/altair/upgradeState.ts b/packages/beacon-state-transition/src/altair/upgradeState.ts index 864e139980f6..8f8af9e3aa5e 100644 --- a/packages/beacon-state-transition/src/altair/upgradeState.ts +++ b/packages/beacon-state-transition/src/altair/upgradeState.ts @@ -1,61 +1,125 @@ -import {altair, ParticipationFlags, phase0, ssz, Uint8} from "@chainsafe/lodestar-types"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../types"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; import {newZeroedArray} from "../util"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {getAttestationParticipationStatus, RootCache} from "./block/processAttestation"; import {getNextSyncCommittee} from "../util/syncCommittee"; +import {CompositeViewDU} from "@chainsafe/ssz"; +import {getCachedBeaconState} from "../cache/stateCache"; /** * Upgrade a state from phase0 to altair. */ -export function upgradeState(state: CachedBeaconStatePhase0): CachedBeaconStateAltair { - const {config} = state; - const pendingAttesations = Array.from(state.previousEpochAttestations); - const postTreeBackedState = upgradeTreeBackedState(config, state); - const postState = createCachedBeaconState(config, postTreeBackedState); - translateParticipation(postState, pendingAttesations); - return postState; -} +export function upgradeState(statePhase0: CachedBeaconStatePhase0): CachedBeaconStateAltair { + const {config} = statePhase0; + + // Get underlying node and cast phase0 tree to altair tree + // + // A phase0 BeaconState tree can be safely casted to an altair BeaconState tree because: + // - Deprecated fields are replaced by new fields at the exact same indexes + // - All new fields are appended at the end + // + // So by just setting all new fields to some value, all the old nodes are dropped + // + // phase0 | op | altair + // ----------------------------- | ---- | ------------ + // genesis_time | - | genesis_time + // genesis_validators_root | - | genesis_validators_root + // slot | - | slot + // fork | - | fork + // latest_block_header | - | latest_block_header + // block_roots | - | block_roots + // state_roots | - | state_roots + // historical_roots | - | historical_roots + // eth1_data | - | eth1_data + // eth1_data_votes | - | eth1_data_votes + // eth1_deposit_index | - | eth1_deposit_index + // validators | - | validators + // balances | - | balances + // randao_mixes | - | randao_mixes + // slashings | - | slashings + // previous_epoch_attestations | diff | previous_epoch_participation + // current_epoch_attestations | diff | current_epoch_participation + // justification_bits | - | justification_bits + // previous_justified_checkpoint | - | previous_justified_checkpoint + // current_justified_checkpoint | - | current_justified_checkpoint + // finalized_checkpoint | - | finalized_checkpoint + // - | new | inactivity_scores + // - | new | current_sync_committee + // - | new | next_sync_committee + + const statePhase0Node = ssz.phase0.BeaconState.commitViewDU(statePhase0); + const stateAltairView = ssz.altair.BeaconState.getViewDU(statePhase0Node); + // Attach existing BeaconStateCache from statePhase0 to new stateAltairView object + const stateAltair = getCachedBeaconState(stateAltairView, statePhase0); -function upgradeTreeBackedState(config: IBeaconConfig, state: CachedBeaconStatePhase0): TreeBacked { - const nextEpochActiveIndices = state.nextShuffling.activeIndices; - const stateTB = ssz.phase0.BeaconState.createTreeBacked(state.tree); - const validatorCount = stateTB.validators.length; - const epoch = state.currentShuffling.epoch; - // TODO: Does this preserve the hashing cache? In altair devnets memory spikes on the fork transition - const postState = ssz.altair.BeaconState.createTreeBacked(stateTB.tree); - postState.fork = { - previousVersion: stateTB.fork.currentVersion, + stateAltair.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: statePhase0.fork.currentVersion, currentVersion: config.ALTAIR_FORK_VERSION, - epoch, - }; - postState.previousEpochParticipation = newZeroedArray(validatorCount) as List; - postState.currentEpochParticipation = newZeroedArray(validatorCount) as List; - postState.inactivityScores = newZeroedArray(validatorCount) as List; - const syncCommittee = getNextSyncCommittee(state, nextEpochActiveIndices, state.epochCtx.effectiveBalanceIncrements); - postState.currentSyncCommittee = syncCommittee; - postState.nextSyncCommittee = syncCommittee; - return postState; + epoch: statePhase0.epochCtx.epoch, + }); + + const validatorCount = statePhase0.validators.length; + const emptyEpochParticipationView = ssz.altair.EpochParticipation.toViewDU(newZeroedArray(validatorCount)); + const emptyEpochParticipationNode = ssz.altair.EpochParticipation.commitViewDU(emptyEpochParticipationView); + stateAltair.previousEpochParticipation = emptyEpochParticipationView; + // Cloned instance with same immutable Node + stateAltair.currentEpochParticipation = ssz.altair.EpochParticipation.getViewDU(emptyEpochParticipationNode); + + stateAltair.inactivityScores = ssz.altair.InactivityScores.toViewDU(newZeroedArray(validatorCount)); + + const {syncCommittee, indices} = getNextSyncCommittee( + stateAltair, + stateAltair.epochCtx.nextShuffling.activeIndices, + stateAltair.epochCtx.effectiveBalanceIncrements + ); + const syncCommitteeView = ssz.altair.SyncCommittee.toViewDU(syncCommittee); + + stateAltair.currentSyncCommittee = syncCommitteeView; + stateAltair.nextSyncCommittee = syncCommitteeView; + stateAltair.epochCtx.setSyncCommitteesIndexed(indices); + + const pendingAttesations = statePhase0.previousEpochAttestations; + translateParticipation(stateAltair, pendingAttesations); + + // Commit new added fields ViewDU to the root node + stateAltair.commit(); + // Clear cache to ensure the cache of phase0 fields is not used by new altair fields + // [15] previous_epoch_attestations -> previous_epoch_participation + // [16] current_epoch_attestations -> current_epoch_participation + // + // TODO: This could only drop the caches of index 15,16. However this would couple this code tightly with SSZ ViewDU + // internals. If the cache is not cleared, consuming the ViewDU instance could break in strange ways. + stateAltair["clearCache"](); + + return stateAltair; } /** * Translate_participation in https://github.com/ethereum/eth2.0-specs/blob/dev/specs/altair/fork.md */ -function translateParticipation(state: CachedBeaconStateAltair, pendingAttesations: phase0.PendingAttestation[]): void { +function translateParticipation( + state: CachedBeaconStateAltair, + pendingAttesations: CompositeViewDU +): void { const {epochCtx} = state; const rootCache = new RootCache(state as CachedBeaconStateAllForks); const epochParticipation = state.previousEpochParticipation; - for (const attestation of pendingAttesations) { + + for (const attestation of pendingAttesations.getAllReadonly()) { const data = attestation.data; - const flagsAttestation = getAttestationParticipationStatus(data, attestation.inclusionDelay, rootCache, epochCtx); + const attestationFlags = getAttestationParticipationStatus( + data, + attestation.inclusionDelay, + epochCtx.epoch, + rootCache + ); + + const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeIndices); - const attestingIndices = state.getAttestingIndices(data, attestation.aggregationBits); for (const index of attestingIndices) { - const flags = epochParticipation.get(index) as ParticipationFlags; - // Merge (OR) `flagsAttestation` (new flags) with `flags` (current flags) - epochParticipation.set(index, flags | flagsAttestation); + // ParticipationFlags type uses option {setBitwiseOR: true}, .set() does a |= operation + epochParticipation.set(index, attestationFlags); } } } diff --git a/packages/beacon-state-transition/src/bellatrix/block/index.ts b/packages/beacon-state-transition/src/bellatrix/block/index.ts index bdb21ff47bac..abbcd2d1048c 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/index.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/index.ts @@ -26,7 +26,7 @@ export function processBlock( } processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); processOperations(state, block.body, verifySignatures); processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts index 9459e286530b..92ef117fde2c 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts @@ -1,5 +1,5 @@ import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {byteArrayEquals, List, toHexString} from "@chainsafe/ssz"; +import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStateBellatrix} from "../../types"; import {getRandaoMix} from "../../util"; import {ExecutionEngine} from "../executionEngine"; @@ -24,7 +24,7 @@ export function processExecutionPayload( } // Verify random - const expectedRandom = getRandaoMix(state, state.currentShuffling.epoch); + const expectedRandom = getRandaoMix(state, state.epochCtx.epoch); if (!byteArrayEquals(payload.random as Uint8Array, expectedRandom as Uint8Array)) { throw Error( `Invalid execution payload random ${toHexString(payload.random)} expected=${toHexString(expectedRandom)}` @@ -51,7 +51,7 @@ export function processExecutionPayload( } // Cache execution payload header - state.latestExecutionPayloadHeader = { + state.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU({ parentHash: payload.parentHash, feeRecipient: payload.feeRecipient, stateRoot: payload.stateRoot, @@ -65,6 +65,6 @@ export function processExecutionPayload( extraData: payload.extraData, baseFeePerGas: payload.baseFeePerGas, blockHash: payload.blockHash, - transactionsRoot: ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions as List), - }; + transactionsRoot: ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions), + }); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts b/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts index 53fbc8cbd950..922dee0e63a7 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts @@ -1,13 +1,12 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {bellatrix} from "@chainsafe/lodestar-types"; +import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix, CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; import {processProposerSlashing} from "./processProposerSlashing"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processAttestations} from "../../altair/block/processAttestation"; import {processDeposit} from "../../altair/block/processDeposit"; import {processVoluntaryExit} from "../../altair/block/processVoluntaryExit"; -import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; export function processOperations( state: CachedBeaconStateBellatrix, @@ -22,23 +21,23 @@ export function processOperations( ); } - for (const proposerSlashing of readonlyValues(body.proposerSlashings)) { + for (const proposerSlashing of body.proposerSlashings) { processProposerSlashing(state, proposerSlashing, verifySignatures); } - for (const attesterSlashing of readonlyValues(body.attesterSlashings)) { + for (const attesterSlashing of body.attesterSlashings) { processAttesterSlashing(state, attesterSlashing, verifySignatures); } processAttestations( (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, - Array.from(readonlyValues(body.attestations)), + body.attestations, verifySignatures ); - for (const deposit of readonlyValues(body.deposits)) { + for (const deposit of body.deposits) { processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, deposit); } - for (const voluntaryExit of readonlyValues(body.voluntaryExits)) { + for (const voluntaryExit of body.voluntaryExits) { processVoluntaryExit( (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, voluntaryExit, diff --git a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts index b717c0b5153b..887689818ba1 100644 --- a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts +++ b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts @@ -1,34 +1,65 @@ -import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair, CachedBeaconStateBellatrix} from "../types"; -import {TreeBacked} from "@chainsafe/ssz"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {getCachedBeaconState} from "../cache/stateCache"; /** * Upgrade a state from altair to bellatrix. */ -export function upgradeState(state: CachedBeaconStateAltair): CachedBeaconStateBellatrix { - const {config} = state; - const postTreeBackedState = upgradeTreeBackedState(config, state); - // TODO: This seems very sub-optimal, review - return createCachedBeaconState(config, postTreeBackedState); -} +export function upgradeState(stateAltair: CachedBeaconStateAltair): CachedBeaconStateBellatrix { + const {config} = stateAltair; + + // Get underlying node and cast altair tree to bellatrix tree + // + // An altair BeaconState tree can be safely casted to a bellatrix BeaconState tree because: + // - All new fields are appended at the end + // + // altair | op | altair + // ----------------------------- | --- | ------------ + // genesis_time | - | genesis_time + // genesis_validators_root | - | genesis_validators_root + // slot | - | slot + // fork | - | fork + // latest_block_header | - | latest_block_header + // block_roots | - | block_roots + // state_roots | - | state_roots + // historical_roots | - | historical_roots + // eth1_data | - | eth1_data + // eth1_data_votes | - | eth1_data_votes + // eth1_deposit_index | - | eth1_deposit_index + // validators | - | validators + // balances | - | balances + // randao_mixes | - | randao_mixes + // slashings | - | slashings + // previous_epoch_participation | - | previous_epoch_participation + // current_epoch_participation | - | current_epoch_participation + // justification_bits | - | justification_bits + // previous_justified_checkpoint | - | previous_justified_checkpoint + // current_justified_checkpoint | - | current_justified_checkpoint + // finalized_checkpoint | - | finalized_checkpoint + // inactivity_scores | - | inactivity_scores + // current_sync_committee | - | current_sync_committee + // next_sync_committee | - | next_sync_committee + // - | new | latest_execution_payload_header -function upgradeTreeBackedState( - config: IBeaconConfig, - state: CachedBeaconStateAltair -): TreeBacked { - const stateTB = ssz.phase0.BeaconState.createTreeBacked(state.tree); + const stateAltairNode = ssz.altair.BeaconState.commitViewDU(stateAltair); + const stateBellatrixView = ssz.bellatrix.BeaconState.getViewDU(stateAltairNode); + // Attach existing BeaconStateCache from stateAltair to new stateBellatrixView object + const stateBellatrix = getCachedBeaconState(stateBellatrixView, stateAltair); - // TODO: Does this preserve the hashing cache? In altair devnets memory spikes on the fork transition - const postState = ssz.bellatrix.BeaconState.createTreeBacked(stateTB.tree); - postState.fork = { - previousVersion: stateTB.fork.currentVersion, + stateBellatrix.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateAltair.fork.currentVersion, currentVersion: config.BELLATRIX_FORK_VERSION, - epoch: state.currentShuffling.epoch, - }; + epoch: stateAltair.epochCtx.epoch, + }); + // Execution-layer - postState.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked(); + stateBellatrix.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU( + ssz.bellatrix.ExecutionPayloadHeader.defaultValue + ); + + // Commit new added fields ViewDU to the root node + stateBellatrix.commit(); + // No need to clear cache since no index is replaced, only appended at the end - return postState; + return stateBellatrix; } diff --git a/packages/beacon-state-transition/src/bellatrix/utils.ts b/packages/beacon-state-transition/src/bellatrix/utils.ts index 529bffd22b0e..7a713b94ef11 100644 --- a/packages/beacon-state-transition/src/bellatrix/utils.ts +++ b/packages/beacon-state-transition/src/bellatrix/utils.ts @@ -1,13 +1,19 @@ import {allForks, bellatrix, ssz} from "@chainsafe/lodestar-types"; +import { + BeaconStateBellatrix, + BeaconStateAllForks, + CachedBeaconStateBellatrix, + CachedBeaconStateAllForks, +} from "../types"; /** * Execution enabled = merge is done. * When (A) state has execution data OR (B) block has execution data */ -export function isExecutionEnabled(state: bellatrix.BeaconState, body: bellatrix.BeaconBlockBody): boolean { +export function isExecutionEnabled(state: BeaconStateBellatrix, body: bellatrix.BeaconBlockBody): boolean { return ( isMergeTransitionComplete(state) || - !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue()) + !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue) ); } @@ -15,10 +21,10 @@ export function isExecutionEnabled(state: bellatrix.BeaconState, body: bellatrix * Merge block is the SINGLE block that transitions from POW to POS. * state has no execution data AND this block has execution data */ -export function isMergeTransitionBlock(state: bellatrix.BeaconState, body: bellatrix.BeaconBlockBody): boolean { +export function isMergeTransitionBlock(state: BeaconStateBellatrix, body: bellatrix.BeaconBlockBody): boolean { return ( !isMergeTransitionComplete(state) && - !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue()) + !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue) ); } @@ -26,16 +32,22 @@ export function isMergeTransitionBlock(state: bellatrix.BeaconState, body: bella * Merge is complete when the state includes execution layer data: * state.latestExecutionPayloadHeader NOT EMPTY */ -export function isMergeTransitionComplete(state: bellatrix.BeaconState): boolean { +export function isMergeTransitionComplete(state: BeaconStateBellatrix): boolean { return !ssz.bellatrix.ExecutionPayloadHeader.equals( state.latestExecutionPayloadHeader, - ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked() + // TODO: Performance + ssz.bellatrix.ExecutionPayloadHeader.defaultValue ); } /** Type guard for bellatrix.BeaconState */ -export function isBellatrixStateType(state: allForks.BeaconState): state is bellatrix.BeaconState { - return (state as bellatrix.BeaconState).latestExecutionPayloadHeader !== undefined; +export function isBellatrixStateType(state: BeaconStateAllForks): state is BeaconStateBellatrix { + return (state as BeaconStateBellatrix).latestExecutionPayloadHeader !== undefined; +} + +/** Type guard for bellatrix.CachedBeaconState */ +export function isBellatrixCachedStateType(state: CachedBeaconStateAllForks): state is CachedBeaconStateBellatrix { + return (state as CachedBeaconStateBellatrix).latestExecutionPayloadHeader !== undefined; } /** Type guard for bellatrix.BeaconBlockBody */ diff --git a/packages/beacon-state-transition/src/cache/balanceList.ts b/packages/beacon-state-transition/src/cache/balanceList.ts deleted file mode 100644 index 41270266df3d..000000000000 --- a/packages/beacon-state-transition/src/cache/balanceList.ts +++ /dev/null @@ -1,67 +0,0 @@ -import {Number64ListType} from "@chainsafe/ssz"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; - -/** - * Manage balances of BeaconState, use this instead of state.balances - */ -export class BalanceList { - tree: Tree; - type: Number64ListType; - - constructor(type: Number64ListType, tree: Tree) { - this.type = type; - this.tree = tree; - } - - get length(): number { - return this.type.tree_getLength(this.tree); - } - - get(index: number): number | undefined { - return this.type.tree_getProperty(this.tree, index) as number | undefined; - } - - set(index: number, value: number): void { - this.type.tree_setProperty(this.tree, index, value); - } - - applyDelta(index: number, delta: number): number { - return this.type.tree_applyDeltaAtIndex(this.tree, index, delta); - } - - applyDeltaInBatch(deltaByIndex: Map): void { - this.type.tree_applyDeltaInBatch(this.tree, deltaByIndex); - } - - /** Return the new balances */ - updateAll(deltas: number[]): number[] { - const [newTree, newBalances] = this.type.tree_newTreeFromDeltas(this.tree, deltas); - this.tree.rootNode = newTree.rootNode; - this.type.tree_setLength(this.tree, newBalances.length); - return newBalances; - } - - push(value: number): number { - return this.type.tree_push(this.tree, value); - } - - pop(): number { - return this.type.tree_pop(this.tree); - } - - *[Symbol.iterator](): Iterator { - for (let i = 0; i < this.length; i++) { - yield this.get(i) as number; - } - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: number, index: number, list: this) => boolean): number | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: number, index: number, list: this) => boolean): number { - return -1; - } -} diff --git a/packages/beacon-state-transition/src/cache/cachedBeaconState.ts b/packages/beacon-state-transition/src/cache/cachedBeaconState.ts deleted file mode 100644 index 8daea5ab50de..000000000000 --- a/packages/beacon-state-transition/src/cache/cachedBeaconState.ts +++ /dev/null @@ -1,355 +0,0 @@ -import { - BasicListType, - CompositeValue, - ContainerType, - isCompositeType, - isTreeBacked, - ITreeBacked, - List, - Number64ListType, - readonlyValues, - TreeBacked, -} from "@chainsafe/ssz"; -import {allForks, altair, Number64, ParticipationFlags} from "@chainsafe/lodestar-types"; -import {createIBeaconConfig, IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; -import {EpochContext, EpochContextOpts} from "./epochContext"; -import {BalanceList} from "./balanceList"; -import {CachedEpochParticipation, CachedEpochParticipationProxyHandler} from "./cachedEpochParticipation"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedInactivityScoreList, CachedInactivityScoreListProxyHandler} from "./cachedInactivityScoreList"; -import {newFilledArray} from "../util/array"; - -/** - * `BeaconState` with various caches - * - * Currently contains the following: - * - The full list of network params, ssz types, and fork schedule - * - The ssz type for the state - * - The full merkle tree representation of the state - * - A cache of shufflings, committees, proposers, expanded pubkeys - * - A flat copy of validators (for fast access/iteration) - * - * ### BeaconState data representation tradeoffs: - * - * Requirements of a BeaconState: - * - Block processing and epoch processing be performant to not block the node. This functions requires to iterate over - * very large arrays fast, while doing random mutations or big mutations. After them the state must be hashed. - * Processing times: (ideal / current / maximum) - * - block processing: 20ms / 200ms / 500ms - * - epoch processing: 200ms / 2s / 4s - * - * - BeaconState must be memory efficient. Data should only be represented once in a succint manner. Uint8Arrays are - * expensive, native types are not. - * - BeaconState must be hashed efficiently. Data must be merkelized before hashing so the conversion to merkelized - * must be fast or be done already. It must must persist a hashing cache that should be structurally shared between - * states for memory efficiency. - * - BeaconState raw data changes sparsingly, so it should be structurally shared between states for memory efficiency - * - * Summary of goals: - * - Structurally share data + hashing cache - * - Very fast read and iteration over large arrays - * - Fast bulk writes and somewhat fast single writes - * - Fast merkelization of data for hashing - * - * #### state.validators - * - * 91% of all memory merkelized is state.validators. In normal network conditions state.validators changes rarely. - * However for epoch processing the entire array must be iterated and read. So we need fast reads and slow writes. - * Tradeoffs to achieve that: - * - Represent leaf data with native JS types (deserialized form) - * - Use a single Tree for structurally sharing leaf data + hashing cache - * - Keep only the root cached on leaf nodes - * - Micro-optimizations (TODO): - * - Keep also the root of the node above pubkey and withdrawal creds. Will never change - * - Keep pubkey + withdrawal creds in the same Uint8Array - * - Have a global pubkey + withdrawal creds Uint8Array global cache, like with the index2pubkey cache - */ -export type CachedBeaconState = - // Wrapper object ({validators, clone()}) - BeaconStateContext & - // Epoch Cache - EpochContext & - // SSZ ops - ITreeBacked & - // Beacon State interface - T; - -export function createCachedBeaconState( - chainForkConfig: IChainForkConfig, - state: TreeBacked, - opts?: EpochContextOpts -): CachedBeaconState { - const config = createIBeaconConfig(chainForkConfig, state.genesisValidatorsRoot); - - let cachedPreviousParticipation, cachedCurrentParticipation; - const forkName = config.getForkName(state.slot); - const epochCtx = EpochContext.createFromState(config, state, opts); - let cachedInactivityScores: MutableVector; - if (forkName === ForkName.phase0) { - // TODO: More efficient way of getting the length? - const validatorCount = state.validators.length; - // Can these arrays be zero-ed for phase0? Are they actually used? - cachedPreviousParticipation = MutableVector.from(newFilledArray(validatorCount, 0)); - cachedCurrentParticipation = MutableVector.from(newFilledArray(validatorCount, 0)); - cachedInactivityScores = MutableVector.empty(); - } else { - const altairState = (state as unknown) as TreeBacked; - cachedPreviousParticipation = MutableVector.from( - Array.from(readonlyValues(altairState.previousEpochParticipation)) - ); - cachedCurrentParticipation = MutableVector.from(Array.from(readonlyValues(altairState.currentEpochParticipation))); - cachedInactivityScores = MutableVector.from(readonlyValues(altairState.inactivityScores)); - } - return new Proxy( - new BeaconStateContext( - state.type as ContainerType, - state.tree, - cachedPreviousParticipation, - cachedCurrentParticipation, - cachedInactivityScores, - epochCtx - ), - (CachedBeaconStateProxyHandler as unknown) as ProxyHandler> - ) as CachedBeaconState; -} - -/** - * Cache useful data associated to a specific state. - * Optimize processing speed of block processing + gossip validation while having a low memory cost. - * - * Previously BeaconStateContext included: - * ```ts - * validators: CachedValidatorList & T["validators"]; - * balances: CachedBalanceList & T["balances"]; - * inactivityScores: CachedInactivityScoreList & List; - * ``` - * - * Those caches where removed since they are no strictly necessary to make the epoch transition faster, - * but have a high memory cost. Note that all data was duplicated between the Tree and MutableVector. - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with StructBacked validator objects for fast accessing and iteration - * - * ### validators - * state.validators is the heaviest data structure in the state. As TreeBacked, the leafs account for 91% with - * 200_000 validators. It requires ~ 2_000_000 Uint8Array instances with total memory of ~ 400MB. - * However its contents don't change very often. Validators only change when; - * - they first deposit - * - they dip from 32 effective balance to 31 (pretty much only when inactive for very long, or slashed) - * - they activate (once) - * - they exit (once) - * - they get slashed (max once) - * - * ### balances - * The balances array completely changes at the epoch boundary, where almost all the validator balances - * are updated. However it may have tiny changes during block processing if: - * - On a valid deposit - * - Validator gets slashed - * - On altair, the block proposer. Optimized to only happen once per block - * - * ### inactivityScores - * inactivityScores can be changed only: - * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not - * a lot on normal network conditions. - * - During block processing, when a validator joins a new 0 entry is pushed - * - * RESULT: Don't keep a duplicated structure around always. During block processing just push to the tree. During - * epoch processing some temporary flat structures are computed but dropped after processing the epoch. - */ -export class BeaconStateContext { - config: IBeaconConfig; - /** - * Epoch cache: Caches constant data through the epoch: @see EpochContext - * - Proposer indexes: 32 x Number - * - Shufflings: 3 x $VALIDATOR_COUNT x Number - */ - epochCtx: EpochContext; - /** The BeaconState ssz type */ - type: ContainerType; - /** The original BeaconState as a Tree */ - tree: Tree; - /** - * Returns a BalanceList instance with some convenient methods to work with Tree more efficiently. - * Notice that we want to work with state.balanceList instead of state.balances. - * - * The balances array completely changes at the epoch boundary, where almost all the validator balances - * are updated. However it may have tiny changes during block processing if: - * - On a valid deposit - * - Validator gets slashed? - * - On altair, the block proposer - * - */ - balanceList: BalanceList; - /** - * Returns a Proxy to CachedEpochParticipation - * - * Stores state.previousEpochParticipation in two duplicated forms (both structures are structurally shared): - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with each validator participation flags (uint8) in object form - * - * epochParticipation changes continuously through the epoch for each partipation bit of each valid attestation in the state. - * The entire structure is dropped after two epochs. - * - * TODO: Consider representing participation as a uint8 always, and have a fast transformation fuction with precomputed values. - * Here using a Uint8Array is probably the most efficient way of representing this structure. Then we only need a way to get - * and set the values fast to the tree. Maybe batching? - */ - previousEpochParticipation: CachedEpochParticipation & List; - /** Same as previousEpochParticipation */ - currentEpochParticipation: CachedEpochParticipation & List; - /** - * Returns a Proxy to CachedInactivityScoreList - * - * Stores state.inactivityScores in two duplicated forms (both structures are structurally shared): - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with a uint64 for each validator - * - * inactivityScores can be changed only: - * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not - * a lot on normal network conditions. - * - During block processing, when a validator joins a new 0 entry is pushed - * - * TODO: Don't keep a duplicated structure around always. During block processing just push to the tree, - * and maybe batch the changes. Then on process_inactivity_updates() compute the total deltas, and depending - * on the number of changes convert tree to array, apply diff, write to tree again. Or if there are just a few - * changes update the tree directly. - */ - inactivityScores: CachedInactivityScoreList & List; - - constructor( - type: ContainerType, - tree: Tree, - previousEpochParticipationCache: MutableVector, - currentEpochParticipationCache: MutableVector, - inactivityScoresCache: MutableVector, - epochCtx: EpochContext - ) { - this.config = epochCtx.config; - this.type = type; - this.tree = tree; - this.epochCtx = epochCtx; - this.balanceList = new BalanceList( - this.type.fields["balances"] as Number64ListType, - this.type.tree_getProperty(this.tree, "balances") as Tree - ); - this.previousEpochParticipation = (new Proxy( - new CachedEpochParticipation({ - type: this.type.fields["previousEpochParticipation"] as BasicListType>, - tree: this.type.tree_getProperty(this.tree, "previousEpochParticipation") as Tree, - persistent: previousEpochParticipationCache, - }), - CachedEpochParticipationProxyHandler - ) as unknown) as CachedEpochParticipation & List; - this.currentEpochParticipation = (new Proxy( - new CachedEpochParticipation({ - type: this.type.fields["currentEpochParticipation"] as BasicListType>, - tree: this.type.tree_getProperty(this.tree, "currentEpochParticipation") as Tree, - persistent: currentEpochParticipationCache, - }), - CachedEpochParticipationProxyHandler - ) as unknown) as CachedEpochParticipation & List; - this.inactivityScores = (new Proxy( - new CachedInactivityScoreList( - this.type.fields["inactivityScores"] as BasicListType>, - this.type.tree_getProperty(this.tree, "inactivityScores") as Tree, - inactivityScoresCache - ), - CachedInactivityScoreListProxyHandler - ) as unknown) as CachedInactivityScoreList & List; - } - - clone(): CachedBeaconState { - return new Proxy( - new BeaconStateContext( - this.type, - this.tree.clone(), - this.previousEpochParticipation.persistent.clone(), - this.currentEpochParticipation.persistent.clone(), - this.inactivityScores.persistent.clone(), - this.epochCtx.copy() - ), - (CachedBeaconStateProxyHandler as unknown) as ProxyHandler> - ) as CachedBeaconState; - } - - /** - * Toggle all `MutableVector` caches to use `TransientVector` - */ - setStateCachesAsTransient(): void { - this.previousEpochParticipation.persistent.asTransient(); - this.currentEpochParticipation.persistent.asTransient(); - this.inactivityScores.persistent.asTransient(); - } - - /** - * Toggle all `MutableVector` caches to use `PersistentVector` - */ - setStateCachesAsPersistent(): void { - this.previousEpochParticipation.persistent.asPersistent(); - this.currentEpochParticipation.persistent.asPersistent(); - this.inactivityScores.persistent.asPersistent(); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedBeaconStateProxyHandler: ProxyHandler> = { - get(target: CachedBeaconState, key: string): unknown { - if (key === "balanceList") { - return target.balanceList; - } else if (key === "previousEpochParticipation") { - return target.previousEpochParticipation; - } else if (key === "currentEpochParticipation") { - return target.currentEpochParticipation; - } else if (key === "inactivityScores") { - return target.inactivityScores; - } else if (target.type.fields[key] !== undefined) { - const propType = target.type.fields[key]; - const propValue = target.type.tree_getProperty(target.tree, key); - if (!isCompositeType(propType)) { - return propValue; - } else { - return propType.createTreeBacked(propValue as Tree); - } - } else if (key in target.epochCtx) { - return target.epochCtx[key as keyof EpochContext]; - } else if (key in target) { - return target[key as keyof CachedBeaconState]; - } else { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked]; - } - } - return undefined; - }, - set(target: CachedBeaconState, key: string, value: unknown): boolean { - if (key === "validators") { - throw new Error("Cannot set validators"); - } else if (key === "balanceList" || key === "balances") { - throw new Error("Cannot set either balanceList or balances"); - } else if (key === "previousEpochParticipation") { - throw new Error("Cannot set previousEpochParticipation"); - } else if (key === "currentEpochParticipation") { - throw new Error("Cannot set currentEpochParticipation"); - } else if (key === "inactivityScores") { - throw new Error("Cannot set inactivityScores"); - } else if (target.type.fields[key] !== undefined) { - const propType = target.type.fields[key]; - if (!isCompositeType(propType)) { - return target.type.tree_setProperty(target.tree, key, value); - } else { - if (isTreeBacked(value)) { - return target.type.tree_setProperty(target.tree, key, value.tree); - } else { - return target.type.tree_setProperty( - target.tree, - key, - propType.struct_convertToTree((value as unknown) as CompositeValue) - ); - } - } - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts b/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts deleted file mode 100644 index 4126d6643c99..000000000000 --- a/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts +++ /dev/null @@ -1,119 +0,0 @@ -import {BasicListType, List, TreeBacked} from "@chainsafe/ssz"; -import {ParticipationFlags, Uint8} from "@chainsafe/lodestar-types"; -import {MutableVector, PersistentVector, TransientVector} from "@chainsafe/persistent-ts"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {unsafeUint8ArrayToTree} from "../util/unsafeUint8ArrayToTree"; - -interface ICachedEpochParticipationOpts { - type?: BasicListType>; - tree?: Tree; - persistent: MutableVector; -} - -export class CachedEpochParticipation implements List { - [index: number]: ParticipationFlags; - type?: BasicListType>; - tree?: Tree; - persistent: MutableVector; - - constructor(opts: ICachedEpochParticipationOpts) { - this.type = opts.type; - this.tree = opts.tree; - this.persistent = opts.persistent; - } - - get length(): number { - return this.persistent.length; - } - - get(index: number): ParticipationFlags | undefined { - return this.persistent.get(index) ?? undefined; - } - - set(index: number, value: ParticipationFlags): void { - this.persistent.set(index, value); - if (this.type && this.tree) this.type.tree_setProperty(this.tree, index, value); - } - - updateAllStatus(data: PersistentVector | TransientVector): void { - this.persistent.vector = data; - - if (this.type && this.tree) { - const packedData = new Uint8Array(data.length); - data.forEach((d, i) => (packedData[i] = d)); - this.tree.rootNode = unsafeUint8ArrayToTree(packedData, this.type.getChunkDepth()); - this.type.tree_setLength(this.tree, data.length); - } - } - - push(value: ParticipationFlags): number { - this.persistent.push(value); - if (this.type && this.tree) this.type.tree_push(this.tree, value); - return this.persistent.length; - } - - pop(): ParticipationFlags { - const popped = this.persistent.pop(); - if (this.type && this.tree) this.type.tree_pop(this.tree); - if (popped === undefined) return (undefined as unknown) as ParticipationFlags; - return popped; - } - - *[Symbol.iterator](): Iterator { - for (const data of this.persistent) { - yield data; - } - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: ParticipationFlags, index: number, list: this) => boolean): ParticipationFlags | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: ParticipationFlags, index: number, list: this) => boolean): number { - return -1; - } - - forEach(fn: (value: ParticipationFlags, index: number, list: this) => void): void { - this.persistent.forEach((value, index) => (fn as (value: ParticipationFlags, index: number) => void)(value, index)); - } - - map(fn: (value: ParticipationFlags, index: number) => T): T[] { - return this.persistent.map((value, index) => fn(value, index)); - } - - forEachStatus(fn: (value: ParticipationFlags, index: number, list: this) => void): void { - this.persistent.forEach(fn as (t: ParticipationFlags, i: number) => void); - } - - mapStatus(fn: (value: ParticipationFlags, index: number) => T): T[] { - return this.persistent.map((value, index) => fn(value, index)); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedEpochParticipationProxyHandler: ProxyHandler = { - get(target: CachedEpochParticipation, key: PropertyKey): unknown { - if (!Number.isNaN(Number(String(key)))) { - return target.get(key as number); - } else if (target[key as keyof CachedEpochParticipation] !== undefined) { - return target[key as keyof CachedEpochParticipation]; - } else { - if (target.type && target.tree) { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked>]; - } - } - return undefined; - } - }, - set(target: CachedEpochParticipation, key: PropertyKey, value: ParticipationFlags): boolean { - if (!Number.isNaN(Number(key))) { - target.set(key as number, value); - return true; - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts b/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts deleted file mode 100644 index dbea96d767a4..000000000000 --- a/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts +++ /dev/null @@ -1,97 +0,0 @@ -import {BasicListType, List, TreeBacked} from "@chainsafe/ssz"; -import {Number64} from "@chainsafe/lodestar-types"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; - -/** - * Inactivity score implementation that synchronizes changes between two underlying implementations: - * an immutable-js-style backing and a merkle tree backing - */ -export class CachedInactivityScoreList implements List { - [index: number]: Number64; - tree: Tree; - type: BasicListType>; - persistent: MutableVector; - - constructor(type: BasicListType>, tree: Tree, persistent: MutableVector) { - this.type = type; - this.tree = tree; - this.persistent = persistent; - } - - get length(): number { - return this.persistent.length; - } - - get(index: number): Number64 | undefined { - return this.persistent.get(index) ?? undefined; - } - - set(index: number, value: Number64): void { - this.persistent.set(index, value); - this.type.tree_setProperty(this.tree, index, value); - } - - setMultiple(newValues: Map): void { - // TODO: based on newValues.size to determine we build the tree from scratch or not - for (const [index, value] of newValues.entries()) { - this.set(index, value); - } - } - - push(value: Number64): number { - this.persistent.push(value); - return this.type.tree_push(this.tree, value); - } - - pop(): Number64 { - this.type.tree_pop(this.tree); - return this.persistent.pop() as Number64; - } - - *[Symbol.iterator](): Iterator { - yield* this.persistent[Symbol.iterator](); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: Number64, index: number, list: this) => boolean): Number64 | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: Number64, index: number, list: this) => boolean): number { - return -1; - } - - forEach(fn: (value: Number64, index: number, list: this) => void): void { - this.persistent.forEach(fn as (value: Number64, index: number) => void); - } - - map(fn: (value: Number64, index: number) => T): T[] { - return this.persistent.map(fn); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedInactivityScoreListProxyHandler: ProxyHandler = { - get(target: CachedInactivityScoreList, key: PropertyKey): unknown { - if (!Number.isNaN(Number(String(key)))) { - return target.get(key as number); - } else if (target[key as keyof CachedInactivityScoreList] !== undefined) { - return target[key as keyof CachedInactivityScoreList]; - } else { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked>]; - } - return undefined; - } - }, - set(target: CachedInactivityScoreList, key: PropertyKey, value: Number64): boolean { - if (!Number.isNaN(Number(key))) { - target.set(key as number, value); - return true; - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts b/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts index b14b2d4797f6..63c6924397f2 100644 --- a/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts +++ b/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts @@ -1,3 +1,6 @@ +import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; + /** * Alias to allow easier refactoring. * TODO: Estimate the risk of future proof of MAX_EFFECTIVE_BALANCE_INCREMENT < 255 @@ -21,3 +24,17 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef return new Uint8Array(byteLen); } + +/** + * Shows how EffectiveBalanceIncrements is meant to be populated. + * In practice this function should not be used, since it more efficient to loop the validators array once and do + * more tasks than only populating effectiveBalanceIncrements + */ +export function getEffectiveBalanceIncrements(state: BeaconStateAllForks): EffectiveBalanceIncrements { + const validatorsArr = state.validators.getAllReadonlyValues(); + const effectiveBalanceIncrements = new Uint8Array(validatorsArr.length); + for (let i = 0; i < validatorsArr.length; i++) { + effectiveBalanceIncrements[i] = Math.floor(validatorsArr[i].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + } + return effectiveBalanceIncrements; +} diff --git a/packages/beacon-state-transition/src/cache/epochContext.ts b/packages/beacon-state-transition/src/cache/epochContext.ts index d0d121fa55ef..d5428cd90df9 100644 --- a/packages/beacon-state-transition/src/cache/epochContext.ts +++ b/packages/beacon-state-transition/src/cache/epochContext.ts @@ -1,19 +1,8 @@ -import {BitList, List, readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; import bls, {CoordType} from "@chainsafe/bls"; +import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@chainsafe/lodestar-types"; +import {createIBeaconConfig, IBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config"; import { - BLSSignature, - CommitteeIndex, - Epoch, - Slot, - ValidatorIndex, - phase0, - allForks, - Number64, - altair, - SyncPeriod, -} from "@chainsafe/lodestar-types"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import { + ATTESTATION_SUBNET_COUNT, EFFECTIVE_BALANCE_INCREMENT, FAR_FUTURE_EPOCH, GENESIS_EPOCH, @@ -22,21 +11,20 @@ import { WEIGHT_DENOMINATOR, } from "@chainsafe/lodestar-params"; import {LodestarError} from "@chainsafe/lodestar-utils"; - import { computeActivationExitEpoch, computeEpochAtSlot, - computeProposers, computeStartSlotAtEpoch, getChurnLimit, isActiveValidator, isAggregatorFromCommitteeLength, - zipIndexesCommitteeBits, + computeProposers, computeSyncPeriodAtEpoch, } from "../util"; import {computeEpochShuffling, IEpochShuffling} from "../util/epochShuffling"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements"; import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache"; +import {BeaconStateAllForks, BeaconStateAltair} from "./types"; import { computeSyncCommitteeCache, getSyncCommitteeCache, @@ -48,11 +36,15 @@ import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../ut /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); +export type EpochContextImmutableData = { + config: IBeaconConfig; + pubkey2index: PubkeyIndexMap; + index2pubkey: Index2PubkeyCache; +}; + export type EpochContextOpts = { - pubkey2index?: PubkeyIndexMap; - index2pubkey?: Index2PubkeyCache; - skipSyncPubkeys?: boolean; skipSyncCommitteeCache?: boolean; + skipSyncPubkeys?: boolean; }; /** @@ -149,16 +141,9 @@ export class EpochContext { */ exitQueueChurn: number; - /** - * Returns a SyncCommitteeCache. (Note: phase0 has no sync committee, and returns an empty cache) - * - validatorIndices (of the committee members) - * - validatorIndexMap: Map of ValidatorIndex -> syncCommitteeIndexes - * - * The syncCommittee is immutable and changes as a whole every ~ 27h. - * It contains fixed 512 members so it's rather small. - */ + /** TODO: Indexed SyncCommitteeCache */ currentSyncCommitteeIndexed: SyncCommitteeCache; - /** Same as currentSyncCommitteeIndexed */ + /** TODO: Indexed SyncCommitteeCache */ nextSyncCommitteeIndexed: SyncCommitteeCache; // TODO: Helper stats @@ -213,9 +198,13 @@ export class EpochContext { * * SLOW CODE - 🐢 */ - static createFromState(config: IBeaconConfig, state: allForks.BeaconState, opts?: EpochContextOpts): EpochContext { - const pubkey2index = opts?.pubkey2index || new PubkeyIndexMap(); - const index2pubkey = opts?.index2pubkey || ([] as Index2PubkeyCache); + static createFromState( + state: BeaconStateAllForks, + {config, pubkey2index, index2pubkey}: EpochContextImmutableData, + opts?: EpochContextOpts + ): EpochContext { + // syncPubkeys here to ensure EpochContextImmutableData is popualted before computing the rest of caches + // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache if (!opts?.skipSyncPubkeys) { syncPubkeys(state, pubkey2index, index2pubkey); } @@ -229,7 +218,7 @@ export class EpochContext { let exitQueueEpoch = computeActivationExitEpoch(currentEpoch); let exitQueueChurn = 0; - const validators = readonlyValuesListOfLeafNodeStruct(state.validators); + const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); @@ -297,7 +286,7 @@ export class EpochContext { let nextSyncCommitteeIndexed: SyncCommitteeCache; // Allow to skip populating sync committee for initializeBeaconStateFromEth1() if (afterAltairFork && !opts?.skipSyncCommitteeCache) { - const altairState = state as altair.BeaconState; + const altairState = state as BeaconStateAltair; currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkey2index); nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkey2index); } else { @@ -337,7 +326,7 @@ export class EpochContext { syncParticipantReward, syncProposerReward, baseRewardPerIncrement, - totalActiveBalanceIncrements: totalActiveBalanceIncrements, + totalActiveBalanceIncrements, churnLimit, exitQueueEpoch, exitQueueChurn, @@ -351,7 +340,7 @@ export class EpochContext { /** * Copies a given EpochContext while avoiding copying its immutable parts. */ - copy(): EpochContext { + clone(): EpochContext { // warning: pubkey cache is not copied, it is shared, as eth1 is not expected to reorder validators. // Shallow copy all data from current epoch context to the next // All data is completely replaced, or only-appended @@ -388,7 +377,7 @@ export class EpochContext { * new epoch. */ afterProcessEpoch( - state: allForks.BeaconState, + state: BeaconStateAllForks, epochProcess: { nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; nextEpochTotalActiveBalanceByIncrement: number; @@ -398,6 +387,7 @@ export class EpochContext { this.currentShuffling = this.nextShuffling; const currEpoch = this.currentShuffling.epoch; const nextEpoch = currEpoch + 1; + this.nextShuffling = computeEpochShuffling(state, epochProcess.nextEpochShufflingActiveValidatorIndices, nextEpoch); this.proposers = computeProposers(state, this.currentShuffling, this.effectiveBalanceIncrements); @@ -467,6 +457,16 @@ export class EpochContext { return this.getShufflingAtEpoch(epoch).committeesPerSlot; } + /** + * Compute the correct subnet for a slot/committee index + */ + computeSubnetForSlot(slot: number, committeeIndex: number): number { + const slotsSinceEpochStart = slot % SLOTS_PER_EPOCH; + const committeesPerSlot = this.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); + const committeesSinceEpochStart = committeesPerSlot * slotsSinceEpochStart; + return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; + } + getBeaconProposer(slot: Slot): ValidatorIndex { const epoch = computeEpochAtSlot(slot); if (epoch !== this.currentShuffling.epoch) { @@ -483,23 +483,17 @@ export class EpochContext { getIndexedAttestation(attestation: phase0.Attestation): phase0.IndexedAttestation { const {aggregationBits, data} = attestation; const committeeIndices = this.getBeaconCommittee(data.slot, data.index); - const attestingIndices = zipIndexesCommitteeBits(committeeIndices, aggregationBits); + const attestingIndices = aggregationBits.intersectValues(committeeIndices); // sort in-place attestingIndices.sort((a, b) => a - b); return { - attestingIndices: attestingIndices as List, + attestingIndices: attestingIndices, data: data, signature: attestation.signature, }; } - getAttestingIndices(data: phase0.AttestationData, bits: BitList): ValidatorIndex[] { - const committeeIndices = this.getBeaconCommittee(data.slot, data.index); - const validatorIndices = zipIndexesCommitteeBits(committeeIndices, bits); - return validatorIndices; - } - getCommitteeAssignments( epoch: Epoch, requestedValidatorIndices: ValidatorIndex[] @@ -554,7 +548,7 @@ export class EpochContext { const committee = this.getBeaconCommittee(slot, i); if (committee.includes(validatorIndex)) { return { - validators: committee as List, + validators: committee, committeeIndex: i, slot, }; @@ -591,18 +585,6 @@ export class EpochContext { } } - effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { - if (index >= this.effectiveBalanceIncrements.length) { - // Clone and extend effectiveBalanceIncrements - const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - // Note: getEffectiveBalanceIncrementsWithLen() returns a Uint8Array larger than `index + 1` to reduce copy-ing - this.effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(index + 1); - this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); - } - - this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); - } - /** * Note: The range of slots a validator has to perform duties is off by one. * The previous slot wording means that if your validator is in a sync committee for a period that runs from slot @@ -634,15 +616,37 @@ export class EpochContext { this.currentSyncCommitteeIndexed = this.nextSyncCommitteeIndexed; this.nextSyncCommitteeIndexed = getSyncCommitteeCache(nextSyncCommitteeIndices); } + + /** On phase0 -> altair fork, set both current and nextSyncCommitteeIndexed */ + setSyncCommitteesIndexed(nextSyncCommitteeIndices: number[]): void { + this.nextSyncCommitteeIndexed = getSyncCommitteeCache(nextSyncCommitteeIndices); + this.currentSyncCommitteeIndexed = this.nextSyncCommitteeIndexed; + } + + effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { + if (index >= this.effectiveBalanceIncrements.length) { + // Clone and extend effectiveBalanceIncrements + const effectiveBalanceIncrements = this.effectiveBalanceIncrements; + this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } + + this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + } +} + +function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { + // TODO: Research what's the best number to minimize both memory cost and copy costs + return 1024 * Math.ceil(validatorCount / 1024); } // Copied from lodestar-api package to avoid depending on the package type AttesterDuty = { validatorIndex: ValidatorIndex; committeeIndex: CommitteeIndex; - committeeLength: Number64; - committeesAtSlot: Number64; - validatorCommitteeIndex: Number64; + committeeLength: number; + committeesAtSlot: number; + validatorCommitteeIndex: number; slot: Slot; }; @@ -657,3 +661,15 @@ type EpochContextErrorType = { }; export class EpochContextError extends LodestarError {} + +export function createEmptyEpochContextImmutableData( + chainConfig: IChainConfig, + state: Pick +): EpochContextImmutableData { + return { + config: createIBeaconConfig(chainConfig, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }; +} diff --git a/packages/beacon-state-transition/src/cache/epochProcess.ts b/packages/beacon-state-transition/src/cache/epochProcess.ts index 9fbf06c3f4af..9f33ea7fc607 100644 --- a/packages/beacon-state-transition/src/cache/epochProcess.ts +++ b/packages/beacon-state-transition/src/cache/epochProcess.ts @@ -1,15 +1,12 @@ -import {Epoch, ValidatorIndex, allForks, phase0} from "@chainsafe/lodestar-types"; +import {Epoch, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import { - EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkName, MAX_EFFECTIVE_BALANCE, } from "@chainsafe/lodestar-params"; -import {readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; -import {isActiveValidator, newZeroedArray} from "../util"; import { IAttesterStatus, createIAttesterStatus, @@ -23,9 +20,11 @@ import { FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus"; -import {CachedBeaconState} from "../cache/cachedBeaconState"; import {statusProcessEpoch} from "../phase0/epoch/processPendingAttestations"; -import {computeBaseRewardPerIncrement} from "../util/syncCommittee"; +import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from ".."; +import {computeBaseRewardPerIncrement} from "../util/altair"; + +/* eslint-disable @typescript-eslint/naming-convention */ /** * EpochProcess is the parent object of: @@ -39,7 +38,7 @@ import {computeBaseRewardPerIncrement} from "../util/syncCommittee"; * - Only loop state.validators once for all `process_*` fns * - Only loop status array once */ -export type EpochProcess = { +export interface EpochProcess { prevEpoch: Epoch; currentEpoch: Epoch; /** @@ -54,6 +53,16 @@ export type EpochProcess = { headStakeByIncrement: number; }; currEpochUnslashedTargetStakeByIncrement: number; + + /** + * Validator indices that are either + * - active in previous epoch + * - slashed and not yet withdrawable + * + * getRewardsAndPenalties() and processInactivityUpdates() iterate this list + */ + eligibleValidatorIndices: ValidatorIndex[]; + /** * Indices which will receive the slashing penalty * ``` @@ -76,6 +85,7 @@ export type EpochProcess = { * that happen to be in the same committee, which is very unlikely. */ indicesToSlash: ValidatorIndex[]; + /** * Indices of validators that just joinned and will be eligible for the active queue. * ``` @@ -89,6 +99,7 @@ export type EpochProcess = { * For mainnet spec = 512 */ indicesEligibleForActivationQueue: ValidatorIndex[]; + /** * Indices of validators that may become active once churn and finaly allow. * ``` @@ -98,6 +109,7 @@ export type EpochProcess = { * For less than 327680 validators, churnLimit = 4 (minimum possible), so max processed is 4. */ indicesEligibleForActivation: ValidatorIndex[]; + /** * Indices of validators that will be ejected due to low balance. * ``` @@ -108,8 +120,23 @@ export type EpochProcess = { */ indicesToEject: ValidatorIndex[]; + /** + * Pre-computes status flags for faster checking of statuses during epoch transition. + * Spec requires some reward or penalty to apply to + * - eligible validators + * - un-slashed validators + * - prev attester flag set + * With a status flag to check this conditions at once we just have to mask with an OR of the conditions. + */ statuses: IAttesterStatus[]; + + /** + * balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates(). + * processRewardsAndPenalties() already has a regular Javascript array of balances. + * Then processEffectiveBalanceUpdates() needs to iterate all balances so it can re-use the array pre-computed previously. + */ balances?: number[]; + /** * Active validator indices for currentEpoch + 2. * This is only used in `afterProcessEpoch` to compute epoch shuffling, it's not efficient to calculate it at that time @@ -120,6 +147,7 @@ export type EpochProcess = { * | afterEpochProcess | read it | */ nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + /** * Altair specific, this is total active balances for the next epoch. * This is only used in `afterProcessEpoch` to compute base reward and sync participant reward. @@ -138,19 +166,20 @@ export type EpochProcess = { * Used in `processEffectiveBalanceUpdates` to save one loop over validators after epoch process. */ isActiveNextEpoch: boolean[]; -}; +} -export function beforeProcessEpoch(state: CachedBeaconState): EpochProcess { +export function beforeProcessEpoch(state: CachedBeaconStateAllForks): EpochProcess { const {config, epochCtx} = state; const forkName = config.getForkName(state.slot); const currentEpoch = epochCtx.currentShuffling.epoch; const prevEpoch = epochCtx.previousShuffling.epoch; - // active validator indices for nextShuffling is ready, we want to precalculate for the one after that - const nextShufflingEpoch = currentEpoch + 2; const nextEpoch = currentEpoch + 1; + // active validator indices for nextShuffling is ready, we want to precalculate for the one after that + const nextEpoch2 = currentEpoch + 2; const slashingsEpoch = currentEpoch + intDiv(EPOCHS_PER_SLASHINGS_VECTOR, 2); + const eligibleValidatorIndices: ValidatorIndex[] = []; const indicesToSlash: ValidatorIndex[] = []; const indicesEligibleForActivationQueue: ValidatorIndex[] = []; const indicesEligibleForActivation: ValidatorIndex[] = []; @@ -165,13 +194,13 @@ export function beforeProcessEpoch(state: Cached // To optimize memory each validator node in `state.validators` is represented with a special node type // `BranchNodeStruct` that represents the data as struct internally. This utility grabs the struct data directrly // from the nodes without any extra transformation. The returned `validators` array contains native JS objects. - const validators = readonlyValuesListOfLeafNodeStruct(state.validators); + const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; // Clone before being mutated in processEffectiveBalanceUpdates epochCtx.beforeEpochTransition(); - const effectiveBalancesByIncrements = newZeroedArray(validators.length); + const effectiveBalancesByIncrements = epochCtx.effectiveBalanceIncrements; for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -185,19 +214,25 @@ export function beforeProcessEpoch(state: Cached status.flags |= FLAG_UNSLASHED; } - const activePrev = isActiveValidator(validator, prevEpoch); - isActivePrevEpoch.push(activePrev); - if (activePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) { + const {activationEpoch, exitEpoch} = validator; + const isActivePrev = activationEpoch <= prevEpoch && prevEpoch < exitEpoch; + const isActiveCurr = activationEpoch <= currentEpoch && currentEpoch < exitEpoch; + const isActiveNext = activationEpoch <= nextEpoch && nextEpoch < exitEpoch; + const isActiveNext2 = activationEpoch <= nextEpoch2 && nextEpoch2 < exitEpoch; + + isActivePrevEpoch.push(isActivePrev); + + // Both active validators and slashed-but-not-yet-withdrawn validators are eligible to receive penalties. + // This is done to prevent self-slashing from being a way to escape inactivity leaks. + // TODO: Consider using an array of `eligibleValidatorIndices: number[]` + if (isActivePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) { + eligibleValidatorIndices.push(i); status.flags |= FLAG_ELIGIBLE_ATTESTER; } - const active = isActiveValidator(validator, currentEpoch); - // We track effectiveBalanceByIncrement as ETH to fit total network balance in a JS number (53 bits) - const effectiveBalanceByIncrement = Math.floor(validator.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); - effectiveBalancesByIncrements[i] = effectiveBalanceByIncrement; - if (active) { + if (isActiveCurr) { status.active = true; - totalActiveStakeByIncrement += effectiveBalanceByIncrement; + totalActiveStakeByIncrement += effectiveBalancesByIncrements[i]; } // To optimize process_registry_updates(): @@ -249,9 +284,9 @@ export function beforeProcessEpoch(state: Cached statuses.push(status); - isActiveNextEpoch.push(isActiveValidator(validator, nextEpoch)); + isActiveNextEpoch.push(isActiveNext); - if (isActiveValidator(validator, nextShufflingEpoch)) { + if (isActiveNext2) { nextEpochShufflingActiveValidatorIndices.push(i); } } @@ -272,44 +307,44 @@ export function beforeProcessEpoch(state: Cached ); if (forkName === ForkName.phase0) { - const statePhase0 = (state as unknown) as CachedBeaconState; statusProcessEpoch( - statePhase0, + state as CachedBeaconStatePhase0, statuses, - statePhase0.previousEpochAttestations, + (state as CachedBeaconStatePhase0).previousEpochAttestations.getAllReadonly(), prevEpoch, FLAG_PREV_SOURCE_ATTESTER, FLAG_PREV_TARGET_ATTESTER, FLAG_PREV_HEAD_ATTESTER ); statusProcessEpoch( - statePhase0, + state as CachedBeaconStatePhase0, statuses, - statePhase0.currentEpochAttestations, + (state as CachedBeaconStatePhase0).currentEpochAttestations.getAllReadonly(), currentEpoch, FLAG_CURR_SOURCE_ATTESTER, FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER ); } else { - state.previousEpochParticipation.forEachStatus((participationFlags, i) => { + const previousEpochParticipation = (state as CachedBeaconStateAltair).previousEpochParticipation.getAll(); + for (let i = 0; i < previousEpochParticipation.length; i++) { const status = statuses[i]; // this is required to pass random spec tests in altair if (isActivePrevEpoch[i]) { - status.flags |= - // FLAG_PREV are indexes [0,1,2] - status.flags |= participationFlags; + // FLAG_PREV are indexes [0,1,2] + status.flags |= previousEpochParticipation[i]; } - }); - state.currentEpochParticipation.forEachStatus((participationFlags, i) => { + } + + const currentEpochParticipation = (state as CachedBeaconStateAltair).currentEpochParticipation.getAll(); + for (let i = 0; i < currentEpochParticipation.length; i++) { const status = statuses[i]; // this is required to pass random spec tests in altair if (status.active) { - status.flags |= - // FLAG_CURR are indexes [3,4,5], so shift by 3 - status.flags |= participationFlags << 3; + // FLAG_PREV are indexes [3,4,5], so shift by 3 + status.flags |= currentEpochParticipation[i] << 3; } - }); + } } let prevSourceUnslStake = 0; @@ -358,6 +393,7 @@ export function beforeProcessEpoch(state: Cached headStakeByIncrement: prevHeadUnslStake, }, currEpochUnslashedTargetStakeByIncrement: currTargetUnslStake, + eligibleValidatorIndices, indicesToSlash, indicesEligibleForActivationQueue, indicesEligibleForActivation, @@ -367,5 +403,8 @@ export function beforeProcessEpoch(state: Cached nextEpochTotalActiveBalanceByIncrement: 0, isActiveNextEpoch, statuses, + + // Will be assigned in processRewardsAndPenalties() + balances: undefined, }; } diff --git a/packages/beacon-state-transition/src/cache/pubkeyCache.ts b/packages/beacon-state-transition/src/cache/pubkeyCache.ts index ce623ae0b244..0bf64066bcbb 100644 --- a/packages/beacon-state-transition/src/cache/pubkeyCache.ts +++ b/packages/beacon-state-transition/src/cache/pubkeyCache.ts @@ -1,6 +1,6 @@ import bls, {CoordType, PublicKey} from "@chainsafe/bls"; -import {allForks, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {ByteVector} from "@chainsafe/ssz"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "./types"; export type Index2PubkeyCache = PublicKey[]; @@ -14,7 +14,7 @@ type PubkeyHex = string; * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(hex: ByteVector | Uint8Array | string): string { +function toMemoryEfficientHexStr(hex: Uint8Array | string): string { if (typeof hex === "string") { if (hex.startsWith("0x")) { hex = hex.slice(2); @@ -36,7 +36,7 @@ export class PubkeyIndexMap { /** * Must support reading with string for API support where pubkeys are already strings */ - get(key: ByteVector | Uint8Array | PubkeyHex): ValidatorIndex | undefined { + get(key: Uint8Array | PubkeyHex): ValidatorIndex | undefined { return this.map.get(toMemoryEfficientHexStr(key)); } @@ -53,7 +53,7 @@ export class PubkeyIndexMap { * If pubkey caches are empty: SLOW CODE - 🐢 */ export function syncPubkeys( - state: allForks.BeaconState, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap, index2pubkey: Index2PubkeyCache ): void { @@ -66,7 +66,7 @@ export function syncPubkeys( const newCount = state.validators.length; for (let i = pubkey2index.size; i < newCount; i++) { - const pubkey = validators[i].pubkey.valueOf() as Uint8Array; + const pubkey = validators.getReadonly(i).pubkey; pubkey2index.set(pubkey, i); // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. // Afterwards any public key is the state consider validated. diff --git a/packages/beacon-state-transition/src/cache/stateCache.ts b/packages/beacon-state-transition/src/cache/stateCache.ts new file mode 100644 index 000000000000..33c7ee52c4c4 --- /dev/null +++ b/packages/beacon-state-transition/src/cache/stateCache.ts @@ -0,0 +1,151 @@ +import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {EpochContext, EpochContextImmutableData, EpochContextOpts} from "./epochContext"; +import {BeaconStatePhase0, BeaconStateAltair, BeaconStateBellatrix, BeaconStateAllForks} from "./types"; + +export type BeaconStateCache = { + config: IBeaconConfig; + epochCtx: EpochContext; +}; + +/** + * `BeaconState` with various caches + * + * Currently contains the following: + * - The full list of network params, ssz types, and fork schedule + * - The ssz type for the state + * - The full merkle tree representation of the state + * - A cache of shufflings, committees, proposers, expanded pubkeys + * - A flat copy of validators (for fast access/iteration) + * + * ### BeaconState data representation tradeoffs: + * + * Requirements of a BeaconState: + * - Block processing and epoch processing be performant to not block the node. This functions requires to iterate over + * very large arrays fast, while doing random mutations or big mutations. After them the state must be hashed. + * Processing times: (ideal / current / maximum) + * - block processing: 20ms / 200ms / 500ms + * - epoch processing: 200ms / 2s / 4s + * + * - BeaconState must be memory efficient. Data should only be represented once in a succint manner. Uint8Arrays are + * expensive, native types are not. + * - BeaconState must be hashed efficiently. Data must be merkelized before hashing so the conversion to merkelized + * must be fast or be done already. It must must persist a hashing cache that should be structurally shared between + * states for memory efficiency. + * - BeaconState raw data changes sparsingly, so it should be structurally shared between states for memory efficiency + * + * Summary of goals: + * - Structurally share data + hashing cache + * - Very fast read and iteration over large arrays + * - Fast bulk writes and somewhat fast single writes + * - Fast merkelization of data for hashing + * + * #### state.validators + * + * 91% of all memory merkelized is state.validators. In normal network conditions state.validators changes rarely. + * However for epoch processing the entire array must be iterated and read. So we need fast reads and slow writes. + * Tradeoffs to achieve that: + * - Represent leaf data with native JS types (deserialized form) + * - Use a single Tree for structurally sharing leaf data + hashing cache + * - Keep only the root cached on leaf nodes + * - Micro-optimizations (TODO): + * - Keep also the root of the node above pubkey and withdrawal creds. Will never change + * - Keep pubkey + withdrawal creds in the same Uint8Array + * - Have a global pubkey + withdrawal creds Uint8Array global cache, like with the index2pubkey cache + * + * ------------------ + * + * _Previous JSDocs for `BeaconStateContext`_ + * + * Cache useful data associated to a specific state. + * Optimize processing speed of block processing + gossip validation while having a low memory cost. + * + * Previously BeaconStateContext included: + * ```ts + * validators: CachedValidatorList & T["validators"]; + * balances: CachedBalanceList & T["balances"]; + * inactivityScores: CachedInactivityScoreList & Number64[]; + * ``` + * + * Those caches where removed since they are no strictly necessary to make the epoch transition faster, + * but have a high memory cost. Note that all data was duplicated between the Tree and MutableVector. + * 1. TreeView, for efficient hashing + * 2. MutableVector (persistent-ts) with StructBacked validator objects for fast accessing and iteration + * + * ### validators + * state.validators is the heaviest data structure in the state. As TreeView, the leafs account for 91% with + * 200_000 validators. It requires ~ 2_000_000 Uint8Array instances with total memory of ~ 400MB. + * However its contents don't change very often. Validators only change when; + * - they first deposit + * - they dip from 32 effective balance to 31 (pretty much only when inactive for very long, or slashed) + * - they activate (once) + * - they exit (once) + * - they get slashed (max once) + * + * ### balances + * The balances array completely changes at the epoch boundary, where almost all the validator balances + * are updated. However it may have tiny changes during block processing if: + * - On a valid deposit + * - Validator gets slashed + * - On altair, the block proposer. Optimized to only happen once per block + * + * ### epochParticipation + * epochParticipation changes continuously through the epoch for each partipation bit of each valid attestation in the state. + * The entire structure is dropped after two epochs. + * + * ### inactivityScores + * inactivityScores can be changed only: + * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not + * a lot on normal network conditions. + * - During block processing, when a validator joins a new 0 entry is pushed + * + * RESULT: Don't keep a duplicated structure around always. During block processing just push to the tree. During + * epoch processing some temporary flat structures are computed but dropped after processing the epoch. + */ +export type CachedBeaconState = T & BeaconStateCache; + +export type CachedBeaconStatePhase0 = CachedBeaconState; +export type CachedBeaconStateAltair = CachedBeaconState; +export type CachedBeaconStateBellatrix = CachedBeaconState; +export type CachedBeaconStateAllForks = CachedBeaconState; + +/** + * Create CachedBeaconState computing a new EpochContext instance + */ +export function createCachedBeaconState( + state: T, + immutableData: EpochContextImmutableData, + opts?: EpochContextOpts +): T & BeaconStateCache { + return getCachedBeaconState(state, { + config: immutableData.config, + epochCtx: EpochContext.createFromState(state, immutableData, opts), + }); +} + +/** + * Attach an already computed BeaconStateCache to a BeaconState object + */ +export function getCachedBeaconState( + state: T, + cache: BeaconStateCache +): T & BeaconStateCache { + const cachedState = state as T & BeaconStateCache; + cachedState.config = cache.config; + cachedState.epochCtx = cache.epochCtx; + + // Overwrite .clone function to preserve cache + // TreeViewDU.clone() creates a new object that does not have the attached cache + const viewDUClone = cachedState.clone.bind(cachedState); + + function clone(this: T & BeaconStateCache): T & BeaconStateCache { + const viewDUCloned = viewDUClone(); + return getCachedBeaconState(viewDUCloned, { + config: this.config, + epochCtx: this.epochCtx.clone(), + }) as T & BeaconStateCache; + } + + cachedState.clone = clone as typeof viewDUClone; + + return cachedState; +} diff --git a/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts b/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts index 05ab3b5f6a68..d48c4d13db52 100644 --- a/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts +++ b/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts @@ -1,5 +1,5 @@ -import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {PubkeyIndexMap} from "./pubkeyCache"; type SyncComitteeValidatorIndexMap = Map; @@ -36,7 +36,7 @@ export function getSyncCommitteeCache(validatorIndices: ValidatorIndex[]): SyncC } export function computeSyncCommitteeCache( - syncCommittee: altair.SyncCommittee, + syncCommittee: CompositeViewDU, pubkey2index: PubkeyIndexMap ): SyncCommitteeCache { const validatorIndices = computeSyncCommitteeIndices(syncCommittee, pubkey2index); @@ -74,13 +74,13 @@ export function computeSyncComitteeMap(syncCommitteeIndexes: ValidatorIndex[]): * Extract validator indices from current and next sync committee */ function computeSyncCommitteeIndices( - syncCommittee: altair.SyncCommittee, + syncCommittee: CompositeViewDU, pubkey2index: PubkeyIndexMap ): ValidatorIndex[] { const validatorIndices: ValidatorIndex[] = []; - const pubkeys = readonlyValues(syncCommittee.pubkeys); + const pubkeys = syncCommittee.pubkeys.getAllReadonly(); for (const pubkey of pubkeys) { - const validatorIndex = pubkey2index.get(pubkey.valueOf() as typeof pubkey); + const validatorIndex = pubkey2index.get(pubkey); if (validatorIndex === undefined) { throw Error(`SyncCommittee pubkey is unknown ${toHexString(pubkey)}`); } diff --git a/packages/beacon-state-transition/src/cache/types.ts b/packages/beacon-state-transition/src/cache/types.ts new file mode 100644 index 000000000000..56f91bfd9c9d --- /dev/null +++ b/packages/beacon-state-transition/src/cache/types.ts @@ -0,0 +1,13 @@ +import {ssz} from "@chainsafe/lodestar-types"; +import {CompositeViewDU} from "@chainsafe/ssz"; + +export type BeaconStatePhase0 = CompositeViewDU; +export type BeaconStateAltair = CompositeViewDU; +export type BeaconStateBellatrix = CompositeViewDU; + +// Union at the TreeViewDU level +// - Works well as function argument and as generic type for allForks functions +// +// Quasy equivalent to +// CompositeViewDU +export type BeaconStateAllForks = BeaconStatePhase0 | BeaconStateAltair | BeaconStateBellatrix; diff --git a/packages/beacon-state-transition/src/index.ts b/packages/beacon-state-transition/src/index.ts index 6ac6d1886f85..325d5298ace5 100644 --- a/packages/beacon-state-transition/src/index.ts +++ b/packages/beacon-state-transition/src/index.ts @@ -10,18 +10,24 @@ export * as phase0 from "./phase0"; export * as altair from "./altair"; export * as bellatrix from "./bellatrix"; export * as allForks from "./allForks"; -export {CachedBeaconState, createCachedBeaconState} from "./cache/cachedBeaconState"; export { CachedBeaconStatePhase0, CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks, - CachedBeaconStateAnyFork, + // Non-cached states + BeaconStatePhase0, + BeaconStateAltair, + BeaconStateBellatrix, + BeaconStateAllForks, } from "./types"; -export {EpochContext} from "./cache/epochContext"; +// Main state caches +export {createCachedBeaconState, BeaconStateCache} from "./cache/stateCache"; +export {EpochContext, EpochContextImmutableData, createEmptyEpochContextImmutableData} from "./cache/epochContext"; export {EpochProcess, beforeProcessEpoch} from "./cache/epochProcess"; -export {PubkeyIndexMap, Index2PubkeyCache} from "./cache/pubkeyCache"; +// Aux data-structures +export {PubkeyIndexMap, Index2PubkeyCache} from "./cache/pubkeyCache"; export {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsZeroed} from "./cache/effectiveBalanceIncrements"; diff --git a/packages/beacon-state-transition/src/phase0/block/index.ts b/packages/beacon-state-transition/src/phase0/block/index.ts index eb322b4e28a5..76da378f08da 100644 --- a/packages/beacon-state-transition/src/phase0/block/index.ts +++ b/packages/beacon-state-transition/src/phase0/block/index.ts @@ -24,6 +24,6 @@ export { export function processBlock(state: CachedBeaconStatePhase0, block: phase0.BeaconBlock, verifySignatures = true): void { processBlockHeader(state as CachedBeaconStateAllForks, block); processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); processOperations(state, block.body, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts index 044a2b28c568..b4bcef6151d1 100644 --- a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts @@ -24,7 +24,7 @@ export function processAttestation( validateAttestation(state as CachedBeaconStateAllForks, attestation); - const pendingAttestation = ssz.phase0.PendingAttestation.createTreeBackedFromStruct({ + const pendingAttestation = ssz.phase0.PendingAttestation.toViewDU({ data: data, aggregationBits: attestation.aggregationBits, inclusionDelay: slot - data.slot, @@ -96,10 +96,10 @@ export function validateAttestation(state: CachedBeaconStateAllForks, attestatio } const committee = epochCtx.getBeaconCommittee(data.slot, data.index); - if (attestation.aggregationBits.length !== committee.length) { + if (attestation.aggregationBits.bitLen !== committee.length) { throw new Error( "Attestation aggregation bits length does not match committee length: " + - `aggregationBitsLength=${attestation.aggregationBits.length} committeeLength=${committee.length}` + `aggregationBitsLength=${attestation.aggregationBits.bitLen} committeeLength=${committee.length}` ); } } diff --git a/packages/beacon-state-transition/src/phase0/block/processOperations.ts b/packages/beacon-state-transition/src/phase0/block/processOperations.ts index 659d6cfc0339..17b3e7abbe31 100644 --- a/packages/beacon-state-transition/src/phase0/block/processOperations.ts +++ b/packages/beacon-state-transition/src/phase0/block/processOperations.ts @@ -1,4 +1,3 @@ -import {List, readonlyValues} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; @@ -9,14 +8,6 @@ import {processAttestation} from "./processAttestation"; import {processDeposit} from "./processDeposit"; import {processVoluntaryExit} from "./processVoluntaryExit"; -type Operation = - | phase0.ProposerSlashing - | phase0.AttesterSlashing - | phase0.Attestation - | phase0.Deposit - | phase0.VoluntaryExit; -type OperationFunction = (state: CachedBeaconStatePhase0, op: Operation, verify: boolean) => void; - export function processOperations( state: CachedBeaconStatePhase0, body: phase0.BeaconBlockBody, @@ -30,15 +21,19 @@ export function processOperations( ); } - for (const [operations, processOp] of [ - [body.proposerSlashings, processProposerSlashing], - [body.attesterSlashings, processAttesterSlashing], - [body.attestations, processAttestation], - [body.deposits, processDeposit], - [body.voluntaryExits, processVoluntaryExit], - ] as [List, OperationFunction][]) { - for (const op of readonlyValues(operations)) { - processOp(state, op, verifySignatures); - } + for (const proposerSlashing of body.proposerSlashings) { + processProposerSlashing(state, proposerSlashing, verifySignatures); + } + for (const attesterSlashing of body.attesterSlashings) { + processAttesterSlashing(state, attesterSlashing, verifySignatures); + } + for (const attestation of body.attestations) { + processAttestation(state, attestation, verifySignatures); + } + for (const deposit of body.deposits) { + processDeposit(state, deposit); + } + for (const voluntaryExit of body.voluntaryExits) { + processVoluntaryExit(state, voluntaryExit, verifySignatures); } } diff --git a/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts b/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts index 3f8f89fbb038..38a2f42a42a7 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts @@ -1,4 +1,4 @@ -import {bigIntSqrt} from "@chainsafe/lodestar-utils"; +import {bigIntSqrt, bnToNum} from "@chainsafe/lodestar-utils"; import {BASE_REWARDS_PER_EPOCH as BASE_REWARDS_PER_EPOCH_CONST} from "../../constants"; import {newZeroedArray} from "../../util"; import {EpochProcess, CachedBeaconStatePhase0} from "../../types"; @@ -64,11 +64,11 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces const prevEpochHeadStakeByIncrement = epochProcess.prevEpochUnslashedStake.headStakeByIncrement; // sqrt first, before factoring out the increment for later usage - const balanceSqRoot = Number(bigIntSqrt(totalBalanceInGwei)); + const balanceSqRoot = bnToNum(bigIntSqrt(totalBalanceInGwei)); const finalityDelay = epochProcess.prevEpoch - state.finalizedCheckpoint.epoch; const BASE_REWARDS_PER_EPOCH = BASE_REWARDS_PER_EPOCH_CONST; - const proposerRewardQuotient = Number(PROPOSER_REWARD_QUOTIENT); + const proposerRewardQuotient = PROPOSER_REWARD_QUOTIENT; const isInInactivityLeak = finalityDelay > MIN_EPOCHS_TO_INACTIVITY_PENALTY; // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE @@ -122,6 +122,7 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces rewards[status.proposerIndex] += proposerReward; rewards[i] += Math.floor(maxAttesterReward / status.inclusionDelay); } + if (hasMarkers(status.flags, FLAG_ELIGIBLE_ATTESTER)) { // expected FFG source if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED)) { @@ -160,5 +161,6 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces } } } + return [rewards, penalties]; } diff --git a/packages/beacon-state-transition/src/phase0/epoch/index.ts b/packages/beacon-state-transition/src/phase0/epoch/index.ts index 01eb3ce9d5d2..ba80adaa7f58 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/index.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/index.ts @@ -13,7 +13,7 @@ import {processSlashings} from "./processSlashings"; import {getAttestationDeltas} from "./getAttestationDeltas"; import {processParticipationRecordUpdates} from "./processParticipationRecordUpdates"; -export {processRewardsAndPenalties, processSlashings, getAttestationDeltas}; +export {processRewardsAndPenalties, processSlashings, getAttestationDeltas, processParticipationRecordUpdates}; export function processEpoch(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { processJustificationAndFinalization(state as CachedBeaconStateAllForks, epochProcess); diff --git a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts index 75640d1c88e2..9023488a5384 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts @@ -1,5 +1,4 @@ -import {phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStatePhase0} from "../../types"; /** @@ -9,5 +8,7 @@ import {CachedBeaconStatePhase0} from "../../types"; export function processParticipationRecordUpdates(state: CachedBeaconStatePhase0): void { // rotate current/previous epoch attestations state.previousEpochAttestations = state.currentEpochAttestations; - state.currentEpochAttestations = ([] as phase0.PendingAttestation[]) as List; + + // Reset list to empty + state.currentEpochAttestations = ssz.phase0.EpochAttestations.toViewDU([]); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts b/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts index 1a2e6e349cf1..3f93235ac415 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts @@ -1,8 +1,7 @@ -import {Epoch, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, readonlyValues} from "@chainsafe/ssz"; +import {Epoch, phase0} from "@chainsafe/lodestar-types"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStatePhase0} from "../../types"; -import {computeStartSlotAtEpoch, getBlockRootAtSlot, zipIndexesCommitteeBits} from "../../util"; -import {IAttesterStatus} from "../../util/attesterStatus"; +import {computeStartSlotAtEpoch, getBlockRootAtSlot, IAttesterStatus} from "../../util"; /** * Mutates `statuses` from all pending attestations. @@ -18,33 +17,39 @@ import {IAttesterStatus} from "../../util/attesterStatus"; export function statusProcessEpoch( state: CachedBeaconStatePhase0, statuses: IAttesterStatus[], - attestations: List, + attestations: phase0.PendingAttestation[], epoch: Epoch, sourceFlag: number, targetFlag: number, headFlag: number ): void { const {epochCtx, slot: stateSlot} = state; - const rootType = ssz.Root; const prevEpoch = epochCtx.previousShuffling.epoch; if (attestations.length === 0) { return; } + + // Prevent frequent object get of external CommonJS dependencies + const byteArrayEqualsFn = byteArrayEquals; + const actualTargetBlockRoot = getBlockRootAtSlot(state, computeStartSlotAtEpoch(epoch)); - for (const att of readonlyValues(attestations)) { - const aggregationBits = att.aggregationBits; + + for (const att of attestations) { + // Ignore empty BitArray, from spec test minimal/phase0/epoch_processing/participation_record_updates updated_participation_record + // See https://github.com/ethereum/consensus-specs/issues/2825 + if (att.aggregationBits.bitLen === 0) { + continue; + } + const attData = att.data; const inclusionDelay = att.inclusionDelay; const proposerIndex = att.proposerIndex; const attSlot = attData.slot; - const committeeIndex = attData.index; - const attBeaconBlockRoot = attData.beaconBlockRoot; - const attTarget = attData.target; - const attVotedTargetRoot = rootType.equals(attTarget.root, actualTargetBlockRoot); + const attVotedTargetRoot = byteArrayEqualsFn(attData.target.root, actualTargetBlockRoot); const attVotedHeadRoot = - attSlot < stateSlot && rootType.equals(attBeaconBlockRoot, getBlockRootAtSlot(state, attSlot)); - const committee = epochCtx.getBeaconCommittee(attSlot, committeeIndex); - const participants = zipIndexesCommitteeBits(committee, aggregationBits); + attSlot < stateSlot && byteArrayEqualsFn(attData.beaconBlockRoot, getBlockRootAtSlot(state, attSlot)); + const committee = epochCtx.getBeaconCommittee(attSlot, attData.index); + const participants = att.aggregationBits.intersectValues(committee); if (epoch === prevEpoch) { for (const p of participants) { diff --git a/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts index 3d5a70ece820..be4e16930feb 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts @@ -1,13 +1,12 @@ import {ForkName} from "@chainsafe/lodestar-params"; import {processRewardsAndPenaltiesAllForks} from "../../allForks/epoch/processRewardsAndPenalties"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ export function processRewardsAndPenalties(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processRewardsAndPenaltiesAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, epochProcess); + processRewardsAndPenaltiesAllForks(ForkName.phase0, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/types.ts b/packages/beacon-state-transition/src/types.ts index 837121607129..37db21388242 100644 --- a/packages/beacon-state-transition/src/types.ts +++ b/packages/beacon-state-transition/src/types.ts @@ -1,15 +1,11 @@ -import {allForks, altair, bellatrix, phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconState} from "./cache/cachedBeaconState"; - export {EpochContext} from "./cache/epochContext"; export {EpochProcess} from "./cache/epochProcess"; -export type CachedBeaconStatePhase0 = CachedBeaconState; -export type CachedBeaconStateAltair = CachedBeaconState; -export type CachedBeaconStateBellatrix = CachedBeaconState; -export type CachedBeaconStateAllForks = CachedBeaconState; -export type CachedBeaconStateAnyFork = - | CachedBeaconStatePhase0 - | CachedBeaconStateAltair - | CachedBeaconStateBellatrix - | CachedBeaconStateAllForks; +export { + CachedBeaconStatePhase0, + CachedBeaconStateAltair, + CachedBeaconStateBellatrix, + CachedBeaconStateAllForks, +} from "./cache/stateCache"; + +export {BeaconStatePhase0, BeaconStateAltair, BeaconStateBellatrix, BeaconStateAllForks} from "./cache/types"; diff --git a/packages/beacon-state-transition/src/util/aggregationBits.ts b/packages/beacon-state-transition/src/util/aggregationBits.ts deleted file mode 100644 index 62b9b534e155..000000000000 --- a/packages/beacon-state-transition/src/util/aggregationBits.ts +++ /dev/null @@ -1,189 +0,0 @@ -import {BitList, BitListType, BitVector, isTreeBacked, TreeBacked, Type} from "@chainsafe/ssz"; -import {ssz} from "@chainsafe/lodestar-types"; -import {LodestarError} from "@chainsafe/lodestar-utils"; - -const BITS_PER_BYTE = 8; -/** Globally cache this information. @see getUint8ByteToBitBooleanArray */ -const uint8ByteToBitBooleanArrays: boolean[][] = []; - -/** - * Given a byte (0 -> 255), return a Array of boolean with length = 8, big endian. - * Ex: 1 => [true false false false false false false false] - * 5 => [true false true false false fase false false] - */ -export function getUint8ByteToBitBooleanArray(byte: number): boolean[] { - if (uint8ByteToBitBooleanArrays[byte] === undefined) { - uint8ByteToBitBooleanArrays[byte] = computeUint8ByteToBitBooleanArray(byte); - } - return uint8ByteToBitBooleanArrays[byte]; -} - -/** @see getUint8ByteToBitBooleanArray */ -function computeUint8ByteToBitBooleanArray(byte: number): boolean[] { - // this returns little endian - const binaryStr = byte.toString(2); - const binaryLength = binaryStr.length; - return Array.from({length: BITS_PER_BYTE}, (_, j) => { - if (j < binaryLength) { - return binaryStr[binaryLength - j - 1] === "1" ? true : false; - } else { - return false; - } - }); -} - -/** zipIndexes for CommitteeBits. @see zipIndexes */ -export function zipIndexesCommitteeBits(indexes: number[], bits: TreeBacked | BitVector): number[] { - return zipIndexes(indexes, bits, ssz.phase0.CommitteeBits)[0]; -} - -/** zipIndexes for SyncCommitteeBits. @see zipIndexes */ -export function zipIndexesSyncCommitteeBits(indexes: number[], bits: TreeBacked | BitVector): number[] { - return zipIndexes(indexes, bits, ssz.altair.SyncCommitteeBits)[0]; -} - -/** Similar to zipIndexesSyncCommitteeBits but we extract both participant and unparticipant indices*/ -export function zipAllIndexesSyncCommitteeBits( - indexes: number[], - bits: TreeBacked | BitVector -): [number[], number[]] { - return zipIndexes(indexes, bits, ssz.altair.SyncCommitteeBits); -} - -/** - * Performant indexing of a BitList, both as struct or TreeBacked - * Return [0] as participant indices and [1] as unparticipant indices - * @see zipIndexesInBitListTreeBacked - */ -export function zipIndexes( - indexes: number[], - bitlist: TreeBacked | BitArr, - sszType: Type -): [number[], number[]] { - if (isTreeBacked(bitlist)) { - return zipIndexesTreeBacked(indexes, bitlist, sszType); - } else { - const attestingIndices = []; - const unattestingIndices = []; - for (let i = 0, len = indexes.length; i < len; i++) { - if (bitlist[i]) { - attestingIndices.push(indexes[i]); - } else { - unattestingIndices.push(indexes[i]); - } - } - return [attestingIndices, unattestingIndices]; - } -} - -/** - * Returns [0] as indices that participated in `bitlist` and [1] as indices that did not participated in `bitlist`. - * Participation of `indexes[i]` means that the bit at position `i` in `bitlist` is true. - * - * Previously we computed this information with `readonlyValues(TreeBacked)`. - * However this approach is very inneficient since the SSZ parsing of BitList is not optimized. - * This function uses a precomputed array of booleans `Uint8 -> boolean[]` @see uint8ByteToBitBooleanArrays. - * This approach is x15 times faster. - */ -export function zipIndexesTreeBacked( - indexes: number[], - bits: TreeBacked, - sszType: Type -): [number[], number[]] { - const bytes = bitsToUint8Array(bits, sszType); - - const participantIndices: number[] = []; - const unparticipantIndices: number[] = []; - - // Iterate over each byte of bits - for (let iByte = 0, byteLen = bytes.length; iByte < byteLen; iByte++) { - // Get the precomputed boolean array for this byte - const booleansInByte = getUint8ByteToBitBooleanArray(bytes[iByte]); - // For each bit in the byte check participation and add to indexesSelected array - for (let iBit = 0; iBit < BITS_PER_BYTE; iBit++) { - const committeeIndex = indexes[iByte * BITS_PER_BYTE + iBit]; - if (committeeIndex !== undefined) { - if (booleansInByte[iBit]) { - participantIndices.push(committeeIndex); - } else { - unparticipantIndices.push(committeeIndex); - } - } - } - } - - return [participantIndices, unparticipantIndices]; -} - -/** - * Efficiently extract the Uint8Array inside a `TreeBacked` structure. - * @see zipIndexesInBitListTreeBacked for reasoning and advantatges. - */ -export function bitsToUint8Array( - bits: TreeBacked, - sszType: Type -): Uint8Array { - const tree = bits.tree; - const treeType = (sszType as unknown) as BitListType; - const chunkCount = treeType.tree_getChunkCount(tree); - const chunkDepth = treeType.getChunkDepth(); - const nodeIterator = tree.iterateNodesAtDepth(chunkDepth, 0, chunkCount); - const chunks: Uint8Array[] = []; - for (const node of nodeIterator) { - chunks.push(node.root); - } - // the last chunk has 32 bytes but we don't use all of them - return Buffer.concat(chunks).subarray(0, Math.ceil(bits.length / BITS_PER_BYTE)); -} - -/** - * Variant to extract a single bit (for un-aggregated attestations) - */ -export function getSingleBitIndex(bits: BitList | TreeBacked): number { - let index: number | null = null; - - if (isTreeBacked(bits)) { - const bytes = bitsToUint8Array(bits, ssz.phase0.CommitteeBits); - - // Iterate over each byte of bits - for (let iByte = 0, byteLen = bytes.length; iByte < byteLen; iByte++) { - // If it's exactly zero, there won't be any indexes, continue early - if (bytes[iByte] === 0) { - continue; - } - - // Get the precomputed boolean array for this byte - const booleansInByte = getUint8ByteToBitBooleanArray(bytes[iByte]); - // For each bit in the byte check participation and add to indexesSelected array - for (let iBit = 0; iBit < BITS_PER_BYTE; iBit++) { - if (booleansInByte[iBit] === true) { - if (index !== null) throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - index = iByte * BITS_PER_BYTE + iBit; - } - } - } - } else { - for (let i = 0, len = bits.length; i < len; i++) { - if (bits[i] === true) { - if (index !== null) throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - index = i; - } - } - } - - if (index === null) { - throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - } else { - return index; - } -} - -export enum AggregationBitsErrorCode { - NOT_EXACTLY_ONE_BIT_SET = "AGGREGATION_BITS_ERROR_NOT_EXACTLY_ONE_BIT_SET", -} - -type AggregationBitsErrorType = { - code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET; -}; - -export class AggregationBitsError extends LodestarError {} diff --git a/packages/beacon-state-transition/src/util/aggregator.ts b/packages/beacon-state-transition/src/util/aggregator.ts index c39bda9926f9..eef4ad87f1e3 100644 --- a/packages/beacon-state-transition/src/util/aggregator.ts +++ b/packages/beacon-state-transition/src/util/aggregator.ts @@ -1,4 +1,4 @@ -import {hash} from "@chainsafe/ssz"; +import SHA256 from "@chainsafe/as-sha256"; import {BLSSignature} from "@chainsafe/lodestar-types"; import {intDiv, bytesToBigInt} from "@chainsafe/lodestar-utils"; import { @@ -29,5 +29,5 @@ export function isAggregatorFromCommitteeLength(committeeLength: number, slotSig * Using bytesToInt() may cause isSelectionProofValid() to always return false. */ export function isSelectionProofValid(sig: BLSSignature, modulo: number): boolean { - return bytesToBigInt(hash(sig.valueOf() as Uint8Array).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; + return bytesToBigInt(SHA256.digest(sig).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; } diff --git a/packages/beacon-state-transition/src/util/altair.ts b/packages/beacon-state-transition/src/util/altair.ts new file mode 100644 index 000000000000..00e716c9d68a --- /dev/null +++ b/packages/beacon-state-transition/src/util/altair.ts @@ -0,0 +1,13 @@ +import {BASE_REWARD_FACTOR, EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; +import {bigIntSqrt, bnToNum} from "@chainsafe/lodestar-utils"; + +/** + * Before we manage bigIntSqrt(totalActiveStake) as BigInt and return BigInt. + * bigIntSqrt(totalActiveStake) should fit a number (2 ** 53 -1 max) + **/ +export function computeBaseRewardPerIncrement(totalActiveStakeByIncrement: number): number { + return Math.floor( + (EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR) / + bnToNum(bigIntSqrt(BigInt(totalActiveStakeByIncrement) * BigInt(EFFECTIVE_BALANCE_INCREMENT))) + ); +} diff --git a/packages/beacon-state-transition/src/util/array.ts b/packages/beacon-state-transition/src/util/array.ts index 43930a0a9d13..bb763d1359bb 100644 --- a/packages/beacon-state-transition/src/util/array.ts +++ b/packages/beacon-state-transition/src/util/array.ts @@ -30,3 +30,24 @@ export function newFilledArray(n: number, val: T): T[] { } return arr; } + +/** + * Returns an array with all values not in the participants array. + * - All elements in values must be unique + * - Does NOT require sorting + */ +export function getUnparticipantValues(participants: T[], values: T[]): T[] { + const unparticipants: T[] = []; + + let j = 0; + for (let i = 0; i < values.length; i++) { + if (values[i] === participants[j]) { + // Included + j++; + } else { + unparticipants.push(values[i]); + } + } + + return unparticipants; +} diff --git a/packages/beacon-state-transition/src/util/attesterStatus.ts b/packages/beacon-state-transition/src/util/attesterStatus.ts index ea09d251d065..908e3b75cc0d 100644 --- a/packages/beacon-state-transition/src/util/attesterStatus.ts +++ b/packages/beacon-state-transition/src/util/attesterStatus.ts @@ -8,9 +8,9 @@ export const FLAG_CURR_HEAD_ATTESTER = 1 << 5; export const FLAG_UNSLASHED = 1 << 6; export const FLAG_ELIGIBLE_ATTESTER = 1 << 7; // Precompute OR flags -export const FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED = FLAG_PREV_SOURCE_ATTESTER | FLAG_UNSLASHED; -export const FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED = FLAG_PREV_TARGET_ATTESTER | FLAG_UNSLASHED; -export const FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED = FLAG_PREV_HEAD_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_SOURCE_ATTESTER_UNSLASHED = FLAG_PREV_SOURCE_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_TARGET_ATTESTER_UNSLASHED = FLAG_PREV_TARGET_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_HEAD_ATTESTER_UNSLASHED = FLAG_PREV_HEAD_ATTESTER | FLAG_UNSLASHED; /** * During the epoch transition, additional data is precomputed to avoid traversing any state a second diff --git a/packages/beacon-state-transition/src/util/balance.ts b/packages/beacon-state-transition/src/util/balance.ts index 590bc0d23d14..e17e72332384 100644 --- a/packages/beacon-state-transition/src/util/balance.ts +++ b/packages/beacon-state-transition/src/util/balance.ts @@ -3,10 +3,11 @@ */ import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; -import {allForks, Gwei, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {Gwei, ValidatorIndex} from "@chainsafe/lodestar-types"; import {bigIntMax} from "@chainsafe/lodestar-utils"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types"; +import {BeaconStateAllForks} from ".."; +import {CachedBeaconStateAllForks} from "../types"; /** * Return the combined effective balance of the [[indices]]. @@ -14,27 +15,24 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types"; * * SLOW CODE - 🐢 */ -export function getTotalBalance(state: allForks.BeaconState, indices: ValidatorIndex[]): Gwei { - return bigIntMax( - BigInt(EFFECTIVE_BALANCE_INCREMENT), - indices.reduce( - // TODO: Use a fast cache to get the effective balance 🐢 - (total: Gwei, index: ValidatorIndex): Gwei => total + BigInt(state.validators[index].effectiveBalance), - BigInt(0) - ) - ); +export function getTotalBalance(state: BeaconStateAllForks, indices: ValidatorIndex[]): Gwei { + let total = BigInt(0); + + // TODO: Use a fast cache to get the effective balance 🐢 + const validatorsArr = state.validators.getAllReadonlyValues(); + for (let i = 0; i < indices.length; i++) { + total += BigInt(validatorsArr[indices[i]].effectiveBalance); + } + + return bigIntMax(BigInt(EFFECTIVE_BALANCE_INCREMENT), total); } /** * Increase the balance for a validator with the given ``index`` by ``delta``. */ -export function increaseBalance( - state: CachedBeaconStateAllForks | CachedBeaconStateAltair, - index: ValidatorIndex, - delta: number -): void { +export function increaseBalance(state: BeaconStateAllForks, index: ValidatorIndex, delta: number): void { // TODO: Inline this - state.balanceList.applyDelta(index, delta); + state.balances.set(index, state.balances.get(index) + delta); } /** @@ -42,12 +40,10 @@ export function increaseBalance( * * Set to ``0`` when underflow. */ -export function decreaseBalance( - state: CachedBeaconStateAllForks | CachedBeaconStateAltair, - index: ValidatorIndex, - delta: number -): void { - state.balanceList.applyDelta(index, -delta); +export function decreaseBalance(state: BeaconStateAllForks, index: ValidatorIndex, delta: number): void { + const newBalance = state.balances.get(index) - delta; + // TODO: Is it necessary to protect against underflow here? Add unit test + state.balances.set(index, Math.max(0, newBalance)); } /** @@ -58,10 +54,10 @@ export function decreaseBalance( export function getEffectiveBalanceIncrementsZeroInactive( justifiedState: CachedBeaconStateAllForks ): EffectiveBalanceIncrements { - const {activeIndices} = justifiedState.currentShuffling; - // 5x faster than using readonlyValuesListOfLeafNodeStruct + const {activeIndices} = justifiedState.epochCtx.currentShuffling; + // 5x faster than reading from state.validators, with validator Nodes as values const validatorCount = justifiedState.validators.length; - const {effectiveBalanceIncrements} = justifiedState; + const {effectiveBalanceIncrements} = justifiedState.epochCtx; // Slice up to `validatorCount` since it won't be mutated, nor accessed beyond `validatorCount` const effectiveBalanceIncrementsZeroInactive = effectiveBalanceIncrements.slice(0, validatorCount); diff --git a/packages/beacon-state-transition/src/util/blockRoot.ts b/packages/beacon-state-transition/src/util/blockRoot.ts index f32aee953a3e..3863af40c5b8 100644 --- a/packages/beacon-state-transition/src/util/blockRoot.ts +++ b/packages/beacon-state-transition/src/util/blockRoot.ts @@ -8,24 +8,25 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ZERO_HASH} from "../constants"; import {computeStartSlotAtEpoch} from "./epoch"; import {SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; /** * Return the block root at a recent [[slot]]. */ -export function getBlockRootAtSlot(state: allForks.BeaconState, slot: Slot): Root { +export function getBlockRootAtSlot(state: BeaconStateAllForks, slot: Slot): Root { if (slot >= state.slot) { throw Error(`Can only get block root in the past currentSlot=${state.slot} slot=${slot}`); } if (slot < state.slot - SLOTS_PER_HISTORICAL_ROOT) { throw Error(`Cannot get block root more than ${SLOTS_PER_HISTORICAL_ROOT} in the past`); } - return state.blockRoots[slot % SLOTS_PER_HISTORICAL_ROOT]; + return state.blockRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT); } /** * Return the block root at the start of a recent [[epoch]]. */ -export function getBlockRoot(state: allForks.BeaconState, epoch: Epoch): Root { +export function getBlockRoot(state: BeaconStateAllForks, epoch: Epoch): Root { return getBlockRootAtSlot(state, computeStartSlotAtEpoch(epoch)); } /** diff --git a/packages/beacon-state-transition/src/util/domain.ts b/packages/beacon-state-transition/src/util/domain.ts index 997dc32c5007..d74a859ad292 100644 --- a/packages/beacon-state-transition/src/util/domain.ts +++ b/packages/beacon-state-transition/src/util/domain.ts @@ -1,7 +1,7 @@ /** * @module chain/stateTransition/util */ -import {Epoch, Version, Root, DomainType, allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {Epoch, Version, Root, DomainType, phase0, ssz} from "@chainsafe/lodestar-types"; // Only used by processDeposit + lightclient /** @@ -18,7 +18,7 @@ export function computeDomain(domainType: DomainType, forkVersion: Version, gene /** * Return the ForkVersion at an epoch from a Fork type */ -export function getForkVersion(fork: allForks.BeaconState["fork"], epoch: Epoch): Version { +export function getForkVersion(fork: phase0.Fork, epoch: Epoch): Version { return epoch < fork.epoch ? fork.previousVersion : fork.currentVersion; } diff --git a/packages/beacon-state-transition/src/util/epoch.ts b/packages/beacon-state-transition/src/util/epoch.ts index bd411677fc59..7262cd2a605c 100644 --- a/packages/beacon-state-transition/src/util/epoch.ts +++ b/packages/beacon-state-transition/src/util/epoch.ts @@ -34,14 +34,14 @@ export function computeActivationExitEpoch(epoch: Epoch): Epoch { /** * Return the current epoch of the given state. */ -export function getCurrentEpoch(state: allForks.BeaconState): Epoch { +export function getCurrentEpoch(state: Pick): Epoch { return computeEpochAtSlot(state.slot); } /** * Return the previous epoch of the given state. */ -export function getPreviousEpoch(state: allForks.BeaconState): Epoch { +export function getPreviousEpoch(state: Pick): Epoch { const currentEpoch = getCurrentEpoch(state); if (currentEpoch === GENESIS_EPOCH) { return GENESIS_EPOCH; diff --git a/packages/beacon-state-transition/src/util/epochShuffling.ts b/packages/beacon-state-transition/src/util/epochShuffling.ts index 00558b673e55..91283fbdc56a 100644 --- a/packages/beacon-state-transition/src/util/epochShuffling.ts +++ b/packages/beacon-state-transition/src/util/epochShuffling.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex, allForks} from "@chainsafe/lodestar-types"; +import {Epoch, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import { DOMAIN_BEACON_ATTESTER, @@ -6,7 +6,17 @@ import { SLOTS_PER_EPOCH, TARGET_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; -import {getSeed, unshuffleList} from "."; +import {BeaconStateAllForks} from "../types"; +import {getSeed} from "./seed"; +import {unshuffleList} from "./shuffle"; + +/** + * Readonly interface for IEpochShuffling. + */ +export interface IReadonlyEpochShuffling { + readonly epoch: Epoch; + readonly committees: Readonly; +} export interface IEpochShuffling { /** @@ -47,7 +57,7 @@ export function computeCommitteeCount(activeValidatorCount: number): number { } export function computeEpochShuffling( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeIndices: ValidatorIndex[], epoch: Epoch ): IEpochShuffling { diff --git a/packages/beacon-state-transition/src/util/finality.ts b/packages/beacon-state-transition/src/util/finality.ts index bac4dfcc7b32..93f3ca46d614 100644 --- a/packages/beacon-state-transition/src/util/finality.ts +++ b/packages/beacon-state-transition/src/util/finality.ts @@ -2,7 +2,8 @@ import {MIN_EPOCHS_TO_INACTIVITY_PENALTY} from "@chainsafe/lodestar-params"; import {CachedBeaconStateAllForks} from "../types"; export function getFinalityDelay(state: CachedBeaconStateAllForks): number { - return state.previousShuffling.epoch - state.finalizedCheckpoint.epoch; + // previousEpoch = epoch - 1 + return state.epochCtx.epoch - 1 - state.finalizedCheckpoint.epoch; } /** diff --git a/packages/beacon-state-transition/src/util/genesis.ts b/packages/beacon-state-transition/src/util/genesis.ts index 0fd78479a2c8..dd5dd6776413 100644 --- a/packages/beacon-state-transition/src/util/genesis.ts +++ b/packages/beacon-state-transition/src/util/genesis.ts @@ -1,4 +1,3 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import { EFFECTIVE_BALANCE_INCREMENT, @@ -8,25 +7,21 @@ import { GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, } from "@chainsafe/lodestar-params"; -import { - allForks, - altair, - Bytes32, - bellatrix, - Number64, - phase0, - Root, - ssz, - ValidatorIndex, -} from "@chainsafe/lodestar-types"; +import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@chainsafe/lodestar-types"; +import {processDeposit} from "../allForks"; +import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types"; import {computeEpochAtSlot} from "./epoch"; import {getActiveValidatorIndices} from "./validator"; import {getTemporaryBlockHeader} from "./blockRoot"; -import {getNextSyncCommittee} from "../util/syncCommittee"; -import {processDeposit} from "../allForks"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; -import {CachedBeaconStateAllForks} from "../types"; +import {CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; +import {newFilledArray} from "./array"; +import {getNextSyncCommittee} from "./syncCommittee"; +import {createCachedBeaconState} from "../cache/stateCache"; +import {EpochContextImmutableData} from "../cache/epochContext"; + +type DepositDataRootListType = ListCompositeType; +type DepositDataRootViewDU = CompositeViewDU; // TODO: Refactor to work with non-phase0 genesis state @@ -35,7 +30,7 @@ import {CachedBeaconStateAllForks} from "../types"; * @param config * @param state */ -export function isValidGenesisState(config: IChainForkConfig, state: allForks.BeaconState): boolean { +export function isValidGenesisState(config: IChainForkConfig, state: BeaconStateAllForks): boolean { return state.genesisTime >= config.MIN_GENESIS_TIME && isValidGenesisValidators(config, state); } @@ -44,7 +39,7 @@ export function isValidGenesisState(config: IChainForkConfig, state: allForks.Be * @param config * @param state */ -export function isValidGenesisValidators(config: IChainForkConfig, state: allForks.BeaconState): boolean { +export function isValidGenesisValidators(config: IChainForkConfig, state: BeaconStateAllForks): boolean { return ( getActiveValidatorIndices(state, computeEpochAtSlot(GENESIS_SLOT)).length >= config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT @@ -57,14 +52,16 @@ export function isValidGenesisValidators(config: IChainForkConfig, state: allFor * SLOW CODE - 🐢 */ export function getGenesisBeaconState( - config: IBeaconConfig, + config: IChainForkConfig, genesisEth1Data: phase0.Eth1Data, latestBlockHeader: phase0.BeaconBlockHeader -): CachedBeaconStateAllForks { +): BeaconStateAllForks { // Seed RANDAO with Eth1 entropy - const randaoMixes = Array(EPOCHS_PER_HISTORICAL_VECTOR).fill(genesisEth1Data.blockHash); + const randaoMixes = newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, genesisEth1Data.blockHash); + + const beaconStateType = config.getForkTypes(GENESIS_SLOT).BeaconState; + const state = beaconStateType.defaultViewDU; - const state = config.getForkTypes(GENESIS_SLOT).BeaconState.defaultTreeBacked(); // MISC state.slot = GENESIS_SLOT; const version = config.getForkVersion(GENESIS_SLOT); @@ -74,31 +71,24 @@ export function getGenesisBeaconState( const previousForkIndex = Math.max(0, forkIndex - 1); const previousForkName = allForkNames[previousForkIndex]; const previousFork = config.forks[previousForkName]; + // the altair genesis spec test requires previous version to be phase0 although ALTAIR_FORK_EPOCH=0 - state.fork = { + state.fork = ssz.phase0.Fork.toViewDU({ previousVersion: previousFork.version, currentVersion: version, epoch: computeEpochAtSlot(GENESIS_SLOT), - } as phase0.Fork; + }); // Validator registry // Randomness and committees - state.latestBlockHeader = latestBlockHeader; + state.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU(latestBlockHeader); // Ethereum 1.0 chain data - state.eth1Data = genesisEth1Data; - state.randaoMixes = randaoMixes; + state.eth1Data = ssz.phase0.Eth1Data.toViewDU(genesisEth1Data); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU(randaoMixes); - // We need a CachedBeaconState to run processDeposit() which uses various caches. - // However at this point the state's syncCommittees are not known. - // This function can be called by: - // - 1. genesis spec tests: Don't care about the committee cache - // - 2. genesis builder: Only supports starting from genesis at phase0 fork - // - 3. interop state: Only supports starting from genesis at phase0 fork - // So it's okay to skip syncing the sync committee cache here and expect it to be - // populated latter when the altair fork happens for cases 2, 3. - return createCachedBeaconState(config, state, {skipSyncCommitteeCache: true}); + return state; } /** @@ -107,9 +97,9 @@ export function getGenesisBeaconState( * @param state BeaconState * @param eth1BlockHash eth1 block hash */ -export function applyEth1BlockHash(state: allForks.BeaconState, eth1BlockHash: Bytes32): void { +export function applyEth1BlockHash(state: CachedBeaconStateAllForks, eth1BlockHash: Bytes32): void { state.eth1Data.blockHash = eth1BlockHash; - state.randaoMixes = Array(EPOCHS_PER_HISTORICAL_VECTOR).fill(eth1BlockHash); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU(newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, eth1BlockHash)); } /** @@ -120,7 +110,7 @@ export function applyEth1BlockHash(state: allForks.BeaconState, eth1BlockHash: B */ export function applyTimestamp( config: IChainForkConfig, - state: TreeBacked, + state: CachedBeaconStateAllForks, eth1Timestamp: number ): void { state.genesisTime = eth1Timestamp + config.GENESIS_DELAY; @@ -143,12 +133,16 @@ export function applyDeposits( config: IChainForkConfig, state: CachedBeaconStateAllForks, newDeposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked> -): ValidatorIndex[] { + fullDepositDataRootList?: DepositDataRootViewDU +): {activatedValidatorCount: number} { const depositDataRootList: Root[] = []; - if (fullDepositDataRootList) { - for (let index = 0; index < state.eth1Data.depositCount; index++) { - depositDataRootList.push(fullDepositDataRootList[index]); + + const fullDepositDataRootArr = fullDepositDataRootList ? fullDepositDataRootList.getAllReadonlyValues() : null; + + if (fullDepositDataRootArr) { + const depositCount = state.eth1Data.depositCount; + for (let index = 0; index < depositCount; index++) { + depositDataRootList.push(fullDepositDataRootArr[index]); } } @@ -157,13 +151,13 @@ export function applyDeposits( const {DepositData, DepositDataRootList} = ssz.phase0; for (const [index, deposit] of newDeposits.entries()) { - if (fullDepositDataRootList) { - depositDataRootList.push(fullDepositDataRootList[index + initDepositCount]); - state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot(depositDataRootList as List); + if (fullDepositDataRootArr) { + depositDataRootList.push(fullDepositDataRootArr[index + initDepositCount]); + state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot(depositDataRootList); } else if (depositDatas) { const depositDataList = depositDatas.slice(0, index + 1); state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot( - depositDataList.map((d) => DepositData.hashTreeRoot(d)) as List + depositDataList.map((d) => DepositData.hashTreeRoot(d)) ); } @@ -173,81 +167,104 @@ export function applyDeposits( processDeposit(forkName, state, deposit); } - const activeValidatorIndices: ValidatorIndex[] = []; // Process activations - // validators are edited, so we're not iterating (read-only) through the validators - const validatorLength = state.validators.length; - for (let index = 0; index < validatorLength; index++) { - const validator = state.validators[index]; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const balance = state.balanceList.get(index)!; + const {epochCtx} = state; + const balancesArr = state.balances.getAll(); + const validatorCount = state.validators.length; + let activatedValidatorCount = 0; + + for (let i = 0; i < validatorCount; i++) { + // For the case if effective balance has to be updated, get a mutable view + const validator = state.validators.get(i); + + // Already active, ignore + if (validator.activationEpoch === GENESIS_EPOCH) { + continue; + } + + const balance = balancesArr[i]; const effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + validator.effectiveBalance = effectiveBalance; - state.effectiveBalanceIncrementsSet(index, effectiveBalance); + epochCtx.effectiveBalanceIncrementsSet(i, effectiveBalance); if (validator.effectiveBalance === MAX_EFFECTIVE_BALANCE) { validator.activationEligibilityEpoch = GENESIS_EPOCH; validator.activationEpoch = GENESIS_EPOCH; - activeValidatorIndices.push(index); + activatedValidatorCount++; } - // If state is a CachedBeaconState<> validator has to be re-assigned manually - state.validators[index] = validator; } // Set genesis validators root for domain separation and chain versioning - state.genesisValidatorsRoot = config - .getForkTypes(state.slot) - .BeaconState.fields.validators.hashTreeRoot(state.validators); - return activeValidatorIndices; + // .hashTreeRoot() automatically commits() + state.genesisValidatorsRoot = state.validators.hashTreeRoot(); + + return {activatedValidatorCount}; } /** * Mainly used for spec test. * * SLOW CODE - 🐢 - * - * @param config - * @param eth1BlockHash - * @param eth1Timestamp - * @param deposits */ export function initializeBeaconStateFromEth1( config: IChainForkConfig, + immutableData: EpochContextImmutableData, eth1BlockHash: Bytes32, - eth1Timestamp: Number64, + eth1Timestamp: TimeSeconds, deposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked>, - executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked() -): TreeBacked { - const state = getGenesisBeaconState( - // CachedBeaconcState is used for convinience only, we return TreeBacked anyway + fullDepositDataRootList?: DepositDataRootViewDU, + executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU +): CachedBeaconStateAllForks { + const stateView = getGenesisBeaconState( + // CachedBeaconcState is used for convinience only, we return BeaconStateAllForks anyway // so it's safe to do a cast here, we can't use get domain until we have genesisValidatorRoot config as IBeaconConfig, - ssz.phase0.Eth1Data.defaultValue(), - getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) + ssz.phase0.Eth1Data.defaultValue, + getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue) ); + // We need a CachedBeaconState to run processDeposit() which uses various caches. + // However at this point the state's syncCommittees are not known. + // This function can be called by: + // - 1. genesis spec tests: Don't care about the committee cache + // - 2. genesis builder: Only supports starting from genesis at phase0 fork + // - 3. interop state: Only supports starting from genesis at phase0 fork + // So it's okay to skip syncing the sync committee cache here and expect it to be + // populated latter when the altair fork happens for cases 2, 3. + const state = createCachedBeaconState(stateView, immutableData, {skipSyncCommitteeCache: true}); + applyTimestamp(config, state, eth1Timestamp); applyEth1BlockHash(state, eth1BlockHash); // Process deposits - const activeValidatorIndices = applyDeposits(config, state, deposits, fullDepositDataRootList); + applyDeposits(config, state, deposits, fullDepositDataRootList); + + // Commit before reading all validators in `getActiveValidatorIndices()` + state.commit(); + const activeValidatorIndices = getActiveValidatorIndices(state, computeEpochAtSlot(GENESIS_SLOT)); if (GENESIS_SLOT >= config.ALTAIR_FORK_EPOCH) { - const syncCommittees = getNextSyncCommittee(state, activeValidatorIndices, state.effectiveBalanceIncrements); - const stateAltair = state as TreeBacked; + const {syncCommittee} = getNextSyncCommittee( + state, + activeValidatorIndices, + state.epochCtx.effectiveBalanceIncrements + ); + const stateAltair = state as CompositeViewDU; stateAltair.fork.previousVersion = config.ALTAIR_FORK_VERSION; stateAltair.fork.currentVersion = config.ALTAIR_FORK_VERSION; - stateAltair.currentSyncCommittee = syncCommittees; - stateAltair.nextSyncCommittee = syncCommittees; + stateAltair.currentSyncCommittee = ssz.altair.SyncCommittee.toViewDU(syncCommittee); + stateAltair.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(syncCommittee); } if (GENESIS_SLOT >= config.BELLATRIX_FORK_EPOCH) { - const stateBellatrix = state as TreeBacked; + const stateBellatrix = state as CompositeViewDU; stateBellatrix.fork.previousVersion = config.BELLATRIX_FORK_VERSION; stateBellatrix.fork.currentVersion = config.BELLATRIX_FORK_VERSION; stateBellatrix.latestExecutionPayloadHeader = executionPayloadHeader; } + state.commit(); + return state; } diff --git a/packages/beacon-state-transition/src/util/index.ts b/packages/beacon-state-transition/src/util/index.ts index 08075da6bfc2..e3a963ff996e 100644 --- a/packages/beacon-state-transition/src/util/index.ts +++ b/packages/beacon-state-transition/src/util/index.ts @@ -2,7 +2,6 @@ * @module chain/stateTransition/util */ -export * from "./aggregationBits"; export * from "./aggregator"; export * from "./array"; export * from "./attestation"; @@ -22,6 +21,5 @@ export * from "./signatureSets"; export * from "./signingRoot"; export * from "./slot"; export * from "./syncCommittee"; -export * from "./unsafeUint8ArrayToTree"; export * from "./validator"; export * from "./weakSubjectivity"; diff --git a/packages/beacon-state-transition/src/util/seed.ts b/packages/beacon-state-transition/src/util/seed.ts index 77adc2e70394..dde0d3f2366c 100644 --- a/packages/beacon-state-transition/src/util/seed.ts +++ b/packages/beacon-state-transition/src/util/seed.ts @@ -2,8 +2,8 @@ * @module chain/stateTransition/util */ -import {hash} from "@chainsafe/ssz"; -import {Epoch, Bytes32, DomainType, allForks, ValidatorIndex} from "@chainsafe/lodestar-types"; +import SHA256 from "@chainsafe/as-sha256"; +import {Epoch, Bytes32, DomainType, ValidatorIndex} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt, intToBytes} from "@chainsafe/lodestar-utils"; import { DOMAIN_BEACON_PROPOSER, @@ -16,17 +16,17 @@ import { SLOTS_PER_EPOCH, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; import {computeStartSlotAtEpoch} from "./epoch"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; -import {IEpochShuffling} from "./epochShuffling"; import {computeEpochAtSlot} from "./epoch"; /** * Compute proposer indices for an epoch */ export function computeProposers( - state: allForks.BeaconState, - shuffling: IEpochShuffling, + state: BeaconStateAllForks, + shuffling: {epoch: Epoch; activeIndices: ValidatorIndex[]}, effectiveBalanceIncrements: EffectiveBalanceIncrements ): number[] { const epochSeed = getSeed(state, shuffling.epoch, DOMAIN_BEACON_PROPOSER); @@ -37,7 +37,7 @@ export function computeProposers( computeProposerIndex( effectiveBalanceIncrements, shuffling.activeIndices, - hash(Buffer.concat([epochSeed, intToBytes(slot, 8)])) + SHA256.digest(Buffer.concat([epochSeed, intToBytes(slot, 8)])) ) ); } @@ -54,7 +54,9 @@ export function computeProposerIndex( indices: ValidatorIndex[], seed: Uint8Array ): ValidatorIndex { - assert.gt(indices.length, 0, "Validator indices must not be empty"); + if (indices.length === 0) { + throw Error("Validator indices must not be empty"); + } // TODO: Inline outside this function const MAX_RANDOM_BYTE = 2 ** 8 - 1; @@ -64,7 +66,7 @@ export function computeProposerIndex( /* eslint-disable-next-line no-constant-condition */ while (true) { const candidateIndex = indices[computeShuffledIndex(i % indices.length, indices.length, seed)]; - const randByte = hash( + const randByte = SHA256.digest( Buffer.concat([ seed, // @@ -72,7 +74,6 @@ export function computeProposerIndex( ]) )[i % 32]; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const effectiveBalanceIncrement = effectiveBalanceIncrements[candidateIndex]; if (effectiveBalanceIncrement * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE_INCREMENT * randByte) { return candidateIndex; @@ -95,7 +96,7 @@ export function computeProposerIndex( * SLOW CODE - 🐢 */ export function getNextSyncCommitteeIndices( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeValidatorIndices: ValidatorIndex[], effectiveBalanceIncrements: EffectiveBalanceIncrements ): ValidatorIndex[] { @@ -112,7 +113,7 @@ export function getNextSyncCommitteeIndices( while (syncCommitteeIndices.length < SYNC_COMMITTEE_SIZE) { const shuffledIndex = computeShuffledIndex(i % activeValidatorCount, activeValidatorCount, seed); const candidateIndex = activeValidatorIndices[shuffledIndex]; - const randByte = hash( + const randByte = SHA256.digest( Buffer.concat([ seed, // @@ -142,14 +143,14 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By let permuted = index; assert.lt(index, indexCount, "indexCount must be less than index"); assert.lte(indexCount, 2 ** 40, "indexCount too big"); - const _seed = seed.valueOf() as Uint8Array; + const _seed = seed; for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) { const pivot = Number( - bytesToBigInt(hash(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) + bytesToBigInt(SHA256.digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) ); const flip = (pivot + indexCount - permuted) % indexCount; const position = Math.max(permuted, flip); - const source = hash(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); + const source = SHA256.digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); const byte = source[Math.floor((position % 256) / 8)]; const bit = (byte >> position % 8) % 2; permuted = bit ? flip : permuted; @@ -160,15 +161,15 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By /** * Return the randao mix at a recent [[epoch]]. */ -export function getRandaoMix(state: allForks.BeaconState, epoch: Epoch): Bytes32 { - return state.randaoMixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR]; +export function getRandaoMix(state: BeaconStateAllForks, epoch: Epoch): Bytes32 { + return state.randaoMixes.get(epoch % EPOCHS_PER_HISTORICAL_VECTOR); } /** * Return the seed at [[epoch]]. */ -export function getSeed(state: allForks.BeaconState, epoch: Epoch, domainType: DomainType): Uint8Array { +export function getSeed(state: BeaconStateAllForks, epoch: Epoch, domainType: DomainType): Uint8Array { const mix = getRandaoMix(state, epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1); - return hash(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix.valueOf() as Uint8Array])); + return SHA256.digest(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix])); } diff --git a/packages/beacon-state-transition/src/util/shuffle.ts b/packages/beacon-state-transition/src/util/shuffle.ts index 0d2a4167b364..12fe15000c41 100644 --- a/packages/beacon-state-transition/src/util/shuffle.ts +++ b/packages/beacon-state-transition/src/util/shuffle.ts @@ -1,7 +1,7 @@ /** * @module util/objects */ -import {hash} from "@chainsafe/ssz"; +import SHA256 from "@chainsafe/as-sha256"; import {SHUFFLE_ROUND_COUNT} from "@chainsafe/lodestar-params"; import {ValidatorIndex, Bytes32} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt} from "@chainsafe/lodestar-utils"; @@ -97,7 +97,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): } // Seed is always the first 32 bytes of the hash input, we never have to change this part of the buffer. - const _seed = seed.valueOf() as Uint8Array; + const _seed = seed; Buffer.from(_seed).copy(buf, 0, 0, _SHUFFLE_H_SEED_SIZE); // initial values here are not used: overwritten first within the inner for loop. @@ -111,7 +111,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): buf[_SHUFFLE_H_SEED_SIZE] = r; // Seed is already in place, now just hash the correct part of the buffer, and take a uint64 from it, // and modulo it to get a pivot within range. - const h = hash(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); + const h = SHA256.digest(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); const pivot = Number(bytesToBigInt(h.slice(0, 8)) % BigInt(listSize)) >>> 0; // Split up the for-loop in two: @@ -134,7 +134,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start from the pivot position, and work back to the mirror position (of the part left to the pivot). // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(pivot >> 8, buf); // already using first pivot byte below. - source = hash(buf); + source = SHA256.digest(buf); byteV = source[(pivot & 0xff) >> 3]; for (let i = 0, j; i < mirror; i++) { @@ -145,7 +145,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = hash(buf); + source = SHA256.digest(buf); } // Same trick with byte retrieval. Only every 8th. @@ -171,7 +171,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start at the end, and work back to the mirror point. // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(end >> 8, buf); - source = hash(buf); + source = SHA256.digest(buf); byteV = source[(end & 0xff) >> 3]; for (let i = pivot + 1, j; i < mirror; i++) { j = end - i + pivot + 1; @@ -181,7 +181,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = hash(buf); + source = SHA256.digest(buf); } // Same trick with byte retrieval. Only every 8th. diff --git a/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts b/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts index af63d833a66f..46d78041dbce 100644 --- a/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts +++ b/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts @@ -24,7 +24,7 @@ export function proposerShufflingDecisionRoot(state: CachedBeaconStateAllForks): * can be used to key the proposer shuffling for the current epoch. */ function proposerShufflingDecisionSlot(state: CachedBeaconStateAllForks): Slot { - const startSlot = computeStartSlotAtEpoch(state.currentShuffling.epoch); + const startSlot = computeStartSlotAtEpoch(state.epochCtx.epoch); return Math.max(startSlot - 1, 0); } @@ -64,15 +64,14 @@ function attesterShufflingDecisionSlot(state: CachedBeaconStateAllForks, request * - `EpochTooHigh` when `requestedEpoch` is more than 1 after `currentEpoch`. */ function attesterShufflingDecisionEpoch(state: CachedBeaconStateAllForks, requestedEpoch: Epoch): Epoch { - const currentEpoch = state.currentShuffling.epoch; - const previouEpoch = state.previousShuffling.epoch; + const currentEpoch = state.epochCtx.epoch; // Next if (requestedEpoch === currentEpoch + 1) return currentEpoch; // Current - if (requestedEpoch === currentEpoch) return previouEpoch; + if (requestedEpoch === currentEpoch) return Math.max(currentEpoch - 1, 0); // Previous - if (requestedEpoch === currentEpoch - 1) return Math.max(previouEpoch - 1, 0); + if (requestedEpoch === currentEpoch - 1) return Math.max(currentEpoch - 2, 0); if (requestedEpoch < currentEpoch) { throw Error(`EpochTooLow: current ${currentEpoch} requested ${requestedEpoch}`); diff --git a/packages/beacon-state-transition/src/util/signatureSets.ts b/packages/beacon-state-transition/src/util/signatureSets.ts index 8a4ab3c30533..e40ac7fff213 100644 --- a/packages/beacon-state-transition/src/util/signatureSets.ts +++ b/packages/beacon-state-transition/src/util/signatureSets.ts @@ -26,10 +26,10 @@ export function verifySignatureSet(signatureSet: ISignatureSet): boolean { switch (signatureSet.type) { case SignatureSetType.single: - return signature.verify(signatureSet.pubkey, signatureSet.signingRoot.valueOf() as Uint8Array); + return signature.verify(signatureSet.pubkey, signatureSet.signingRoot); case SignatureSetType.aggregate: - return signature.verifyAggregate(signatureSet.pubkeys, signatureSet.signingRoot.valueOf() as Uint8Array); + return signature.verifyAggregate(signatureSet.pubkeys, signatureSet.signingRoot); default: throw Error("Unknown signature set type"); diff --git a/packages/beacon-state-transition/src/util/slot.ts b/packages/beacon-state-transition/src/util/slot.ts index 7647dc12c127..68b8f221d5cc 100644 --- a/packages/beacon-state-transition/src/util/slot.ts +++ b/packages/beacon-state-transition/src/util/slot.ts @@ -1,27 +1,27 @@ import {IChainConfig} from "@chainsafe/lodestar-config"; import {GENESIS_SLOT, INTERVALS_PER_SLOT} from "@chainsafe/lodestar-params"; -import {Number64, Slot, Epoch} from "@chainsafe/lodestar-types"; +import {Slot, Epoch, TimeSeconds} from "@chainsafe/lodestar-types"; import {computeStartSlotAtEpoch, computeEpochAtSlot} from "."; -export function getSlotsSinceGenesis(config: IChainConfig, genesisTime: Number64): Slot { +export function getSlotsSinceGenesis(config: IChainConfig, genesisTime: TimeSeconds): Slot { const diffInSeconds = Date.now() / 1000 - genesisTime; return Math.floor(diffInSeconds / config.SECONDS_PER_SLOT); } -export function getCurrentSlot(config: IChainConfig, genesisTime: Number64): Slot { +export function getCurrentSlot(config: IChainConfig, genesisTime: TimeSeconds): Slot { return GENESIS_SLOT + getSlotsSinceGenesis(config, genesisTime); } -export function computeSlotsSinceEpochStart(slot: Slot, epoch?: Epoch): number { +export function computeSlotsSinceEpochStart(slot: Slot, epoch?: Epoch): Slot { const computeEpoch = epoch ?? computeEpochAtSlot(slot); return slot - computeStartSlotAtEpoch(computeEpoch); } -export function computeTimeAtSlot(config: IChainConfig, slot: Slot, genesisTime: Number64): Number64 { +export function computeTimeAtSlot(config: IChainConfig, slot: Slot, genesisTime: TimeSeconds): TimeSeconds { return genesisTime + slot * config.SECONDS_PER_SLOT; } -export function getCurrentInterval(config: IChainConfig, genesisTime: Number64, secondsIntoSlot: number): number { +export function getCurrentInterval(config: IChainConfig, secondsIntoSlot: number): number { const timePerInterval = Math.floor(config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT); return Math.floor(secondsIntoSlot / timePerInterval); } diff --git a/packages/beacon-state-transition/src/util/syncCommittee.ts b/packages/beacon-state-transition/src/util/syncCommittee.ts index 7b0392da95df..199afdd6ba6f 100644 --- a/packages/beacon-state-transition/src/util/syncCommittee.ts +++ b/packages/beacon-state-transition/src/util/syncCommittee.ts @@ -7,8 +7,9 @@ import { SYNC_REWARD_WEIGHT, WEIGHT_DENOMINATOR, } from "@chainsafe/lodestar-params"; -import {allForks, altair, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; import {bigIntSqrt} from "@chainsafe/lodestar-utils"; +import {BeaconStateAllForks} from "../types"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; import {getNextSyncCommitteeIndices} from "./seed"; @@ -18,16 +19,21 @@ import {getNextSyncCommitteeIndices} from "./seed"; * SLOW CODE - 🐢 */ export function getNextSyncCommittee( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeValidatorIndices: ValidatorIndex[], effectiveBalanceIncrements: EffectiveBalanceIncrements -): altair.SyncCommittee { +): {indices: ValidatorIndex[]; syncCommittee: altair.SyncCommittee} { const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices, effectiveBalanceIncrements); + // Using the index2pubkey cache is slower because it needs the serialized pubkey. - const pubkeys = indices.map((index) => state.validators[index].pubkey); + const pubkeys = indices.map((index) => state.validators.get(index).pubkey); + return { - pubkeys, - aggregatePubkey: aggregatePublicKeys(pubkeys.map((pubkey) => pubkey.valueOf() as Uint8Array)), + indices, + syncCommittee: { + pubkeys, + aggregatePubkey: aggregatePublicKeys(pubkeys), + }, }; } diff --git a/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts b/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts deleted file mode 100644 index f2a57cfba835..000000000000 --- a/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts +++ /dev/null @@ -1,26 +0,0 @@ -import {LeafNode, Node, subtreeFillToContents} from "@chainsafe/persistent-merkle-tree"; - -/** - * Convert a Uint8Array to a merkle tree, using the underlying array's underlying ArrayBuffer - * - * `data` MUST NOT be modified after this, or risk the merkle nodes being modified. - */ -export function unsafeUint8ArrayToTree(data: Uint8Array, depth: number): Node { - const leaves: LeafNode[] = []; - - // Loop 32 bytes at a time, creating leaves from the backing subarray - const maxStartIndex = data.length - 31; - for (let i = 0; i < maxStartIndex; i += 32) { - leaves.push(new LeafNode(data.subarray(i, i + 32))); - } - - // If there is any extra data at the end (less than 32 bytes), append a final leaf - const lengthMod32 = data.length % 32; - if (lengthMod32 !== 0) { - const finalChunk = new Uint8Array(32); - finalChunk.set(data.subarray(data.length - lengthMod32)); - leaves.push(new LeafNode(finalChunk)); - } - - return subtreeFillToContents(leaves, depth); -} diff --git a/packages/beacon-state-transition/src/util/validator.ts b/packages/beacon-state-transition/src/util/validator.ts index 2d146a3536f8..e70c5f706316 100644 --- a/packages/beacon-state-transition/src/util/validator.ts +++ b/packages/beacon-state-transition/src/util/validator.ts @@ -2,10 +2,10 @@ * @module chain/stateTransition/util */ -import {readonlyValues} from "@chainsafe/ssz"; -import {Epoch, phase0, ValidatorIndex, allForks} from "@chainsafe/lodestar-types"; +import {Epoch, phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "../types"; /** * Check if [[validator]] is active @@ -26,15 +26,16 @@ export function isSlashableValidator(validator: phase0.Validator, epoch: Epoch): * * NAIVE - SLOW CODE 🐢 */ -export function getActiveValidatorIndices(state: allForks.BeaconState, epoch: Epoch): ValidatorIndex[] { +export function getActiveValidatorIndices(state: BeaconStateAllForks, epoch: Epoch): ValidatorIndex[] { const indices: ValidatorIndex[] = []; - let index = 0; - for (const validator of readonlyValues(state.validators)) { - if (isActiveValidator(validator, epoch)) { - indices.push(index); + + const validatorsArr = state.validators.getAllReadonlyValues(); + for (let i = 0; i < validatorsArr.length; i++) { + if (isActiveValidator(validatorsArr[i], epoch)) { + indices.push(i); } - index++; } + return indices; } diff --git a/packages/beacon-state-transition/src/util/weakSubjectivity.ts b/packages/beacon-state-transition/src/util/weakSubjectivity.ts index 3dfdfcf67043..169adcd109cf 100644 --- a/packages/beacon-state-transition/src/util/weakSubjectivity.ts +++ b/packages/beacon-state-transition/src/util/weakSubjectivity.ts @@ -5,11 +5,11 @@ import { MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; -import {allForks, Epoch, Root} from "@chainsafe/lodestar-types"; +import {Epoch, Root} from "@chainsafe/lodestar-types"; import {ssz} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {toHexString} from "@chainsafe/ssz"; -import {CachedBeaconStateAllForks} from "../types"; +import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types"; import { getActiveValidatorIndices, getCurrentEpoch, @@ -20,7 +20,7 @@ import { } from ".."; export const ETH_TO_GWEI = 10 ** 9; -const SAFETY_DECAY = BigInt(10); +const SAFETY_DECAY = 10; /** * Returns the epoch of the latest weak subjectivity checkpoint for the given @@ -45,10 +45,10 @@ export function computeWeakSubjectivityPeriodCachedState( config: IChainForkConfig, state: CachedBeaconStateAllForks ): number { - const activeValidatorCount = state.currentShuffling.activeIndices.length; + const activeValidatorCount = state.epochCtx.currentShuffling.activeIndices.length; return computeWeakSubjectivityPeriodFromConstituents( activeValidatorCount, - state.totalActiveBalanceIncrements, + state.epochCtx.totalActiveBalanceIncrements, getChurnLimit(config, activeValidatorCount), config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY ); @@ -58,11 +58,12 @@ export function computeWeakSubjectivityPeriodCachedState( * Same to computeWeakSubjectivityPeriodCachedState but for normal state * This is called only 1 time at app startup so it's ok to calculate totalActiveBalanceIncrements manually */ -export function computeWeakSubjectivityPeriod(config: IChainForkConfig, state: allForks.BeaconState): number { +export function computeWeakSubjectivityPeriod(config: IChainForkConfig, state: BeaconStateAllForks): number { const activeIndices = getActiveValidatorIndices(state, getCurrentEpoch(state)); + const validators = state.validators.getAllReadonlyValues(); let totalActiveBalanceIncrements = 0; for (const index of activeIndices) { - totalActiveBalanceIncrements += Math.floor(state.validators[index].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + totalActiveBalanceIncrements += Math.floor(validators[index].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } if (totalActiveBalanceIncrements <= 1) { totalActiveBalanceIncrements = 1; @@ -86,11 +87,11 @@ export function computeWeakSubjectivityPeriodFromConstituents( // totalBalanceByIncrement = totalBalance / MAX_EFFECTIVE_BALANCE and MAX_EFFECTIVE_BALANCE = ETH_TO_GWEI atm // we need to change this calculation just in case MAX_EFFECTIVE_BALANCE != ETH_TO_GWEI const t = Math.floor(totalBalanceByIncrement / N); - const T = Number(MAX_EFFECTIVE_BALANCE / ETH_TO_GWEI); + const T = MAX_EFFECTIVE_BALANCE / ETH_TO_GWEI; const delta = churnLimit; // eslint-disable-next-line @typescript-eslint/naming-convention const Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH; - const D = Number(SAFETY_DECAY); + const D = SAFETY_DECAY; let wsPeriod = minWithdrawabilityDelay; if (T * (200 + 3 * D) < t * (200 + 12 * D)) { @@ -106,21 +107,21 @@ export function computeWeakSubjectivityPeriodFromConstituents( return wsPeriod; } -export function getLatestBlockRoot(config: IChainForkConfig, state: allForks.BeaconState): Root { +export function getLatestBlockRoot(state: BeaconStateAllForks): Root { const header = ssz.phase0.BeaconBlockHeader.clone(state.latestBlockHeader); if (ssz.Root.equals(header.stateRoot, ZERO_HASH)) { - header.stateRoot = config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state); + header.stateRoot = state.hashTreeRoot(); } return ssz.phase0.BeaconBlockHeader.hashTreeRoot(header); } export function isWithinWeakSubjectivityPeriod( config: IBeaconConfig, - wsState: allForks.BeaconState, + wsState: BeaconStateAllForks, wsCheckpoint: Checkpoint ): boolean { const wsStateEpoch = computeEpochAtSlot(wsState.slot); - const blockRoot = getLatestBlockRoot(config, wsState); + const blockRoot = getLatestBlockRoot(wsState); if (!ssz.Root.equals(blockRoot, wsCheckpoint.root)) { throw new Error( `Roots do not match. expected=${toHexString(wsCheckpoint.root)}, actual=${toHexString(blockRoot)}` diff --git a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts index c384f946a9e0..12f7670b7a4f 100644 --- a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts @@ -1,7 +1,6 @@ import {itBench} from "@dapplion/benchmark"; -import {phase0, ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; -import {generatePerformanceStatePhase0, getPubkeys, numValidators} from "../util"; +import {ssz} from "@chainsafe/lodestar-types"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../util"; import {unshuffleList} from "../../../src"; // Test cost of hashing state after some modifications @@ -9,22 +8,21 @@ import {unshuffleList} from "../../../src"; describe("BeaconState hashTreeRoot", () => { const vc = numValidators; const indicesShuffled: number[] = []; - let stateOg: TreeBacked; + let stateOg: ReturnType; before(function () { this.timeout(300_000); - const {pubkeys} = getPubkeys(); - stateOg = generatePerformanceStatePhase0(pubkeys); + stateOg = generatePerfTestCachedStatePhase0(); stateOg.hashTreeRoot(); for (let i = 0; i < vc; i++) indicesShuffled[i] = i; unshuffleList(indicesShuffled, new Uint8Array([42, 32])); }); - const validator: phase0.Validator = ssz.phase0.Validator.defaultValue(); + const validator = ssz.phase0.Validator.defaultViewDU; const balance = 31e9; - const testCases: {id: string; noTrack?: boolean; fn: (state: TreeBacked) => void}[] = [ + const testCases: {id: string; noTrack?: boolean; fn: (state: typeof stateOg) => void}[] = [ { id: "No change", fn: () => { @@ -39,7 +37,7 @@ describe("BeaconState hashTreeRoot", () => { id: `${count} full validator`, noTrack: count < 512, fn: (state) => { - for (const i of indicesShuffled.slice(0, count)) state.validators[i] = validator; + for (const i of indicesShuffled.slice(0, count)) state.validators.set(i, validator); }, }); } @@ -50,7 +48,7 @@ describe("BeaconState hashTreeRoot", () => { noTrack: count < 512, fn: (state) => { for (const i of indicesShuffled.slice(0, count)) { - state.validators[i].effectiveBalance = balance; + state.validators.get(i).effectiveBalance = balance; } }, }); @@ -62,18 +60,19 @@ describe("BeaconState hashTreeRoot", () => { id: `${count} balances`, noTrack: count < 512, fn: (state) => { - for (const i of indicesShuffled.slice(0, count)) state.balances[i] = balance; + for (const i of indicesShuffled.slice(0, count)) state.balances.set(i, balance); }, }); } for (const {id, noTrack, fn} of testCases) { - itBench, TreeBacked>({ + itBench({ id: `BeaconState.hashTreeRoot - ${id}`, noThreshold: noTrack, beforeEach: () => { const state = stateOg.clone(); fn(state); + state.commit(); return state; }, fn: (state) => { diff --git a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts index c905793c40ea..c0e279845818 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts @@ -25,7 +25,7 @@ describe("epoch shufflings", () => { itBench({ id: `computeProposers - vc ${numValidators}`, fn: () => { - computeProposers(state, state.epochCtx.nextShuffling, state.effectiveBalanceIncrements); + computeProposers(state, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); @@ -39,7 +39,11 @@ describe("epoch shufflings", () => { itBench({ id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { - getNextSyncCommittee(state, state.epochCtx.nextShuffling.activeIndices, state.effectiveBalanceIncrements); + getNextSyncCommittee( + state, + state.epochCtx.nextShuffling.activeIndices, + state.epochCtx.effectiveBalanceIncrements + ); }, }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts index ea55a1f67c20..ea3c2f113782 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts @@ -10,12 +10,16 @@ import { SLOTS_PER_EPOCH, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; +import {phase0} from "@chainsafe/lodestar-types"; import {altair, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; import {BlockAltairOpts, getBlockAltair} from "../../phase0/block/util"; -import {StateAltair, StateAttestations} from "../../types"; -import {ParticipationFlags, phase0} from "@chainsafe/lodestar-types"; -import {updateEpochParticipants} from "../../../../src/altair/block/processAttestation"; +import {StateAltair} from "../../types"; + +type StateAttestations = { + state: CachedBeaconStateAllForks; + attestations: phase0.Attestation[]; +}; // Most of the cost of processAttestation in altair is for updating participation flag tree describe("altair processAttestation", () => { @@ -56,13 +60,22 @@ describe("altair processAttestation", () => { id: `altair processAttestation - ${perfStateId} ${id}`, before: () => { const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; - const block = getBlockAltair(state, opts); - state.hashTreeRoot(); + const block = getBlockAltair(state as CachedBeaconStateAltair, opts); return {state, attestations: block.message.body.attestations as phase0.Attestation[]}; }, - beforeEach: ({state, attestations}) => ({state: state.clone(), attestations}), + beforeEach: ({state, attestations}) => { + const stateCloned = state.clone(); + // Populate state array cache (on the cloned instance) + (stateCloned as CachedBeaconStateAltair).previousEpochParticipation.getAll(); + (stateCloned as CachedBeaconStateAltair).currentEpochParticipation.getAll(); + (stateCloned as CachedBeaconStateAltair).balances.getAll(); + return {state: stateCloned, attestations}; + }, fn: ({state, attestations}) => { altair.processAttestations(state as CachedBeaconStateAltair, attestations, false); + state.commit(); + // After processAttestations normal case vc 250_000 it has to do 6802 hash64 ops + // state.hashTreeRoot(); }, }); } @@ -89,62 +102,27 @@ describe("altair processAttestation - CachedEpochParticipation.setStatus", () => state.hashTreeRoot(); return state; }, - beforeEach: (state) => state.clone(), - fn: (state) => { - const {currentEpochParticipation} = state; - const numAttesters = Math.floor((state.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH); - // just get committees of slot 10 - let count = 0; - for (const committees of state.currentShuffling.committees[10]) { - for (const committee of committees) { - currentEpochParticipation.set(committee, 0b111); - count++; - if (count >= numAttesters) break; - } - } + beforeEach: (state) => { + const stateCloned = state.clone(); + // Populate state array cache (on the cloned instance) + stateCloned.currentEpochParticipation.getAll(); + return stateCloned; }, - }); - } - - for (const {name, ratio} of testCases) { - itBench({ - id: `altair processAttestation - updateEpochParticipants - ${name} join`, - before: () => { - const state = generatePerfTestCachedStateAltair(); - state.hashTreeRoot(); - return state; - }, - beforeEach: (state) => state.clone(), fn: (state) => { const {currentEpochParticipation} = state; - const numAttesters = Math.floor((state.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH); + const numAttesters = Math.floor( + (state.epochCtx.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH + ); // just get committees of slot 10 let count = 0; - const epochStatuses = new Map(); - for (const committees of state.currentShuffling.committees[10]) { + for (const committees of state.epochCtx.currentShuffling.committees[10]) { for (const committee of committees) { - // currentEpochParticipation.setStatus(committee, {timelyHead: true, timelySource: true, timelyTarget: true}); - epochStatuses.set(committee, 0b111); + currentEpochParticipation.set(committee, 0b111); count++; if (count >= numAttesters) break; } } - updateEpochParticipants(epochStatuses, currentEpochParticipation, state.currentShuffling.activeIndices.length); }, }); } - - itBench({ - id: "altair processAttestation - updateAllStatus", - before: () => { - const state = generatePerfTestCachedStateAltair(); - state.hashTreeRoot(); - return state; - }, - beforeEach: (state) => state.clone(), - fn: (state) => { - const {currentEpochParticipation} = state; - currentEpochParticipation.updateAllStatus(currentEpochParticipation.persistent.vector); - }, - }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts index 40f1b0a32bcb..6dd70c2523ac 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts @@ -1,4 +1,5 @@ import {itBench} from "@dapplion/benchmark"; +import {ssz} from "@chainsafe/lodestar-types"; import { ACTIVE_PRESET, MAX_ATTESTATIONS, @@ -9,8 +10,8 @@ import { PresetName, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; -import {allForks, CachedBeaconStateAllForks} from "../../../../src"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; +import {allForks, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../../util"; import {BlockAltairOpts, getBlockAltair} from "../../phase0/block/util"; import {StateBlock} from "../../types"; @@ -31,7 +32,7 @@ import {StateBlock} from "../../types"; // // ### Verifying signatures // Signature verification is done in bulk using batch BLS verification. Performance is proportional to the amount of -// sigs to verify and the cost to construct the signature sets from TreeBacked data. +// sigs to verify and the cost to construct the signature sets from TreeView data. // // - Same as phase0 // - SyncAggregate sigs: 1 x agg (358 bits on avg) - TODO: assuming same participation as attestations @@ -99,24 +100,41 @@ describe("altair processBlock", () => { ]; for (const {id, opts} of testCases) { - itBench({ - id: `altair processBlock - ${perfStateId} ${id}`, - before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; - const block = getBlockAltair(state, opts); - state.hashTreeRoot(); - return {state, block}; - }, - beforeEach: ({state, block}) => ({state: state.clone(), block}), - fn: ({state, block}) => { - allForks.stateTransition(state, block, { - verifyProposer: false, - verifySignatures: false, - verifyStateRoot: false, - }); - // set verifyStateRoot = false, and get the root here because the block root is wrong - state.hashTreeRoot(); - }, - }); + for (const hashState of [false, true]) { + itBench({ + id: `altair processBlock - ${perfStateId} ${id}` + hashState ? " hashState" : "", + before: () => { + const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; + const block = getBlockAltair(state as CachedBeaconStateAltair, opts); + // Populate permanent root caches of the block + ssz.altair.BeaconBlock.hashTreeRoot(block.message); + // Populate tree root caches of the state + state.hashTreeRoot(); + return {state, block}; + }, + beforeEach: ({state, block}) => { + const stateCloned = state.clone(); + // Populate all state array caches (on the cloned instance) + cachedStateAltairPopulateCaches(stateCloned as CachedBeaconStateAltair); + return {state: stateCloned, block}; + }, + fn: ({state, block}) => { + const postState = allForks.stateTransition(state, block, { + verifyProposer: false, + verifySignatures: false, + verifyStateRoot: false, + }); + + // Not necessary to call commit here since it's called inside .stateTransition() + + if (hashState) { + // set verifyStateRoot = false, and get the root here because the block root is wrong + // normalcase vc 250_000: 6947 hashes - 8.683 ms + // worstcase vc 250_000: 15297 hashes - 19.121 ms + postState.hashTreeRoot(); + } + }, + }); + } } }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts new file mode 100644 index 000000000000..31ac1fb98013 --- /dev/null +++ b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts @@ -0,0 +1,54 @@ +import {itBench} from "@dapplion/benchmark"; +import {ACTIVE_PRESET, PresetName, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; +import {allForks, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; +import {getBlockAltair} from "../../phase0/block/util"; +import {phase0} from "@chainsafe/lodestar-types"; + +type StateEth1Data = { + state: CachedBeaconStateAllForks; + eth1Data: phase0.Eth1Data; +}; + +// Most of the cost of processAttestation in altair is for updating participation flag tree +describe("altair processEth1Data", () => { + if (ACTIVE_PRESET !== PresetName.mainnet) { + throw Error(`ACTIVE_PRESET ${ACTIVE_PRESET} must be mainnet`); + } + + const testCases: {id: string}[] = [ + {id: "normalcase"}, + // TODO: What's a worst case for eth1Data? + // {id: "worstcase"} + ]; + + for (const {id} of testCases) { + itBench({ + id: `altair processEth1Data - ${perfStateId} ${id}`, + before: () => { + const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; + const block = getBlockAltair(state as CachedBeaconStateAltair, { + proposerSlashingLen: 0, + attesterSlashingLen: 0, + attestationLen: 90, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + // TODO: There's no data yet on how full syncCommittee will be. Assume same ratio of attestations + syncCommitteeBitsLen: Math.round(SYNC_COMMITTEE_SIZE * 0.7), + }); + state.hashTreeRoot(); + return {state, eth1Data: block.message.body.eth1Data}; + }, + beforeEach: ({state, eth1Data}) => { + const stateCloned = state.clone(); + // Populate nodes cache of eth1DataVotes array (on the cloned instance) + stateCloned.eth1DataVotes.getAllReadonly(); + return {state: stateCloned, eth1Data}; + }, + fn: ({state, eth1Data}) => { + allForks.processEth1Data(state, eth1Data); + }, + }); + } +}); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts index f601fc271244..0f86c4c2a5b4 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts @@ -34,6 +34,7 @@ describe(`altair processEpoch - ${stateId}`, () => { altair.processEpoch(state as CachedBeaconStateAltair, epochProcess); state.epochCtx.afterProcessEpoch(state, epochProcess); // Simulate root computation through the next block to account for changes + // 74184 hash64 ops - 92.730 ms state.hashTreeRoot(); }, }); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts index 397a7d41f271..3cf8e9431807 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts @@ -5,8 +5,7 @@ import {FlagFactors, generateBalanceDeltasEpochProcess} from "../../phase0/epoch import {StateAltairEpoch} from "../../types"; import {mutateInactivityScores} from "./util"; -// PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags -// are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* +// PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set // Worst case: // statuses: everything true diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index 234ce3b2c94f..e6c9b70813dc 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -1,6 +1,6 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; import {itBench} from "@dapplion/benchmark"; -import {altair} from "../../../../src"; +import {processSyncCommitteeUpdates} from "../../../../src/altair"; import {StateAltair} from "../../types"; import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; @@ -12,12 +12,18 @@ describe("altair processSyncCommitteeUpdates", () => { itBench({ id: `altair processSyncCommitteeUpdates - ${vc}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - before: () => { - const state = generatePerfTestCachedStateAltair({goBackOneSlot: true}); - state.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; - return state; + before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), + beforeEach: (state) => { + if (state.epochCtx.epoch + (1 % EPOCHS_PER_SYNC_COMMITTEE_PERIOD) === 0) { + // OK will run + } else { + throw Error("processSyncCommitteeUpdates will not rotate syncCommittees"); + } + + return state.clone(); + }, + fn: (state) => { + processSyncCommitteeUpdates(state); }, - beforeEach: (state) => state.clone(), - fn: (state) => altair.processSyncCommitteeUpdates(state), }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/util.ts b/packages/beacon-state-transition/test/perf/altair/epoch/util.ts index 58f00fc47a07..802d2217511d 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/util.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/util.ts @@ -3,6 +3,6 @@ import {CachedBeaconStateAltair} from "../../../../src"; export function mutateInactivityScores(state: CachedBeaconStateAltair, factorWithPositive: number): void { const vc = state.inactivityScores.length; for (let i = 0; i < vc; i++) { - state.inactivityScores[i] = i < vc * factorWithPositive ? 100 : 0; + state.inactivityScores.set(i, i < vc * factorWithPositive ? 100 : 0); } } diff --git a/packages/beacon-state-transition/test/perf/analyzeBlocks.ts b/packages/beacon-state-transition/test/perf/analyzeBlocks.ts index d535a8c5accc..d6be5e1049c1 100644 --- a/packages/beacon-state-transition/test/perf/analyzeBlocks.ts +++ b/packages/beacon-state-transition/test/perf/analyzeBlocks.ts @@ -59,8 +59,9 @@ async function run(): Promise { voluntaryExits += block.message.body.voluntaryExits.length; for (const attestation of block.message.body.attestations) { + const indexes = Array.from({length: attestation.aggregationBits.bitLen}, () => 0); + aggregationBits += attestation.aggregationBits.intersectValues(indexes).length; inclusionDistance += block.message.slot - attestation.data.slot; - aggregationBits += Array.from(attestation.aggregationBits).filter((bit) => bit === true).length; } } diff --git a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts index c7f5172ab9d7..9455de75fcee 100644 --- a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts +++ b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts @@ -10,13 +10,13 @@ import { computeStartSlotAtEpoch, CachedBeaconStateAllForks, AttesterFlags, - createCachedBeaconState, beforeProcessEpoch, parseAttesterFlags, } from "../../src"; import {Validator} from "../../lib/phase0"; import {csvAppend, readCsv} from "./csv"; import {getInfuraBeaconUrl} from "./infura"; +import {createCachedBeaconStateTest} from "../utils/state"; // Understand the real network characteristics regarding epoch transitions to accurately produce performance test data. // @@ -97,8 +97,8 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< const preEpoch = computeEpochAtSlot(state.slot); const nextEpochSlot = computeStartSlotAtEpoch(preEpoch + 1); - const stateTB = ssz.phase0.BeaconState.createTreeBackedFromStruct(state as phase0.BeaconState); - const postState = createCachedBeaconState(config, stateTB); + const stateTB = ssz.phase0.BeaconState.toViewDU(state as phase0.BeaconState); + const postState = createCachedBeaconStateTest(stateTB, config); const epochProcess = beforeProcessEpoch(postState); allForks.processSlots(postState as CachedBeaconStateAllForks, nextEpochSlot, null); @@ -109,7 +109,7 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< const validatorKeys = Object.keys(validatorChangesCountZero) as (keyof typeof validatorChangesCountZero)[]; for (let i = 0; i < validatorCount; i++) { const validatorPrev = state.validators[i]; - const validatorNext = postState.validators[i]; + const validatorNext = postState.validators.get(i); for (const key of validatorKeys) { const valuePrev = validatorPrev[key]; const valueNext = validatorNext[key]; @@ -172,9 +172,9 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< function countAttBits(atts: phase0.PendingAttestation[]): number { let totalBits = 0; for (const att of atts) { - for (const bit of att.aggregationBits) { - if (bit) totalBits++; - } + const indexes = Array.from({length: att.aggregationBits.bitLen}, () => 0); + const yesCount = att.aggregationBits.intersectValues(indexes).length; + totalBits += yesCount; } return totalBits / atts.length; } diff --git a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts index 5855773c5b16..f9853ed843bd 100644 --- a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts +++ b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts @@ -50,7 +50,7 @@ describe("Tree (persistent-merkle-tree)", () => { const d = 40; const gih = toGindex(d, BigInt(ih)); - const n2 = new LeafNode(Buffer.alloc(32, 2)); + const n2 = LeafNode.fromRoot(Buffer.alloc(32, 2)); let tree: Tree; before(function () { @@ -89,7 +89,7 @@ describe("Tree (persistent-merkle-tree)", () => { }); function getTree(d: number, n: number): Tree { - const leaf = new LeafNode(Buffer.alloc(32, 1)); + const leaf = LeafNode.fromRoot(Buffer.alloc(32, 1)); const startIndex = BigInt(2 ** d); const tree = new Tree(zeroNode(d)); for (let i = BigInt(0), nB = BigInt(n); i < nB; i++) { diff --git a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts index 85bdec06da52..46327e6dc76f 100644 --- a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts @@ -1,8 +1,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@chainsafe/lodestar-params"; import {ssz} from "@chainsafe/lodestar-types"; -import {BitList, List, readonlyValues, TreeBacked} from "@chainsafe/ssz"; -import {zipIndexesCommitteeBits} from "../../../src"; +import {BitArray} from "@chainsafe/ssz"; describe("aggregationBits", () => { setBenchOpts({noThreshold: true}); @@ -11,11 +10,11 @@ describe("aggregationBits", () => { const idPrefix = `aggregationBits - ${len} els`; let indexes: number[]; - let bitlistTree: TreeBacked; + let bitlistTree: BitArray; before(function () { - const aggregationBits = Array.from({length: len}, () => true); - bitlistTree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(aggregationBits as List); + const aggregationBits = BitArray.fromBoolArray(Array.from({length: len}, () => true)); + bitlistTree = ssz.phase0.CommitteeBits.toViewDU(aggregationBits); indexes = Array.from({length: len}, () => 165432); }); @@ -23,11 +22,7 @@ describe("aggregationBits", () => { // aggregationBits - 2048 els - zipIndexesInBitList 50.904 us/op 236.17 us/op 0.22 // This benchmark is very unstable in CI. We already know that zipIndexesInBitList is faster - itBench(`${idPrefix} - readonlyValues`, () => { - Array.from(readonlyValues(bitlistTree)); - }); - itBench(`${idPrefix} - zipIndexesInBitList`, () => { - zipIndexesCommitteeBits(indexes, bitlistTree); + bitlistTree.intersectValues(indexes); }); }); diff --git a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts index 0c9ba3d069c6..0a63f4fd9830 100644 --- a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts @@ -13,18 +13,14 @@ describe("root equals", () => { setBenchOpts({noThreshold: true}); const stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee"); - const rootTree = ssz.Root.createTreeBackedFromStruct(stateRoot); + const rootTree = ssz.Root.toViewDU(stateRoot); // This benchmark is very unstable in CI. We already know that "ssz.Root.equals" is the fastest itBench("ssz.Root.equals", () => { ssz.Root.equals(rootTree, stateRoot); }); - itBench("ssz.Root.equals with valueOf()", () => { - ssz.Root.equals(rootTree.valueOf() as Uint8Array, stateRoot); - }); - - itBench("byteArrayEquals with valueOf()", () => { - byteArrayEquals(rootTree.valueOf() as Uint8Array, stateRoot); + itBench("byteArrayEquals", () => { + byteArrayEquals(rootTree, stateRoot); }); }); diff --git a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts index 48adc087dca1..27c9c04beb1f 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts @@ -30,7 +30,7 @@ import {StateBlock} from "../../types"; // // ### Verifying signatures // Signature verification is done in bulk using batch BLS verification. Performance is proportional to the amount of -// sigs to verify and the cost to construct the signature sets from TreeBacked data. +// sigs to verify and the cost to construct the signature sets from TreeView data. // // - Proposer sig: 1 single // - RandaoReveal sig: 1 single diff --git a/packages/beacon-state-transition/test/perf/phase0/block/util.ts b/packages/beacon-state-transition/test/perf/phase0/block/util.ts index 140940a6d52f..31bd1c953c3b 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/util.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/util.ts @@ -1,16 +1,17 @@ import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {SecretKey} from "@chainsafe/blst"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {BitArray} from "@chainsafe/ssz"; import {DOMAIN_DEPOSIT, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; import {config} from "@chainsafe/lodestar-config/default"; -import {List} from "@chainsafe/ssz"; import { computeDomain, computeEpochAtSlot, computeSigningRoot, ZERO_HASH, CachedBeaconStateAllForks, + CachedBeaconStateAltair, } from "../../../../src"; -import {LeafNode} from "@chainsafe/persistent-merkle-tree"; import {getBlockRoot, getBlockRootAtSlot} from "../../../../src"; export type BlockOpts = { @@ -47,7 +48,7 @@ export function getBlockPhase0( const attesterSlashingStartIndex = proposerSlashingStartIndex + proposerSlashingLen * exitedIndexStep; const voluntaryExitStartIndex = attesterSlashingStartIndex + attesterSlashingLen * bitsLen * exitedIndexStep; - const proposerSlashings = ([] as phase0.ProposerSlashing[]) as List; + const proposerSlashings = [] as phase0.ProposerSlashing[]; for (let i = 0; i < proposerSlashingLen; i++) { const proposerIndex = proposerSlashingStartIndex + i * exitedIndexStep; proposerSlashings.push({ @@ -64,11 +65,11 @@ export function getBlockPhase0( const attSlot = stateSlot - 2; const attEpoch = computeEpochAtSlot(attSlot); - const attesterSlashings = ([] as phase0.AttesterSlashing[]) as List; + const attesterSlashings = [] as phase0.AttesterSlashing[]; for (let i = 0; i < attesterSlashingLen; i++) { // Double vote for 128 participants const startIndex = attesterSlashingStartIndex + i * bitsLen * exitedIndexStep; - const attestingIndices = linspace(startIndex, bitsLen, exitedIndexStep) as List; + const attestingIndices = linspace(startIndex, bitsLen, exitedIndexStep); const attData: phase0.AttestationData = { slot: attSlot, @@ -91,8 +92,8 @@ export function getBlockPhase0( }); } - const attestations = ([] as phase0.Attestation[]) as List; - const committeeCountPerSlot = preState.getCommitteeCountPerSlot(attEpoch); + const attestations = [] as phase0.Attestation[]; + const committeeCountPerSlot = preState.epochCtx.getCommitteeCountPerSlot(attEpoch); const attSource = attEpoch === stateEpoch ? preState.currentJustifiedCheckpoint : preState.previousJustifiedCheckpoint; for (let i = 0; i < attestationLen; i++) { @@ -100,7 +101,7 @@ export function getBlockPhase0( const attCommittee = preState.epochCtx.getBeaconCommittee(attSlot, attIndex); // Spread attesting indices through the whole range, offset on each attestation attestations.push({ - aggregationBits: getAggregationBits(attCommittee.length, bitsLen) as List, + aggregationBits: getAggregationBits(attCommittee.length, bitsLen), data: { slot: attSlot, index: attIndex, @@ -115,7 +116,7 @@ export function getBlockPhase0( // Moved to different function since it's a bit complex const deposits = getDeposits(preState, depositsLen); - const voluntaryExits = ([] as phase0.SignedVoluntaryExit[]) as List; + const voluntaryExits = [] as phase0.SignedVoluntaryExit[]; for (let i = 0; i < voluntaryExitLen; i++) { voluntaryExits.push({ message: { @@ -127,10 +128,10 @@ export function getBlockPhase0( } const slot = preState.slot + 1; - return ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct({ + return { message: { slot, - proposerIndex: preState.getBeaconProposer(slot), + proposerIndex: preState.epochCtx.getBeaconProposer(slot), parentRoot: ssz.phase0.BeaconBlockHeader.hashTreeRoot(preState.latestBlockHeader), // TODO: Compute the state root properly! stateRoot: rootA, @@ -152,14 +153,14 @@ export function getBlockPhase0( }, }, signature: emptySig, - }); + }; } /** * Get an altair block. * This mutates the input preState as well to mark attestations not seen by the network. */ -export function getBlockAltair(preState: CachedBeaconStateAllForks, opts: BlockAltairOpts): altair.SignedBeaconBlock { +export function getBlockAltair(preState: CachedBeaconStateAltair, opts: BlockAltairOpts): altair.SignedBeaconBlock { const emptySig = Buffer.alloc(96); const phase0Block = getBlockPhase0(preState as CachedBeaconStateAllForks, opts); const stateEpoch = computeEpochAtSlot(preState.slot); @@ -167,7 +168,11 @@ export function getBlockAltair(preState: CachedBeaconStateAllForks, opts: BlockA const attEpoch = computeEpochAtSlot(attestation.data.slot); const epochParticipation = attEpoch === stateEpoch ? preState.currentEpochParticipation : preState.previousEpochParticipation; - const attestingIndices = preState.getAttestingIndices(attestation.data, attestation.aggregationBits); + + const committeeindices = preState.epochCtx.getBeaconCommittee(attestation.data.slot, attestation.data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeindices); + + // TODO: Is this necessary? for (const index of attestingIndices) { epochParticipation.set(index, 0); } @@ -191,13 +196,18 @@ export function getBlockAltair(preState: CachedBeaconStateAllForks, opts: BlockA * Generate valid deposits with valid signatures and valid merkle proofs. * NOTE: Mutates `preState` to add the new `eth1Data.depositRoot` */ -function getDeposits(preState: CachedBeaconStateAllForks, count: number): List { - const depositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); +function getDeposits(preState: CachedBeaconStateAllForks, count: number): phase0.Deposit[] { + const depositRootViewDU = ssz.phase0.DepositDataRootList.toViewDU([]); const depositCount = preState.eth1Data.depositCount; const withdrawalCredentials = Buffer.alloc(32, 0xee); const depositsData: phase0.DepositData[] = []; - const deposits = ([] as phase0.Deposit[]) as List; + const deposits = [] as phase0.Deposit[]; + + // Fill depositRootViewDU up to depositCount + // Instead of actually filling it, just mutate the length to allow .set() + depositRootViewDU["_length"] = depositCount + count; + depositRootViewDU["dirtyLength"] = true; for (let i = 0; i < count; i++) { const sk = SecretKey.fromBytes(Buffer.alloc(32, i + 1)); @@ -206,28 +216,30 @@ function getDeposits(preState: CachedBeaconStateAllForks, count: number): List

0) { preState.eth1Data.depositCount = depositCount + count; - preState.eth1Data.depositRoot = depositRootTree.hashTreeRoot(); + preState.eth1Data.depositRoot = depositRootViewDU.hashTreeRoot(); } return deposits; @@ -242,10 +254,10 @@ function linspace(from: number, count: number, step: number): number[] { return arr; } -function getAggregationBits(len: number, participants: number): boolean[] { - const bits: boolean[] = []; - for (let i = 0; i < len; i++) { - bits.push(i < participants); +function getAggregationBits(len: number, participants: number): BitArray { + const bits = BitArray.fromBitLen(len); + for (let i = 0; i < participants; i++) { + bits.set(i, true); } return bits; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts index fa36d37d47e7..03ba47dc45ec 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts @@ -1,15 +1,10 @@ import {itBench} from "@dapplion/benchmark"; import {ssz} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/default"; -import { - allForks, - beforeProcessEpoch, - CachedBeaconStateAllForks, - createCachedBeaconState, - EpochProcess, -} from "../../../../src"; +import {allForks, beforeProcessEpoch, CachedBeaconStateAllForks, EpochProcess} from "../../../../src"; import {numValidators} from "../../util"; import {StateEpoch} from "../../types"; +import {createCachedBeaconStateTest} from "../../../utils/state"; // PERF: Cost 'proportional' to $VALIDATOR_COUNT, to iterate over all balances. Then cost is proportional to the amount // of validators whose effectiveBalance changed. Worst case is a massive network leak or a big slashing event which @@ -54,16 +49,16 @@ function getEffectiveBalanceTestData( state: CachedBeaconStateAllForks; epochProcess: EpochProcess; } { - const stateTree = ssz.phase0.BeaconState.defaultTreeBacked(); + const stateTree = ssz.phase0.BeaconState.defaultViewDU; stateTree.slot = 1; - const activeValidator = { - ...ssz.phase0.Validator.defaultTreeBacked(), + const activeValidator = ssz.phase0.Validator.toViewDU({ + ...ssz.phase0.Validator.defaultValue, exitEpoch: Infinity, withdrawableEpoch: Infinity, // Set current effective balance to max effectiveBalance: 32e9, - }; + }); const balances: number[] = []; for (let i = 0; i < vc; i++) { @@ -76,8 +71,10 @@ function getEffectiveBalanceTestData( stateTree.validators.push(activeValidator); } - const cachedBeaconState = createCachedBeaconState(config, stateTree, {skipSyncPubkeys: true}); - const epochProcess = beforeProcessEpoch(cachedBeaconState); + stateTree.commit(); + + const cachedBeaconState = createCachedBeaconStateTest(stateTree, config, {skipSyncPubkeys: true}); + const epochProcess = beforeProcessEpoch(cachedBeaconState as CachedBeaconStateAllForks); epochProcess.balances = balances; return { diff --git a/packages/beacon-state-transition/test/perf/sanityCheck.test.ts b/packages/beacon-state-transition/test/perf/sanityCheck.test.ts index 384193ca226d..225f786f8fcd 100644 --- a/packages/beacon-state-transition/test/perf/sanityCheck.test.ts +++ b/packages/beacon-state-transition/test/perf/sanityCheck.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; import {ACTIVE_PRESET, EFFECTIVE_BALANCE_INCREMENT, PresetName} from "@chainsafe/lodestar-params"; -import {beforeProcessEpoch} from "../../src"; +import {beforeProcessEpoch, CachedBeaconStateAllForks} from "../../src"; import {generatePerfTestCachedStateAltair, generatePerfTestCachedStatePhase0, perfStateId} from "./util"; describe("Perf test sanity check", function () { @@ -30,7 +30,7 @@ describe("Perf test sanity check", function () { it("targetStake is in the same range", () => { const phase0State = generatePerfTestCachedStatePhase0(); - const epochProcess = beforeProcessEpoch(phase0State); + const epochProcess = beforeProcessEpoch(phase0State as CachedBeaconStateAllForks); expect( BigInt(epochProcess.prevEpochUnslashedStake.targetStakeByIncrement) * BigInt(EFFECTIVE_BALANCE_INCREMENT) > targetStake diff --git a/packages/beacon-state-transition/test/perf/types.ts b/packages/beacon-state-transition/test/perf/types.ts index 67238e68b472..7cc0dd3379fa 100644 --- a/packages/beacon-state-transition/test/perf/types.ts +++ b/packages/beacon-state-transition/test/perf/types.ts @@ -1,4 +1,4 @@ -import {allForks, phase0, CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src"; +import {allForks, CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src"; import {EpochProcess} from "../../src/types"; // Type aliases to typesafe itBench() calls @@ -6,10 +6,6 @@ import {EpochProcess} from "../../src/types"; export type State = CachedBeaconStateAllForks; export type StateAltair = CachedBeaconStateAltair; export type StateBlock = {state: CachedBeaconStateAllForks; block: allForks.SignedBeaconBlock}; -export type StateAttestations = { - state: CachedBeaconStateAllForks; - attestations: phase0.Attestation[]; -}; export type StateEpoch = {state: CachedBeaconStateAllForks; epochProcess: EpochProcess}; export type StatePhase0Epoch = {state: CachedBeaconStatePhase0; epochProcess: EpochProcess}; export type StateAltairEpoch = {state: CachedBeaconStateAltair; epochProcess: EpochProcess}; diff --git a/packages/beacon-state-transition/test/perf/util.ts b/packages/beacon-state-transition/test/perf/util.ts index 8358b832bf3c..849fb2f882f5 100644 --- a/packages/beacon-state-transition/test/perf/util.ts +++ b/packages/beacon-state-transition/test/perf/util.ts @@ -1,47 +1,51 @@ import fs from "node:fs"; import path from "node:path"; import {config} from "@chainsafe/lodestar-config/default"; -import {phase0, ssz, Slot, altair, ParticipationFlags} from "@chainsafe/lodestar-types"; +import {phase0, ssz, Slot, altair} from "@chainsafe/lodestar-types"; import bls, {CoordType, PublicKey, SecretKey} from "@chainsafe/bls"; -import {fromHexString, List, TreeBacked} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import { allForks, interopSecretKey, computeEpochAtSlot, getActiveValidatorIndices, PubkeyIndexMap, + newFilledArray, createCachedBeaconState, - getNextSyncCommittee, computeCommitteeCount, } from "../../src"; -import {createIChainForkConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src/types"; +import {createIBeaconConfig, createIChainForkConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import { + CachedBeaconStateAllForks, + CachedBeaconStatePhase0, + CachedBeaconStateAltair, + BeaconStatePhase0, + BeaconStateAltair, + BeaconStateAllForks, +} from "../../src/types"; import {profilerLogger} from "../utils/logger"; import {interopPubkeysCached} from "../utils/interop"; -import {PendingAttestation} from "@chainsafe/lodestar-types/phase0"; -import {intDiv} from "@chainsafe/lodestar-utils"; import { - EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_ETH1_VOTING_PERIOD, EPOCHS_PER_HISTORICAL_VECTOR, MAX_ATTESTATIONS, - MAX_VALIDATORS_PER_COMMITTEE, + MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT, - TIMELY_HEAD_FLAG_INDEX, - TIMELY_SOURCE_FLAG_INDEX, - TIMELY_TARGET_FLAG_INDEX, } from "@chainsafe/lodestar-params"; import {NetworkName, networksChainConfig} from "@chainsafe/lodestar-config/networks"; import {getClient} from "@chainsafe/lodestar-api"; import {getInfuraBeaconUrl} from "./infura"; import {testCachePath} from "../cache"; +import {getNextSyncCommittee} from "../../src/util/syncCommittee"; +import {createCachedBeaconStateTest} from "../utils/state"; +import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements"; -let phase0State: TreeBacked | null = null; +let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; let phase0CachedState23638: CachedBeaconStatePhase0 | null = null; -let phase0SignedBlock: TreeBacked | null = null; -let altairState: TreeBacked | null = null; +let phase0SignedBlock: phase0.SignedBeaconBlock | null = null; +let altairState: BeaconStateAltair | null = null; let altairCachedState23637: CachedBeaconStateAltair | null = null; let altairCachedState23638: CachedBeaconStateAltair | null = null; const logger = profilerLogger(); @@ -88,10 +92,8 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey { return sk; } -export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { - // Generate only some publicKeys - const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); - +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +function getPubkeyCaches({pubkeysMod, pubkeysModObj}: ReturnType) { // Manually sync pubkeys to prevent doing BLS opts 110_000 times const pubkey2index = new PubkeyIndexMap(); const index2pubkey = [] as PublicKey[]; @@ -102,15 +104,101 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean index2pubkey.push(pubkeyObj); } - const origState = generatePerformanceStatePhase0(pubkeys); + // Since most pubkeys are equal the size of pubkey2index is not numValidators. + // Fill with junk up to numValidators + for (let i = pubkey2index.size; i < numValidators; i++) { + const buf = Buffer.alloc(48, 0); + buf.writeInt32LE(i); + pubkey2index.set(buf, i); + } + + return {pubkey2index, index2pubkey}; +} + +export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { + // Generate only some publicKeys + const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); + const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + + if (!phase0State) { + const state = buildPerformanceStatePhase0(); + + // no justificationBits + phase0State = ssz.phase0.BeaconState.toViewDU(state); + logger.verbose("Loaded phase0 state", { + slot: state.slot, + numValidators: state.validators.length, + }); + + // cache roots + phase0State.hashTreeRoot(); + } + if (!phase0CachedState23637) { - const state = origState.clone(); + const state = phase0State.clone(); state.slot -= 1; - phase0CachedState23637 = createCachedBeaconState(config, state, { + phase0CachedState23637 = createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, }); + + const currentEpoch = computeEpochAtSlot(state.slot - 1); + const previousEpoch = currentEpoch - 1; + + // previous epoch attestations + const numPrevAttestations = SLOTS_PER_EPOCH * MAX_ATTESTATIONS; + const activeValidatorCount = pubkeys.length; + const committeesPerSlot = computeCommitteeCount(activeValidatorCount); + for (let i = 0; i < numPrevAttestations; i++) { + const slotInEpoch = i % SLOTS_PER_EPOCH; + const slot = previousEpoch * SLOTS_PER_EPOCH + slotInEpoch; + const index = i % committeesPerSlot; + const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch); + const committee = shuffling.committees[slotInEpoch][index]; + phase0CachedState23637.previousEpochAttestations.push( + ssz.phase0.PendingAttestation.toViewDU({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)), + data: { + beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT), + index, + slot, + source: state.previousJustifiedCheckpoint, + target: state.currentJustifiedCheckpoint, + }, + inclusionDelay: 1, + proposerIndex: i, + }) + ); + } + + // current epoch attestations + const numCurAttestations = (SLOTS_PER_EPOCH - 1) * MAX_ATTESTATIONS; + for (let i = 0; i < numCurAttestations; i++) { + const slotInEpoch = i % SLOTS_PER_EPOCH; + const slot = currentEpoch * SLOTS_PER_EPOCH + slotInEpoch; + const index = i % committeesPerSlot; + const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch); + const committee = shuffling.committees[slotInEpoch][index]; + + phase0CachedState23637.currentEpochAttestations.push( + ssz.phase0.PendingAttestation.toViewDU({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)), + data: { + beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT), + index, + slot, + source: state.currentJustifiedCheckpoint, + target: { + epoch: currentEpoch, + root: phase0CachedState23637.blockRoots.get((currentEpoch * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT), + }, + }, + inclusionDelay: 1, + proposerIndex: i, + }) + ); + } } if (!phase0CachedState23638) { phase0CachedState23638 = allForks.processSlots( @@ -124,29 +212,33 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean return resultingState.clone(); } +export function cachedStateAltairPopulateCaches(state: CachedBeaconStateAltair): void { + // Populate caches + state.blockRoots.getAllReadonly(); + state.eth1DataVotes.getAllReadonly(); + state.validators.getAllReadonly(); + state.balances.getAll(); + state.previousEpochParticipation.getAll(); + state.currentEpochParticipation.getAll(); + state.inactivityScores.getAll(); +} + export function generatePerfTestCachedStateAltair(opts?: {goBackOneSlot: boolean}): CachedBeaconStateAltair { const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); + const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + // eslint-disable-next-line @typescript-eslint/naming-convention const altairConfig = createIChainForkConfig({ALTAIR_FORK_EPOCH: 0}); - // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; - for (let i = 0; i < numValidators; i++) { - const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); - } - const origState = generatePerformanceStateAltair(pubkeys); + if (!altairCachedState23637) { const state = origState.clone(); state.slot -= 1; - altairCachedState23637 = createCachedBeaconState(altairConfig, state, { + altairCachedState23637 = createCachedBeaconState(state, { + config: createIBeaconConfig(altairConfig, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, }); } if (!altairCachedState23638) { @@ -164,36 +256,33 @@ export function generatePerfTestCachedStateAltair(opts?: {goBackOneSlot: boolean /** * This is generated from Medalla state 756416 */ -export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): TreeBacked { +export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): BeaconStateAltair { if (!altairState) { const pubkeys = pubkeysArg || getPubkeys().pubkeys; - const defaultState = ssz.altair.BeaconState.defaultValue(); - const state = buildPerformanceStateAllForks(defaultState) as altair.BeaconState; - const TIMELY_SOURCE = 1 << TIMELY_SOURCE_FLAG_INDEX; - const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; - const TIMELY_HEAD = 1 << TIMELY_HEAD_FLAG_INDEX; - const fullParticpation = TIMELY_SOURCE | TIMELY_TARGET | TIMELY_HEAD; - if (fullParticpation !== 7) { - throw new Error("Not correct fullParticipation"); - } - state.previousEpochParticipation = Array.from( - {length: pubkeys.length}, - () => fullParticpation - ) as List; - state.currentEpochParticipation = Array.from( - {length: pubkeys.length}, - () => fullParticpation - ) as List; - state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2) as List; + const statePhase0 = buildPerformanceStatePhase0(); + const state = (statePhase0 as allForks.BeaconState) as altair.BeaconState; + + state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111); + state.currentEpochParticipation = state.previousEpochParticipation; + state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2); + + // Placeholder syncCommittees + state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue; + state.nextSyncCommittee = state.currentSyncCommittee; + + // Now the state is fully populated to convert to ViewDU + altairState = ssz.altair.BeaconState.toViewDU(state); + + // Now set correct syncCommittees const epoch = computeEpochAtSlot(state.slot); - const activeValidatorIndices = getActiveValidatorIndices(state, epoch); - const effectiveBalanceIncrements = new Uint8Array( - Array.from(state.validators).map((v) => v.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT) - ); - const syncCommittee = getNextSyncCommittee(state, activeValidatorIndices, effectiveBalanceIncrements); + const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch); + + const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState); + const {syncCommittee} = getNextSyncCommittee(altairState, activeValidatorIndices, effectiveBalanceIncrements); state.currentSyncCommittee = syncCommittee; state.nextSyncCommittee = syncCommittee; - altairState = ssz.altair.BeaconState.createTreeBackedFromStruct(state); + + altairState = ssz.altair.BeaconState.toViewDU(state); logger.verbose("Loaded phase0 state", { slot: altairState.slot, numValidators: altairState.validators.length, @@ -204,152 +293,99 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): TreeB return altairState.clone(); } -/** - * This is generated from Medalla state 756416 - */ -export function generatePerformanceStatePhase0(pubkeysArg?: Uint8Array[]): TreeBacked { - if (!phase0State) { - const pubkeys = pubkeysArg || getPubkeys().pubkeys; - const defaultState = ssz.phase0.BeaconState.defaultValue(); - const state = buildPerformanceStateAllForks(defaultState) as phase0.BeaconState; - const currentEpoch = computeEpochAtSlot(state.slot - 1); - const previousEpoch = currentEpoch - 1; - // previous epoch attestations - const numPrevAttestations = SLOTS_PER_EPOCH * MAX_ATTESTATIONS; - const activeValidatorCount = pubkeys.length; - const committeesPerSlot = computeCommitteeCount(activeValidatorCount); - state.previousEpochAttestations = Array.from({length: numPrevAttestations}, (_, i) => { - const slotInEpoch = intDiv(i, MAX_ATTESTATIONS); - return { - aggregationBits: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true) as List, - data: { - beaconBlockRoot: state.blockRoots[slotInEpoch % SLOTS_PER_HISTORICAL_ROOT], - index: i % committeesPerSlot, - slot: previousEpoch * SLOTS_PER_EPOCH + slotInEpoch, - source: state.previousJustifiedCheckpoint, - target: state.currentJustifiedCheckpoint, - }, - inclusionDelay: 1, - proposerIndex: i, - }; - }) as List; - // current epoch attestations - const numCurAttestations = (SLOTS_PER_EPOCH - 1) * MAX_ATTESTATIONS; - state.currentEpochAttestations = Array.from({length: numCurAttestations}, (_, i) => { - const slotInEpoch = intDiv(i, MAX_ATTESTATIONS); - return { - aggregationBits: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true) as List, - data: { - beaconBlockRoot: state.blockRoots[slotInEpoch % SLOTS_PER_HISTORICAL_ROOT], - index: i % committeesPerSlot, - slot: currentEpoch * SLOTS_PER_EPOCH + slotInEpoch, - source: state.currentJustifiedCheckpoint, - target: { - epoch: currentEpoch, - root: state.blockRoots[(currentEpoch * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT], - }, - }, - inclusionDelay: 1, - proposerIndex: i, - }; - }) as List; - // no justificationBits - phase0State = ssz.phase0.BeaconState.createTreeBackedFromStruct(state); - logger.verbose("Loaded phase0 state", { - slot: phase0State.slot, - numValidators: phase0State.validators.length, - }); - // cache roots - phase0State.hashTreeRoot(); - } - return phase0State.clone(); -} - /** * This is generated from Medalla block 756417 */ -export function generatePerformanceBlockPhase0(): TreeBacked { +export function generatePerformanceBlockPhase0(): phase0.SignedBeaconBlock { if (!phase0SignedBlock) { - const block = ssz.phase0.SignedBeaconBlock.defaultValue(); + const block = ssz.phase0.SignedBeaconBlock.defaultValue; const parentState = generatePerfTestCachedStatePhase0(); block.message.slot = parentState.slot; - block.message.proposerIndex = parentState.getBeaconProposer(parentState.slot); + block.message.proposerIndex = parentState.epochCtx.getBeaconProposer(parentState.slot); block.message.parentRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(parentState.latestBlockHeader); block.message.stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee"); block.message.body.randaoReveal = fromHexString( "0x8a5d2673c48f22f6ed19462efec35645db490df29eed2f56321dbe4a89b2463b0c902095a7ab74a2dc5b7f67edb1a19507ea3d4361d5af9cb0a524945c91638dfd6568841486813a2c45142659d6d9403f5081febb123a7931edbc248b9d0025" ); // eth1Data, graffiti, attestations - phase0SignedBlock = ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct(block); + phase0SignedBlock = block; logger.verbose("Loaded block", {slot: phase0SignedBlock.message.slot}); } - return phase0SignedBlock.clone(); + + return phase0SignedBlock; } -function buildPerformanceStateAllForks(state: allForks.BeaconState, pubkeysArg?: Uint8Array[]): allForks.BeaconState { +function buildPerformanceStatePhase0(pubkeysArg?: Uint8Array[]): phase0.BeaconState { + const slot = epoch * SLOTS_PER_EPOCH; const pubkeys = pubkeysArg || getPubkeys().pubkeys; - state.genesisTime = 1596546008; - state.genesisValidatorsRoot = fromHexString("0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"); - state.slot = epoch * SLOTS_PER_EPOCH; - state.fork = { - currentVersion: fromHexString("0x00000001"), - previousVersion: fromHexString("0x00000001"), - epoch: 0, - }; - state.latestBlockHeader = { - slot: state.slot - 1, - proposerIndex: 80882, - parentRoot: fromHexString("0x5b83c3078e474b86af60043eda82a34c3c2e5ebf83146b14d9d909aea4163ef2"), - stateRoot: fromHexString("0x2761ae355e8a53c11e0e37d5e417f8984db0c53fa83f1bc65f89c6af35a196a7"), - bodyRoot: fromHexString("0x249a1962eef90e122fa2447040bfac102798b1dba9c73e5593bc5aa32eb92bfd"), - }; - state.blockRoots = Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)); - state.stateRoots = Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)); - // historicalRoots - state.eth1Data = { - depositCount: pubkeys.length, - depositRoot: fromHexString("0xcb1f89a924cfd31224823db5a41b1643f10faa7aedf231f1e28887f6ee98c047"), - blockHash: fromHexString("0x701fb2869ce16d0f1d14f6705725adb0dec6799da29006dfc6fff83960298f21"), - }; - state.eth1DataVotes = (Array.from( + const currentEpoch = computeEpochAtSlot(slot - 1); + + return { + // Misc + genesisTime: 1596546008, + genesisValidatorsRoot: fromHexString("0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"), + slot: epoch * SLOTS_PER_EPOCH, + fork: { + currentVersion: fromHexString("0x00000001"), + previousVersion: fromHexString("0x00000001"), + epoch: 0, + }, + // History + latestBlockHeader: { + slot: slot - 1, + proposerIndex: 80882, + parentRoot: fromHexString("0x5b83c3078e474b86af60043eda82a34c3c2e5ebf83146b14d9d909aea4163ef2"), + stateRoot: fromHexString("0x2761ae355e8a53c11e0e37d5e417f8984db0c53fa83f1bc65f89c6af35a196a7"), + bodyRoot: fromHexString("0x249a1962eef90e122fa2447040bfac102798b1dba9c73e5593bc5aa32eb92bfd"), + }, + blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)), + stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)), + historicalRoots: [], + // Eth1 + eth1Data: { + depositCount: pubkeys.length, + depositRoot: fromHexString("0xcb1f89a924cfd31224823db5a41b1643f10faa7aedf231f1e28887f6ee98c047"), + blockHash: fromHexString("0x701fb2869ce16d0f1d14f6705725adb0dec6799da29006dfc6fff83960298f21"), + }, // minus one so that inserting 1 from block works - {length: EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH - 1}, - (_, i) => { - return { - depositCount: i, - depositRoot: Buffer.alloc(32, i), - blockHash: Buffer.alloc(32, i), - }; - } - ) as unknown) as List; - state.eth1DepositIndex = pubkeys.length; - state.validators = pubkeys.map((_, i) => ({ - pubkey: pubkeys[i], - withdrawalCredentials: Buffer.alloc(32, i), - effectiveBalance: 31000000000, - slashed: false, - activationEligibilityEpoch: 0, - activationEpoch: 0, - exitEpoch: Infinity, - withdrawableEpoch: Infinity, - })) as List; - state.balances = Array.from({length: pubkeys.length}, () => 31217089836) as List; - state.randaoMixes = Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)); - // no slashings - const currentEpoch = computeEpochAtSlot(state.slot - 1); - state.previousJustifiedCheckpoint = { - epoch: currentEpoch - 2, - root: fromHexString("0x3fe60bf06a57b0956cd1f8181d26649cf8bf79e48bf82f55562e04b33d4785d4"), - }; - state.currentJustifiedCheckpoint = { - epoch: currentEpoch - 1, - root: fromHexString("0x3ba0913d2fb5e4cbcfb0d39eb15803157c1e769d63b8619285d8fdabbd8181c7"), + eth1DataVotes: newFilledArray(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH - 1, { + depositCount: 1, + depositRoot: Buffer.alloc(32, 1), + blockHash: Buffer.alloc(32, 1), + }), + eth1DepositIndex: pubkeys.length, + // Registry + validators: pubkeys.map((_, i) => ({ + pubkey: pubkeys[i], + withdrawalCredentials: Buffer.alloc(32, i), + effectiveBalance: 31000000000, + slashed: false, + activationEligibilityEpoch: 0, + activationEpoch: 0, + exitEpoch: Infinity, + withdrawableEpoch: Infinity, + })), + balances: Array.from({length: pubkeys.length}, () => 31217089836), + randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)), + // Slashings + slashings: ssz.phase0.Slashings.defaultValue, + previousEpochAttestations: [], + currentEpochAttestations: [], + // Finality + justificationBits: BitArray.fromBitLen(4), + previousJustifiedCheckpoint: { + epoch: currentEpoch - 2, + root: fromHexString("0x3fe60bf06a57b0956cd1f8181d26649cf8bf79e48bf82f55562e04b33d4785d4"), + }, + currentJustifiedCheckpoint: { + epoch: currentEpoch - 1, + root: fromHexString("0x3ba0913d2fb5e4cbcfb0d39eb15803157c1e769d63b8619285d8fdabbd8181c7"), + }, + finalizedCheckpoint: { + epoch: currentEpoch - 3, + root: fromHexString("0x122b8ff579d0c8f8a8b66326bdfec3f685007d2842f01615a0768870961ccc17"), + }, }; - state.finalizedCheckpoint = { - epoch: currentEpoch - 3, - root: fromHexString("0x122b8ff579d0c8f8a8b66326bdfec3f685007d2842f01615a0768870961ccc17"), - }; - return state; } export function generateTestCachedBeaconStateOnlyValidators({ @@ -372,13 +408,13 @@ export function generateTestCachedBeaconStateOnlyValidators({ index2pubkey.push(pubkeyObj); } - const state = ssz.phase0.BeaconState.defaultTreeBacked(); + const state = ssz.phase0.BeaconState.defaultViewDU; state.slot = slot; - const activeValidator = ssz.phase0.Validator.createTreeBackedFromStruct({ + const activeValidator = ssz.phase0.Validator.toViewDU({ pubkey: Buffer.alloc(48, 0), withdrawalCredentials: Buffer.alloc(32, 0), - effectiveBalance: 31000000000, + effectiveBalance: MAX_EFFECTIVE_BALANCE, slashed: false, activationEligibilityEpoch: 0, activationEpoch: 0, @@ -389,17 +425,27 @@ export function generateTestCachedBeaconStateOnlyValidators({ for (let i = 0; i < vc; i++) { const validator = activeValidator.clone(); validator.pubkey = pubkeys[i]; - state.validators[i] = validator; + state.validators.push(validator); } - state.balances = Array.from({length: pubkeys.length}, () => 31217089836) as List; - state.randaoMixes = Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)); + state.balances = ssz.phase0.Balances.toViewDU(newFilledArray(pubkeys.length, MAX_EFFECTIVE_BALANCE)); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU( + newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, Buffer.alloc(32, 0xdd)) + ); + + // Commit ViewDU changes + state.commit(); - return createCachedBeaconState(config, state as TreeBacked, { + // Sanity check for .commit() above + if (state.validators.length !== vc) { + throw Error(`Wrong number of validators in the state: ${state.validators.length} !== ${vc}`); + } + + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, - }); + }) as CachedBeaconStateAllForks; } const initialValue = null; @@ -471,6 +517,6 @@ export async function getNetworkCachedState( fs.writeFileSync(filepath, stateSsz); } - const stateTB = config.getForkTypes(slot).BeaconState.createTreeBackedFromBytes(stateSsz); - return createCachedBeaconState(config, stateTB); + const stateView = config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz); + return createCachedBeaconStateTest(stateView as BeaconStateAllForks, config) as CachedBeaconStateAllForks; } diff --git a/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 9cba6c12fc50..bbd78b4a9611 100644 --- a/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -1,25 +1,21 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; -import {allForks, ssz} from "@chainsafe/lodestar-types"; import {generateAttestationData} from "../../utils/attestation"; import {expect} from "chai"; import {EMPTY_SIGNATURE} from "../../../src"; -import {phase0, createCachedBeaconState} from "../../../src"; -import {generateState} from "../../utils/state"; +import {phase0} from "../../../src"; +import {generateCachedState} from "../../utils/state"; import {generateValidators} from "../../utils/validator"; describe("validate indexed attestation", () => { - const treeState = ssz.phase0.BeaconState.createTreeBackedFromStruct( - generateState({ - validators: generateValidators(100, { - balance: MAX_EFFECTIVE_BALANCE, - activation: 0, - withdrawableEpoch: FAR_FUTURE_EPOCH, - exit: FAR_FUTURE_EPOCH, - }), - }) - ); + const state = generateCachedState(config, { + validators: generateValidators(100, { + balance: MAX_EFFECTIVE_BALANCE, + activation: 0, + withdrawableEpoch: FAR_FUTURE_EPOCH, + exit: FAR_FUTURE_EPOCH, + }), + }); const testValues = [ { @@ -42,10 +38,9 @@ describe("validate indexed attestation", () => { for (const testValue of testValues) { it(testValue.name, function () { const attestationData = generateAttestationData(0, 1); - const state = createCachedBeaconState(config, treeState.clone() as TreeBacked); const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: testValue.indices as List, + attestingIndices: testValue.indices, data: attestationData, signature: EMPTY_SIGNATURE, }; diff --git a/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts b/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts new file mode 100644 index 000000000000..d37f444e6888 --- /dev/null +++ b/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts @@ -0,0 +1,57 @@ +import {ssz} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/lodestar-utils"; +import {expect} from "chai"; +import {createCachedBeaconStateTest} from "../utils/state"; + +describe("CachedBeaconState", () => { + it("Clone and mutate", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU; + const state1 = createCachedBeaconStateTest(stateView); + const state2 = state1.clone(); + + state1.slot = 1; + expect(state2.slot).to.equal(0, "state2.slot was mutated"); + + const prevRoot = state2.currentJustifiedCheckpoint.root; + const newRoot = Buffer.alloc(32, 1); + state1.currentJustifiedCheckpoint.root = newRoot; + expect(toHexString(state2.currentJustifiedCheckpoint.root)).to.equal( + toHexString(prevRoot), + "state2.currentJustifiedCheckpoint.root was mutated" + ); + + state1.epochCtx.epoch = 1; + expect(state2.epochCtx.epoch).to.equal(0, "state2.epochCtx.epoch was mutated"); + }); + + it("Auto-commit on hashTreeRoot", () => { + // Use Checkpoint instead of BeaconState to speed up the test + const cp1 = ssz.phase0.Checkpoint.defaultViewDU; + const cp2 = ssz.phase0.Checkpoint.defaultViewDU; + + cp1.epoch = 1; + cp2.epoch = 1; + + // Only commit state1 beforehand + cp1.commit(); + expect(toHexString(cp1.hashTreeRoot())).to.equal( + toHexString(cp2.hashTreeRoot()), + ".hashTreeRoot() does not automatically commit" + ); + }); + + it("Auto-commit on serialize", () => { + const cp1 = ssz.phase0.Checkpoint.defaultViewDU; + const cp2 = ssz.phase0.Checkpoint.defaultViewDU; + + cp1.epoch = 1; + cp2.epoch = 1; + + // Only commit state1 beforehand + cp1.commit(); + expect(toHexString(cp1.serialize())).to.equal( + toHexString(cp2.serialize()), + ".serialize() does not automatically commit" + ); + }); +}); diff --git a/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts index 44bc9961b634..de4f5e13624f 100644 --- a/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -1,19 +1,19 @@ import crypto from "node:crypto"; import bls from "@chainsafe/bls"; -import {BitList, List, TreeBacked} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; -import {ValidatorIndex, BLSSignature, ssz} from "@chainsafe/lodestar-types"; +import {ValidatorIndex, BLSSignature} from "@chainsafe/lodestar-types"; import {ZERO_HASH} from "../../../src/constants"; -import {generateState} from "../../utils/state"; +import {generateCachedState} from "../../utils/state"; import {generateValidators} from "../../utils/validator"; import {expect} from "chai"; -import {phase0, createCachedBeaconState, allForks} from "../../../src"; +import {phase0, allForks} from "../../../src"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; + +const EMPTY_SIGNATURE = Buffer.alloc(96); describe("signatureSets", () => { it("should aggregate all signatures from a block", () => { - const EMPTY_SIGNATURE = Buffer.alloc(96); - const block: phase0.BeaconBlock = { slot: 0, proposerIndex: 0, @@ -32,20 +32,17 @@ describe("signatureSets", () => { {proposerIndex: 0, signature: EMPTY_SIGNATURE}, {proposerIndex: 0, signature: EMPTY_SIGNATURE} ), - ] as List, + ], attesterSlashings: [ getMockAttesterSlashings( - {attestingIndices: [0] as List, signature: EMPTY_SIGNATURE}, - {attestingIndices: [0] as List, signature: EMPTY_SIGNATURE} + {attestingIndices: [0], signature: EMPTY_SIGNATURE}, + {attestingIndices: [0], signature: EMPTY_SIGNATURE} ), - ] as List, - attestations: [ - getMockAttestations({attestingIndices: [0] as List, signature: EMPTY_SIGNATURE}), - ] as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: [ - getMockSignedVoluntaryExit({validatorIndex: 0, signature: EMPTY_SIGNATURE}), - ] as List, + ], + // Set to 1 since there's only one validator per committee + attestations: [getMockAttestations(1)], + deposits: [] as phase0.Deposit[], + voluntaryExits: [getMockSignedVoluntaryExit({validatorIndex: 0, signature: EMPTY_SIGNATURE})], }, }; @@ -64,10 +61,7 @@ describe("signatureSets", () => { validator.pubkey = bls.SecretKey.fromKeygen().toPublicKey().toBytes(); } - const state = createCachedBeaconState( - config, - ssz.phase0.BeaconState.createTreeBackedFromStruct(generateState({validators})) as TreeBacked - ); + const state = generateCachedState(config, {validators}); const signatureSets = allForks.getAllBlockSignatureSets(state, signedBlock); expect(signatureSets.length).to.equal( @@ -113,7 +107,7 @@ function getMockSignedBeaconBlockHeader(data: IBlockProposerData): phase0.Signed } interface IIndexAttestationData { - attestingIndices: List; + attestingIndices: ValidatorIndex[]; signature: BLSSignature; } @@ -148,11 +142,11 @@ function getAttestationData(): phase0.AttestationData { }; } -function getMockAttestations(data: IIndexAttestationData): phase0.Attestation { +function getMockAttestations(bitLen: number): phase0.Attestation { return { - aggregationBits: [true] as BitList, + aggregationBits: BitArray.fromSingleBit(bitLen, 0), data: getAttestationData(), - signature: data.signature, + signature: EMPTY_SIGNATURE, }; } diff --git a/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts b/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts deleted file mode 100644 index d3ea9d57d29e..000000000000 --- a/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import {MAX_VALIDATORS_PER_COMMITTEE, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; -import {ssz} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; -import {expect} from "chai"; -import { - getUint8ByteToBitBooleanArray, - bitsToUint8Array, - zipIndexesSyncCommitteeBits, - zipAllIndexesSyncCommitteeBits, - getSingleBitIndex, - AggregationBitsErrorCode, -} from "../../../src"; - -const BITS_PER_BYTE = 8; - -describe("aggregationBits", function () { - const testCases: {name: string; data: boolean[]; numBytes: number}[] = [ - {name: "8 bits all true", data: [true, true, true, true, true, true, true, true], numBytes: 1}, - {name: "8 bits with true and false", data: [false, false, false, false, false, true, false, true], numBytes: 1}, - { - name: "10 bits with true and false", - data: [false, false, false, false, false, true, false, true, true, true], - numBytes: 2, - }, - { - name: "" + MAX_VALIDATORS_PER_COMMITTEE + " bits all true", - data: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true), - numBytes: Math.ceil(MAX_VALIDATORS_PER_COMMITTEE / 8), - }, - ]; - - it("getUint8ByteToBitBooleanArray", () => { - expect(getUint8ByteToBitBooleanArray(1)).to.be.deep.equal([true, false, false, false, false, false, false, false]); - expect(getUint8ByteToBitBooleanArray(5)).to.be.deep.equal([true, false, true, false, false, false, false, false]); - }); - - for (const {name, data, numBytes} of testCases) { - it(name, () => { - const tree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(data as List); - const aggregationBytes = bitsToUint8Array(tree, ssz.phase0.CommitteeBits); - expect(aggregationBytes.length).to.be.equal(numBytes, "number of bytes is incorrect"); - const aggregationBits: boolean[] = []; - for (let i = 0; i < tree.length; i++) { - aggregationBits.push(getAggregationBit(aggregationBytes, i)); - } - expect(aggregationBits).to.be.deep.equal(data, "incorrect extracted aggregationBits"); - }); - } - - it("getUint8ByteToBitBooleanArray - all values in 8 bytes", () => { - for (let i = 0; i <= 0xff; i++) { - const boolArr = getUint8ByteToBitBooleanArray(i); - const tree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(boolArr as List); - const bytes = tree.serialize(); - expect(bytes[0]).to.equal(i, `Wrong serialization of ${i}: ${JSON.stringify(boolArr)}`); - } - }); -}); - -describe("zipIndexesSyncCommitteeBits and zipAllIndexesSyncCommitteeBits", function () { - const committeeIndices = Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => i * 2); - const pivot = 3; - // 3 first bits are true - const syncCommitteeBits = Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => { - return i < pivot ? true : false; - }); - - // remaining bits are false - const expectedUnparticipantIndices: number[] = []; - for (let i = pivot; i < SYNC_COMMITTEE_SIZE; i++) { - expectedUnparticipantIndices.push(committeeIndices[i]); - } - - it("should extract from TreeBacked SyncAggregate", function () { - const syncAggregate = ssz.altair.SyncAggregate.defaultTreeBacked(); - syncAggregate.syncCommitteeBits = syncCommitteeBits; - expect(zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits)).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from TreeBacked SyncAggregate" - ); - const [participantIndices, unparticipantIndices] = zipAllIndexesSyncCommitteeBits( - committeeIndices, - syncAggregate.syncCommitteeBits - ); - expect(participantIndices).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from TreeBacked SyncAggregate" - ); - expect(unparticipantIndices).to.be.deep.equal( - expectedUnparticipantIndices, - "Incorrect unparticipant indices from TreeBacked SyncAggregate" - ); - }); - - it("should extract from struct SyncAggregate", function () { - const syncAggregate = ssz.altair.SyncAggregate.defaultValue(); - syncAggregate.syncCommitteeBits = syncCommitteeBits; - expect(zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits)).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from struct SyncAggregate" - ); - const [participantIndices, unparticipantIndices] = zipAllIndexesSyncCommitteeBits( - committeeIndices, - syncAggregate.syncCommitteeBits - ); - expect(participantIndices).to.be.deep.equal([0, 2, 4], "Incorrect participant indices from struct SyncAggregate"); - expect(unparticipantIndices).to.be.deep.equal( - expectedUnparticipantIndices, - "Incorrect unparticipant indices from struct SyncAggregate" - ); - }); -}); - -describe("getSingleBitIndex", () => { - const len = MAX_VALIDATORS_PER_COMMITTEE; - - const testCases: {id: string; bitList: boolean[]; res: number | Error | string}[] = [ - {id: "bit 0 true", bitList: [true, false, false, false, false, false, false, false, false, false], res: 0}, - {id: "bit 4 true", bitList: [false, false, false, false, true, false, false, false, false, false], res: 4}, - {id: "bit 9 true", bitList: [false, false, false, false, false, false, false, false, false, true], res: 9}, - { - id: "2 bits true", - bitList: [true, false, false, false, true, false, false, false, false, false], - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - { - id: `${len} all true`, - bitList: Array.from({length: len}, () => true), - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - { - id: `${len} all false`, - bitList: Array.from({length: len}, () => false), - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - ]; - - for (const {id, res, bitList} of testCases) { - const struct = bitList as List; - const treeBacked = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(struct); - - it(`${id} - struct`, () => { - if (typeof res === "number") { - expect(getSingleBitIndex(struct)).to.equal(res); - } else { - expect(() => getSingleBitIndex(struct)).to.throw(res as Error); - } - }); - - it(`${id} - treeBacked`, () => { - if (typeof res === "number") { - expect(getSingleBitIndex(treeBacked)).to.equal(res); - } else { - expect(() => getSingleBitIndex(treeBacked)).to.throw(res as Error); - } - }); - } -}); - -/** - * Get aggregation bit (true/false) from an aggregation bytes array and validator index in committee. - * Notice: If we want to access the bit in batch, using this method is not efficient, check the performance - * test for an example of how to do that. - */ -export function getAggregationBit(attBytes: number[] | Uint8Array, indexInCommittee: number): boolean { - const byteIndex = Math.floor(indexInCommittee / BITS_PER_BYTE); - const indexInByte = indexInCommittee % BITS_PER_BYTE; - return getUint8ByteToBitBooleanArray(attBytes[byteIndex])[indexInByte]; -} diff --git a/packages/beacon-state-transition/test/unit/util/balance.test.ts b/packages/beacon-state-transition/test/unit/util/balance.test.ts index a09b9c374e83..e6ebe2723ada 100644 --- a/packages/beacon-state-transition/test/unit/util/balance.test.ts +++ b/packages/beacon-state-transition/test/unit/util/balance.test.ts @@ -1,15 +1,14 @@ import {assert, expect} from "chai"; import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; -import {List, readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; -import {phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; import {increaseBalance, decreaseBalance, getTotalBalance, isActiveValidator} from "../../../src/util"; +import {getEffectiveBalanceIncrementsZeroed, getEffectiveBalanceIncrementsZeroInactive} from "../../../src"; import {generateValidators} from "../../utils/validator"; import {generateCachedState, generateState} from "../../utils/state"; -import {getEffectiveBalanceIncrementsZeroInactive, getEffectiveBalanceIncrementsZeroed} from "../../../src"; describe("getTotalBalance", () => { it("should return correct balances", () => { @@ -19,7 +18,7 @@ describe("getTotalBalance", () => { for (const v of validators) { v.effectiveBalance = validatorBalance; } - const state: phase0.BeaconState = generateState({validators: validators}); + const state = generateState({validators: validators}); const validatorIndices: ValidatorIndex[] = Array.from({length: num}, (_, i) => i); const result = getTotalBalance(state, validatorIndices); @@ -30,8 +29,8 @@ describe("getTotalBalance", () => { it("should return correct balances", () => { const num = 5; const validators = generateValidators(num); - const balances = Array.from({length: num}, () => 0) as List; - const state: phase0.BeaconState = generateState({validators: validators, balances}); + const balances = Array.from({length: num}, () => 0); + const state = generateState({validators: validators, balances}); const validatorIndices: ValidatorIndex[] = Array.from({length: num}, (_, i) => i); const result = getTotalBalance(state, validatorIndices); @@ -43,16 +42,13 @@ describe("getTotalBalance", () => { describe("increaseBalance", () => { it("should add to a validators balance", () => { const state = generateCachedState(); + state.balances.push(0); + expect(state.balances.get(0)).to.be.equal(0); - state.validators.push(generateValidators(1)[0]); - state.balanceList.push(0); - expect(state.balanceList.get(0)).to.be.equal(0); - expect(state.balances[0]).to.be.equal(0); const delta = 5; for (let i = 1; i < 10; i++) { increaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(delta * i); - expect(state.balances[0]).to.be.equal(delta * i); + expect(state.balances.get(0)).to.be.equal(delta * i); } }); }); @@ -60,29 +56,27 @@ describe("increaseBalance", () => { describe("decreaseBalance", () => { it("should subtract from a validators balance", () => { const state = generateCachedState(); - state.validators.push(generateValidators(1)[0]); const initial = 100; - state.balanceList.push(initial); + state.balances.push(initial); + const delta = 5; for (let i = 1; i < 10; i++) { decreaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(initial - delta * i); - expect(state.balances[0]).to.be.equal(initial - delta * i); + expect(state.balances.get(0)).to.be.equal(initial - delta * i); } }); + it("should not make a validators balance < 0", () => { const state = generateCachedState(); - state.validators.push(generateValidators(1)[0]); const initial = 10; - state.balanceList.push(initial); + state.balances.push(initial); const delta = 11; decreaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(0); - expect(state.balances[0]).to.be.equal(0); + expect(state.balances.get(0)).to.be.equal(0); }); }); -describe("getEffectiveBalances", () => { +describe("getEffectiveBalanceIncrementsZeroInactive", () => { it("should get correct effective balances", () => { const justifiedState = generateCachedState(minimalConfig, { validators: [ @@ -92,10 +86,10 @@ describe("getEffectiveBalances", () => { ...generateValidators(4, {activation: 0, exit: Infinity, balance: 32e9}), // not active ...generateValidators(5, {activation: Infinity, exit: Infinity, balance: 32e9}), - ] as List, + ], }); - const justifiedEpoch = justifiedState.currentShuffling.epoch; - const validators = readonlyValuesListOfLeafNodeStruct(justifiedState.validators); + const justifiedEpoch = justifiedState.epochCtx.currentShuffling.epoch; + const validators = justifiedState.validators.getAllReadonlyValues(); const effectiveBalances = getEffectiveBalanceIncrementsZeroed(validators.length); for (let i = 0, len = validators.length; i < len; i++) { diff --git a/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts index 2d0e5280ff25..396dab631dc2 100644 --- a/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts @@ -1,10 +1,16 @@ +import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {config} from "@chainsafe/lodestar-config/default"; import {ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "../../../src"; +import {createCachedBeaconState, PubkeyIndexMap} from "../../../src"; describe("CachedBeaconState", () => { it("Create empty CachedBeaconState", () => { - const emptyState = ssz.phase0.BeaconState.defaultTreeBacked(); - createCachedBeaconState(config, emptyState); + const emptyState = ssz.phase0.BeaconState.defaultViewDU; + + createCachedBeaconState(emptyState, { + config: createIBeaconConfig(config, emptyState.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); }); }); diff --git a/packages/beacon-state-transition/test/unit/util/epoch.test.ts b/packages/beacon-state-transition/test/unit/util/epoch.test.ts index 8afdb55cf1d1..ececf95c0384 100644 --- a/packages/beacon-state-transition/test/unit/util/epoch.test.ts +++ b/packages/beacon-state-transition/test/unit/util/epoch.test.ts @@ -1,7 +1,7 @@ import {assert} from "chai"; import {GENESIS_SLOT, MAX_SEED_LOOKAHEAD} from "@chainsafe/lodestar-params"; -import {phase0, Epoch, Slot} from "@chainsafe/lodestar-types"; +import {Epoch, Slot} from "@chainsafe/lodestar-types"; import { computeStartSlotAtEpoch, getPreviousEpoch, @@ -61,7 +61,7 @@ describe("getPreviousEpoch", () => { for (const testValue of testValues) { it("epoch should return previous epoch", () => { - const state: phase0.BeaconState = generateState({slot: testValue.slot}); + const state = generateState({slot: testValue.slot}); const result = getPreviousEpoch(state); assert.equal(result, testValue.expectedEpoch); }); diff --git a/packages/beacon-state-transition/test/unit/util/flags.test.ts b/packages/beacon-state-transition/test/unit/util/flags.test.ts new file mode 100644 index 000000000000..6566ef8205be --- /dev/null +++ b/packages/beacon-state-transition/test/unit/util/flags.test.ts @@ -0,0 +1,40 @@ +import {expect} from "chai"; + +describe("Altair status flags", () => { + for (let prev = 0b000; prev <= 0b111; prev++) { + for (let att = 0b000; att <= 0b111; att++) { + it(`prevFlag ${toStr(prev)} attFlag ${toStr(att)}`, () => { + expect( + // Actual function + toStr(getResFlags(prev, att)) + ).to.equal( + // Naive but correct implementation + toStr(getResFlagsNaive(prev, att)) + ); + }); + } + } +}); + +function toStr(flags: number): string { + return flags.toString(2).padStart(3, "0"); +} + +function getResFlags(prev: number, att: number): number { + return ~prev & att; +} + +function getResFlagsNaive(prev: number, att: number): number { + let out = 0; + + for (let i = 0; i < 3; i++) { + const mask = 1 << i; + const hasPrev = (prev & mask) === mask; + const hasAtt = (att & mask) === mask; + if (!hasPrev && hasAtt) { + out |= mask; + } + } + + return out; +} diff --git a/packages/beacon-state-transition/test/unit/util/interface.test.ts b/packages/beacon-state-transition/test/unit/util/interface.test.ts deleted file mode 100644 index c48f9782aa1a..000000000000 --- a/packages/beacon-state-transition/test/unit/util/interface.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import {config} from "@chainsafe/lodestar-config/default"; -import {ssz} from "@chainsafe/lodestar-types"; -import {fromHexString, List, TreeBacked} from "@chainsafe/ssz"; -import {expect} from "chai"; -import {phase0, CachedBeaconStatePhase0, createCachedBeaconState} from "../../../src"; -import {generateState} from "../../utils/state"; - -const NUM_VALIDATORS = 1001; - -describe("CachedBeaconState", function () { - let state: TreeBacked; - let wrappedState: CachedBeaconStatePhase0; - - before(function () { - this.timeout(0); - const validators: phase0.Validator[] = []; - for (let i = 0; i < NUM_VALIDATORS; i++) { - validators.push({ - pubkey: fromHexString( - // randomly pregenerated pubkey - "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" - ), - withdrawalCredentials: Buffer.alloc(32), - effectiveBalance: 1000000, - slashed: false, - activationEligibilityEpoch: i + 10, - activationEpoch: i, - exitEpoch: i + 20, - withdrawableEpoch: i + 30, - }); - } - const defaultState = generateState({validators: validators as List}); - state = ssz.phase0.BeaconState.createTreeBackedFromStruct(defaultState); - }); - - beforeEach(() => { - state = state.clone(); - wrappedState = createCachedBeaconState(config, state); - }); - - it("should read the same value of TreeBacked", () => { - expect(state.validators[1000].activationEpoch).to.be.equal(1000); - expect(wrappedState.validators[1000].activationEpoch).to.be.equal(1000); - }); - - it("should modify both state and wrappedState", () => { - const oldFlatValidator = wrappedState.validators[1000]; - const validator = wrappedState.validators[1000]; - validator.activationEpoch = 2020; - validator.exitEpoch = 2030; - - expect(wrappedState.validators[1000].activationEpoch).to.be.equal(2020); - expect(wrappedState.validators[1000].exitEpoch).to.be.equal(2030); - // other property is the same - expect(wrappedState.validators[1000].effectiveBalance).to.be.equal(oldFlatValidator.effectiveBalance); - expect(wrappedState.validators[1000].slashed).to.be.equal(oldFlatValidator.slashed); - expect(wrappedState.validators[1000].activationEligibilityEpoch).to.be.equal( - oldFlatValidator.activationEligibilityEpoch - ); - expect(wrappedState.validators[1000].withdrawableEpoch).to.be.equal(oldFlatValidator.withdrawableEpoch); - - expect(state.validators[1000].activationEpoch).to.be.equal(2020); - expect(state.validators[1000].exitEpoch).to.be.equal(2030); - // other property is the same - expect(state.validators[1000].effectiveBalance).to.be.equal(oldFlatValidator.effectiveBalance); - expect(state.validators[1000].slashed).to.be.equal(oldFlatValidator.slashed); - expect(state.validators[1000].activationEligibilityEpoch).to.be.equal(oldFlatValidator.activationEligibilityEpoch); - expect(state.validators[1000].withdrawableEpoch).to.be.equal(oldFlatValidator.withdrawableEpoch); - }); - - it("should add validator to both state and wrappedState", () => { - wrappedState.validators.push({ - pubkey: Buffer.alloc(48), - withdrawalCredentials: Buffer.alloc(32), - effectiveBalance: 1000000, - slashed: false, - activationEligibilityEpoch: NUM_VALIDATORS + 10, - activationEpoch: NUM_VALIDATORS, - exitEpoch: NUM_VALIDATORS + 20, - withdrawableEpoch: NUM_VALIDATORS + 30, - }); - - expect(wrappedState.validators.length).to.be.equal(NUM_VALIDATORS + 1); - expect(state.validators.length).to.be.equal(NUM_VALIDATORS + 1); - expect(wrappedState.validators[NUM_VALIDATORS].activationEpoch).to.be.equal(NUM_VALIDATORS); - expect(state.validators[NUM_VALIDATORS].activationEpoch).to.be.equal(NUM_VALIDATORS); - }); -}); diff --git a/packages/beacon-state-transition/test/unit/util/seed.test.ts b/packages/beacon-state-transition/test/unit/util/seed.test.ts index e5623ce2b7bb..071a6afd9f5e 100644 --- a/packages/beacon-state-transition/test/unit/util/seed.test.ts +++ b/packages/beacon-state-transition/test/unit/util/seed.test.ts @@ -1,27 +1,30 @@ -import {assert} from "chai"; +import {expect} from "chai"; import {GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {getRandaoMix} from "../../../src/util"; import {generateState} from "../../utils/state"; +import {toHexString} from "@chainsafe/ssz"; describe("getRandaoMix", () => { + const randaoMix1 = Buffer.alloc(32, 1); + const randaoMix2 = Buffer.alloc(32, 2); + it("should return first randao mix for GENESIS_EPOCH", () => { // Empty state in 2nd epoch - const state = generateState({ - slot: GENESIS_SLOT + SLOTS_PER_EPOCH, - randaoMixes: [Buffer.from([0xab]), Buffer.from([0xcd])], - }); + const state = generateState({slot: GENESIS_SLOT + SLOTS_PER_EPOCH}); + state.randaoMixes.set(0, randaoMix1); + const res = getRandaoMix(state, GENESIS_EPOCH); - assert(Buffer.from(res as Uint8Array).equals(Uint8Array.from([0xab]))); + expect(toHexString(res)).to.equal(toHexString(randaoMix1)); }); it("should return second randao mix for GENESIS_EPOCH + 1", () => { // Empty state in 2nd epoch - const state = generateState({ - slot: GENESIS_SLOT + SLOTS_PER_EPOCH * 2, - randaoMixes: [Buffer.from([0xab]), Buffer.from([0xcd]), Buffer.from([0xef])], - }); + const state = generateState({slot: GENESIS_SLOT + SLOTS_PER_EPOCH * 2}); + state.randaoMixes.set(0, randaoMix1); + state.randaoMixes.set(1, randaoMix2); + const res = getRandaoMix(state, GENESIS_EPOCH + 1); - assert(Buffer.from(res as Uint8Array).equals(Uint8Array.from([0xcd]))); + expect(toHexString(res)).to.equal(toHexString(randaoMix2)); }); }); diff --git a/packages/beacon-state-transition/test/unit/util/validator.test.ts b/packages/beacon-state-transition/test/unit/util/validator.test.ts index dd1e81862422..bceaa996a30a 100644 --- a/packages/beacon-state-transition/test/unit/util/validator.test.ts +++ b/packages/beacon-state-transition/test/unit/util/validator.test.ts @@ -1,7 +1,6 @@ import {assert, expect} from "chai"; -import {List} from "@chainsafe/ssz"; -import {phase0} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {getActiveValidatorIndices, isActiveValidator, isSlashableValidator} from "../../../src/util"; @@ -17,10 +16,11 @@ describe("getActiveValidatorIndices", () => { const state = generateState(); const activationEpoch = 1; const exitEpoch = 10; - state.validators = Array.from({length: 10}, () => - generateValidator({activation: activationEpoch, exit: exitEpoch}) - ) as List; - const allActiveIndices = Array.from(state.validators).map((_, i) => i); + state.validators = ssz.phase0.Validators.toViewDU( + Array.from({length: 10}, () => generateValidator({activation: activationEpoch, exit: exitEpoch})) + ); + + const allActiveIndices = state.validators.getAllReadonlyValues().map((_, i) => i); const allInactiveIndices: any = []; assert.deepEqual(getActiveValidatorIndices(state, activationEpoch), allActiveIndices); assert.deepEqual(getActiveValidatorIndices(state, exitEpoch), allInactiveIndices); diff --git a/packages/beacon-state-transition/test/utils/attestation.ts b/packages/beacon-state-transition/test/utils/attestation.ts index c30c2dddc1f5..b3dc827e0012 100644 --- a/packages/beacon-state-transition/test/utils/attestation.ts +++ b/packages/beacon-state-transition/test/utils/attestation.ts @@ -1,5 +1,5 @@ -import {List} from "@chainsafe/ssz"; import {phase0, Epoch} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; /** * Generates a fake attestation data for test purposes. @@ -26,7 +26,7 @@ export function generateAttestationData(sourceEpoch: Epoch, targetEpoch: Epoch): export function generateEmptyAttestation(): phase0.Attestation { return { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: 1, index: 0, diff --git a/packages/beacon-state-transition/test/utils/block.ts b/packages/beacon-state-transition/test/utils/block.ts index 7c165890c6c8..867d7d639068 100644 --- a/packages/beacon-state-transition/test/utils/block.ts +++ b/packages/beacon-state-transition/test/utils/block.ts @@ -1,5 +1,4 @@ import crypto from "node:crypto"; -import {List} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {ZERO_HASH} from "../../src/constants"; @@ -17,11 +16,11 @@ export function generateEmptyBlock(): phase0.BeaconBlock { depositCount: 0, }, graffiti: crypto.randomBytes(32), - proposerSlashings: ([] as phase0.ProposerSlashing[]) as List, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [] as phase0.ProposerSlashing[], + attesterSlashings: [] as phase0.AttesterSlashing[], + attestations: [] as phase0.Attestation[], + deposits: [] as phase0.Deposit[], + voluntaryExits: [] as phase0.SignedVoluntaryExit[], }, }; } diff --git a/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts b/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts deleted file mode 100644 index eeb1e14829d9..000000000000 --- a/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts +++ /dev/null @@ -1,6 +0,0 @@ -import {phase0} from "@chainsafe/lodestar-types"; - -export interface IStateTestCase { - pre: phase0.BeaconState; - post: phase0.BeaconState; -} diff --git a/packages/beacon-state-transition/test/utils/state.ts b/packages/beacon-state-transition/test/utils/state.ts index 2564ab611688..f4edc6bde94a 100644 --- a/packages/beacon-state-transition/test/utils/state.ts +++ b/packages/beacon-state-transition/test/utils/state.ts @@ -1,4 +1,4 @@ -import {List, Vector} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; import { EPOCHS_PER_HISTORICAL_VECTOR, @@ -7,15 +7,23 @@ import { GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT, } from "@chainsafe/lodestar-params"; -import {phase0, ssz} from "@chainsafe/lodestar-types"; +import {phase0, Root, ssz} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/default"; import {ZERO_HASH} from "../../src/constants"; import {newZeroedBigIntArray} from "../../src/util"; import {generateEmptyBlock} from "./block"; -import {CachedBeaconStateAllForks, createCachedBeaconState} from "../../src"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import { + BeaconStatePhase0, + CachedBeaconStateAllForks, + BeaconStateAllForks, + createCachedBeaconState, + PubkeyIndexMap, +} from "../../src"; +import {createIBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateCache} from "../../src/cache/stateCache"; +import {EpochContextOpts} from "../../src/cache/epochContext"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. @@ -28,8 +36,8 @@ type TestBeaconState = Partial; * @param {TestBeaconState} opts * @returns {BeaconState} */ -export function generateState(opts?: TestBeaconState): phase0.BeaconState { - return { +export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { + return ssz.phase0.BeaconState.toViewDU({ genesisTime: Math.floor(Date.now() / 1000), genesisValidatorsRoot: ZERO_HASH, slot: GENESIS_SLOT, @@ -47,21 +55,21 @@ export function generateState(opts?: TestBeaconState): phase0.BeaconState { }, blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: ([] as Vector[]) as List>, + historicalRoots: [] as Root[], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: ([] as phase0.Eth1Data[]) as List, + eth1DataVotes: [] as phase0.Eth1Data[], eth1DepositIndex: 0, - validators: ([] as phase0.Validator[]) as List, - balances: ([] as number[]) as List, + validators: [] as phase0.Validator[], + balances: [] as number[], randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: newZeroedBigIntArray(EPOCHS_PER_SLASHINGS_VECTOR), - previousEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - currentEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - justificationBits: [false, false, false, false], + previousEpochAttestations: [] as phase0.PendingAttestation[], + currentEpochAttestations: [] as phase0.PendingAttestation[], + justificationBits: BitArray.fromBitLen(4), previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, root: ZERO_HASH, @@ -75,7 +83,7 @@ export function generateState(opts?: TestBeaconState): phase0.BeaconState { root: ZERO_HASH, }, ...opts, - }; + }); } export function generateCachedState( @@ -83,5 +91,27 @@ export function generateCachedState( opts: TestBeaconState = {} ): CachedBeaconStateAllForks { const state = generateState(opts); - return createCachedBeaconState(config, config.getForkTypes(state.slot).BeaconState.createTreeBackedFromStruct(state)); + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }) as CachedBeaconStateAllForks; +} + +export function createCachedBeaconStateTest( + state: T, + configCustom: IChainForkConfig = config, + opts?: EpochContextOpts +): T & BeaconStateCache { + return createCachedBeaconState( + state, + { + config: createIBeaconConfig(configCustom, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }, + opts + ); } diff --git a/packages/beacon-state-transition/test/utils/validator.ts b/packages/beacon-state-transition/test/utils/validator.ts index 5927eff067b3..ff63e94ba7d8 100644 --- a/packages/beacon-state-transition/test/utils/validator.ts +++ b/packages/beacon-state-transition/test/utils/validator.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {FAR_FUTURE_EPOCH} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; @@ -40,6 +40,6 @@ export function generateValidator(opts: IValidatorGeneratorOpts = {}): phase0.Va * @param {number} n * @returns {Validator[]} */ -export function generateValidators(n: number, opts?: IValidatorGeneratorOpts): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: IValidatorGeneratorOpts): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } diff --git a/packages/cli/package.json b/packages/cli/package.json index ff9128d312f7..8d585b4b0afb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -63,7 +63,7 @@ "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/lodestar-validator": "^0.35.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "@types/lockfile": "^1.0.1", "bip39": "^3.0.2", "deepmerge": "^4.2.2", diff --git a/packages/cli/src/cmds/account/cmds/validator/create.ts b/packages/cli/src/cmds/account/cmds/validator/create.ts index ef9ae8ec3584..1ef8939ab097 100644 --- a/packages/cli/src/cmds/account/cmds/validator/create.ts +++ b/packages/cli/src/cmds/account/cmds/validator/create.ts @@ -80,7 +80,7 @@ and pre-computed deposit RPL data", const {name, passphraseFile, storeWithdrawalKeystore, count} = args; const accountPaths = getAccountPaths(args); const maxEffectiveBalance = MAX_EFFECTIVE_BALANCE; - const depositGwei = Number(args.depositGwei || 0) || maxEffectiveBalance; + const depositGwei = args.depositGwei !== undefined ? parseInt(args.depositGwei, 10) : maxEffectiveBalance; if (depositGwei > maxEffectiveBalance) throw new YargsError(`depositGwei ${depositGwei} is higher than MAX_EFFECTIVE_BALANCE ${maxEffectiveBalance}`); diff --git a/packages/cli/src/cmds/account/cmds/validator/recover.ts b/packages/cli/src/cmds/account/cmds/validator/recover.ts index 08e163632834..f1366dc69dd3 100644 --- a/packages/cli/src/cmds/account/cmds/validator/recover.ts +++ b/packages/cli/src/cmds/account/cmds/validator/recover.ts @@ -65,7 +65,7 @@ export const recover: ICliCommand, - wsState: TreeBacked, + store: BeaconStateAllForks, + wsState: BeaconStateAllForks, wsCheckpoint: Checkpoint -): Promise<{anchorState: TreeBacked; wsCheckpoint: Checkpoint}> { +): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { // Check if the store's state and wsState are compatible if ( store.genesisTime !== wsState.genesisTime || @@ -53,7 +52,7 @@ async function initAndVerifyWeakSubjectivityState( let anchorCheckpoint = wsCheckpoint; if (store.slot > wsState.slot) { anchorState = store; - anchorCheckpoint = getCheckpointFromState(config, store); + anchorCheckpoint = getCheckpointFromState(store); logger.verbose( "Db state is ahead of the provided checkpoint state, using the db state to initialize the beacon chain" ); @@ -85,7 +84,7 @@ export async function initBeaconState( db: IBeaconDb, logger: ILogger, signal: AbortSignal -): Promise<{anchorState: TreeBacked; wsCheckpoint?: Checkpoint}> { +): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint?: Checkpoint}> { // fetch the latest state stored in the db // this will be used in all cases, if it exists, either used during verification of a weak subjectivity state, or used directly as the anchor state const lastDbState = await db.stateArchive.lastValue(); @@ -95,12 +94,12 @@ export async function initBeaconState( // if a weak subjectivity checkpoint has been provided, it is used for additional verification // otherwise, the state itself is used for verification (not bad, because the trusted state has been explicitly provided) const stateBytes = await downloadOrLoadFile(args.weakSubjectivityStateFile); - const wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).createTreeBackedFromBytes(stateBytes); + const wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); const config = createIBeaconConfig(chainForkConfig, wsState.genesisValidatorsRoot); const store = lastDbState ?? wsState; const checkpoint = args.weakSubjectivityCheckpoint ? getCheckpointFromArg(args.weakSubjectivityCheckpoint) - : getCheckpointFromState(config, wsState); + : getCheckpointFromState(wsState); return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsState, checkpoint); } else if (args.weakSubjectivitySyncLatest) { // weak subjectivity sync from a state that needs to be fetched: @@ -137,7 +136,7 @@ export async function initBeaconState( const genesisStateFile = args.genesisStateFile || getGenesisFileUrl(args.network || defaultNetwork); if (genesisStateFile && !args.forceGenesis) { const stateBytes = await downloadOrLoadFile(genesisStateFile); - let anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).createTreeBackedFromBytes(stateBytes); + let anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); const config = createIBeaconConfig(chainForkConfig, anchorState.genesisValidatorsRoot); anchorState = await initStateFromAnchorState(config, db, logger, anchorState); return {anchorState}; diff --git a/packages/cli/src/cmds/dev/handler.ts b/packages/cli/src/cmds/dev/handler.ts index bbac52f94df2..a3ca1cdb493b 100644 --- a/packages/cli/src/cmds/dev/handler.ts +++ b/packages/cli/src/cmds/dev/handler.ts @@ -78,7 +78,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise { if (args.genesisStateFile) { const state = config .getForkTypes(GENESIS_SLOT) - .BeaconState.createTreeBackedFromBytes(await fs.promises.readFile(args.genesisStateFile)); + .BeaconState.deserializeToViewDU(await fs.promises.readFile(args.genesisStateFile)); anchorState = await initStateFromAnchorState(config, db, logger, state); } else { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion diff --git a/packages/cli/src/config/beaconNodeOptions.ts b/packages/cli/src/config/beaconNodeOptions.ts index 5c9b888b8942..3495031f40e9 100644 --- a/packages/cli/src/config/beaconNodeOptions.ts +++ b/packages/cli/src/config/beaconNodeOptions.ts @@ -1,5 +1,4 @@ import deepmerge from "deepmerge"; -import {Json} from "@chainsafe/ssz"; import {defaultOptions, IBeaconNodeOptions} from "@chainsafe/lodestar"; import {isPlainObject, RecursivePartial} from "@chainsafe/lodestar-utils"; import {writeFile, readFile} from "../util"; @@ -52,12 +51,12 @@ export class BeaconNodeOptions { } writeTo(filepath: string): void { - writeFile(filepath, this.beaconNodeOptions as Json); + writeFile(filepath, this.beaconNodeOptions); } } export function writeBeaconNodeOptions(filename: string, config: Partial): void { - writeFile(filename, config as Json); + writeFile(filename, config); } /** diff --git a/packages/cli/src/config/peerId.ts b/packages/cli/src/config/peerId.ts index cbaf9ba5d96d..5726a77e6de4 100644 --- a/packages/cli/src/config/peerId.ts +++ b/packages/cli/src/config/peerId.ts @@ -1,5 +1,4 @@ import PeerId from "peer-id"; -import {Json} from "@chainsafe/ssz"; import {writeFile, readFile} from "../util"; export async function createPeerId(): Promise { @@ -7,7 +6,7 @@ export async function createPeerId(): Promise { } export function writePeerId(filepath: string, peerId: PeerId): void { - writeFile(filepath, (peerId.toJSON() as unknown) as Json); + writeFile(filepath, peerId.toJSON()); } export async function readPeerId(filepath: string): Promise { diff --git a/packages/cli/src/depositContract/depositData.ts b/packages/cli/src/depositContract/depositData.ts index b103aee54ecf..241e593d43cd 100644 --- a/packages/cli/src/depositContract/depositData.ts +++ b/packages/cli/src/depositContract/depositData.ts @@ -1,5 +1,5 @@ import {ethers} from "ethers"; -import {hash, Json, toHexString} from "@chainsafe/ssz"; +import {hash, toHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import bls, {SecretKey, PublicKey} from "@chainsafe/bls"; @@ -15,13 +15,12 @@ function getDepositInterface(): ethers.utils.Interface { export function decodeEth1TxData(bytes: string, amount: string): {depositData: phase0.DepositData; root: string} { const depositContract = getDepositInterface(); - const inputs: Json = depositContract.decodeFunctionData("deposit", bytes); + const inputs = depositContract.decodeFunctionData("deposit", bytes); const {deposit_data_root: root} = inputs; const depositData: phase0.DepositData = ssz.phase0.DepositData.fromJson( // attach `amount` to decoded deposit inputs so it can be parsed to a DepositData - {...inputs, amount}, - {case: "snake"} + {...inputs, amount} ); // Sanity check diff --git a/packages/cli/src/networks/index.ts b/packages/cli/src/networks/index.ts index 17f0771b6bd6..0a7b321bab15 100644 --- a/packages/cli/src/networks/index.ts +++ b/packages/cli/src/networks/index.ts @@ -2,12 +2,11 @@ import {SLOTS_PER_EPOCH, ForkName} from "@chainsafe/lodestar-params"; import {getClient} from "@chainsafe/lodestar-api"; import {IBeaconNodeOptions} from "@chainsafe/lodestar"; import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {RecursivePartial, fromHex} from "@chainsafe/lodestar-utils"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; // eslint-disable-next-line no-restricted-imports import {getStateTypeFromBytes} from "@chainsafe/lodestar/lib/util/multifork"; -import {TreeBacked} from "@chainsafe/ssz"; import fs from "node:fs"; import got from "got"; import * as mainnet from "./mainnet"; @@ -145,7 +144,7 @@ export function enrsToNetworkConfig(enrs: string[]): RecursivePartial; wsCheckpoint: Checkpoint}> { +): Promise<{wsState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { try { let wsCheckpoint; const api = getClient(config, {baseUrl: weakSubjectivityServerUrl}); @@ -162,7 +161,7 @@ export async function fetchWeakSubjectivityState( ? api.debug.getState(`${stateSlot}`, "ssz") : api.debug.getStateV2(`${stateSlot}`, "ssz")); - return {wsState: getStateTypeFromBytes(config, stateBytes).createTreeBackedFromBytes(stateBytes), wsCheckpoint}; + return {wsState: getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes), wsCheckpoint}; } catch (e) { throw new Error("Unable to fetch weak subjectivity state: " + (e as Error).message); } diff --git a/packages/cli/src/util/file.ts b/packages/cli/src/util/file.ts index 75b5840a1285..7245581747a1 100644 --- a/packages/cli/src/util/file.ts +++ b/packages/cli/src/util/file.ts @@ -4,7 +4,6 @@ import stream from "node:stream"; import {promisify} from "node:util"; import got from "got"; import {load, dump, FAILSAFE_SCHEMA, Schema, Type} from "js-yaml"; -import {Json} from "@chainsafe/ssz"; export const yamlSchema = new Schema({ include: [FAILSAFE_SCHEMA], @@ -36,7 +35,7 @@ export enum FileFormat { /** * Parse file contents as Json. */ -export function parse(contents: string, fileFormat: FileFormat): T { +export function parse(contents: string, fileFormat: FileFormat): T { switch (fileFormat) { case FileFormat.json: return JSON.parse(contents) as T; @@ -51,7 +50,7 @@ export function parse(contents: string, fileFormat: FileFormat): T { /** * Stringify file contents. */ -export function stringify(obj: T, fileFormat: FileFormat): string { +export function stringify(obj: unknown, fileFormat: FileFormat): string { let contents: string; switch (fileFormat) { case FileFormat.json: @@ -72,7 +71,7 @@ export function stringify(obj: T, fileFormat: FileFormat): string { * * Serialize either to json, yaml, or toml */ -export function writeFile(filepath: string, obj: Json): void { +export function writeFile(filepath: string, obj: unknown): void { mkdir(path.dirname(filepath)); const fileFormat = path.extname(filepath).substr(1); fs.writeFileSync(filepath, stringify(obj, fileFormat as FileFormat), "utf-8"); @@ -84,7 +83,7 @@ export function writeFile(filepath: string, obj: Json): void { * Parse either from json, yaml, or toml * Optional acceptedFormats object can be passed which can be an array of accepted formats, in future can be extended to include parseFn for the accepted formats */ -export function readFile(filepath: string, acceptedFormats?: Json[]): T { +export function readFile(filepath: string, acceptedFormats?: string[]): T { const fileFormat = path.extname(filepath).substr(1); if (acceptedFormats && !acceptedFormats.includes(fileFormat)) throw new Error(`UnsupportedFileFormat: ${filepath}`); const contents = fs.readFileSync(filepath, "utf-8"); @@ -95,7 +94,7 @@ export function readFile(filepath: string, acceptedFormats?: Json[]): * @see readFile * If `filepath` does not exist returns null */ -export function readFileIfExists(filepath: string, acceptedFormats?: Json[]): T | null { +export function readFileIfExists(filepath: string, acceptedFormats?: string[]): T | null { try { return readFile(filepath, acceptedFormats); } catch (e) { diff --git a/packages/config/package.json b/packages/config/package.json index 9aa6717a8c18..eeb25c3d58c0 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -44,6 +44,6 @@ "dependencies": { "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" } } diff --git a/packages/config/src/chainConfig/sszTypes.ts b/packages/config/src/chainConfig/sszTypes.ts deleted file mode 100644 index d8e8a851e186..000000000000 --- a/packages/config/src/chainConfig/sszTypes.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; -import {ssz, StringType} from "@chainsafe/lodestar-types"; -import {IChainConfig} from "./types"; - -const ByteVector20 = new ByteVectorType({ - length: 20, -}); - -export const ChainConfig = new ContainerType({ - fields: { - PRESET_BASE: new StringType(), - - // Transition - TERMINAL_TOTAL_DIFFICULTY: ssz.Uint256, - TERMINAL_BLOCK_HASH: ssz.Root, - TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: ssz.Number64, - - // Genesis - MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: ssz.Number64, - MIN_GENESIS_TIME: ssz.Number64, - GENESIS_FORK_VERSION: ssz.Version, - GENESIS_DELAY: ssz.Number64, - - // Forking - // Altair - ALTAIR_FORK_VERSION: ssz.Version, - ALTAIR_FORK_EPOCH: ssz.Number64, - // Bellatrix - BELLATRIX_FORK_VERSION: ssz.Version, - BELLATRIX_FORK_EPOCH: ssz.Number64, - // Sharding - SHARDING_FORK_VERSION: ssz.Version, - SHARDING_FORK_EPOCH: ssz.Number64, - - // Time parameters - SECONDS_PER_SLOT: ssz.Number64, - SECONDS_PER_ETH1_BLOCK: ssz.Number64, - MIN_VALIDATOR_WITHDRAWABILITY_DELAY: ssz.Number64, - SHARD_COMMITTEE_PERIOD: ssz.Number64, - ETH1_FOLLOW_DISTANCE: ssz.Number64, - - // Validator cycle - INACTIVITY_SCORE_BIAS: ssz.Number64, - INACTIVITY_SCORE_RECOVERY_RATE: ssz.Number64, - EJECTION_BALANCE: ssz.Number64, - MIN_PER_EPOCH_CHURN_LIMIT: ssz.Number64, - CHURN_LIMIT_QUOTIENT: ssz.Number64, - PROPOSER_SCORE_BOOST: ssz.Number64, - - // Deposit contract - DEPOSIT_CHAIN_ID: ssz.Number64, - DEPOSIT_NETWORK_ID: ssz.Number64, - DEPOSIT_CONTRACT_ADDRESS: ByteVector20, - }, - // Expected and container fields are the same here - expectedCase: "notransform", -}); diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index 3b5b6258a458..28bfcc77a00b 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -43,7 +43,7 @@ export function createIForkConfig(config: IChainConfig): IForkConfig { return this.getForkInfo(slot).version; }, getForkTypes(slot: Slot): allForks.AllForksSSZTypes { - return ssz.allForks[this.getForkName(slot)]; + return ssz.allForks[this.getForkName(slot)] as allForks.AllForksSSZTypes; }, }; } diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index bec34c1b5dab..9b53fc631efb 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,6 +1,6 @@ import {ForkName} from "@chainsafe/lodestar-params"; import {DomainType, ForkDigest, phase0, Root, Slot, ssz, Version} from "@chainsafe/lodestar-types"; -import {ByteVector, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {IChainForkConfig} from "../beaconConfig"; import {ForkDigestHex, ICachedGenesis} from "./types"; export {IForkDigestContext} from "./types"; @@ -89,7 +89,7 @@ function computeForkDataRoot(currentVersion: Version, genesisValidatorsRoot: Roo return ssz.phase0.ForkData.hashTreeRoot(forkData); } -function toHexStringNoPrefix(hex: string | ByteVector): string { +function toHexStringNoPrefix(hex: string | Uint8Array): string { return strip0xPrefix(typeof hex === "string" ? hex : toHexString(hex)); } diff --git a/packages/db/package.json b/packages/db/package.json index a2a7fd936106..058087cdcac3 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -38,7 +38,7 @@ "dependencies": { "@chainsafe/lodestar-config": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "@types/levelup": "^4.3.3", "it-all": "^1.0.2", "level": "^7.0.0", diff --git a/packages/db/src/abstractRepository.ts b/packages/db/src/abstractRepository.ts index 956130795ace..ea2cf58f0d8a 100644 --- a/packages/db/src/abstractRepository.ts +++ b/packages/db/src/abstractRepository.ts @@ -1,5 +1,5 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {ArrayLike, Type} from "@chainsafe/ssz"; +import {Type} from "@chainsafe/ssz"; import {BUCKET_LENGTH} from "."; import {IFilterOptions, IKeyValue} from "./controller"; import {Db} from "./controller/interface"; @@ -11,8 +11,8 @@ export type Id = Uint8Array | string | number | bigint; /** * Repository is a high level kv storage - * managing a Uint8rray to Uint8rray kv database - * It translates typed keys and values to Uint8rrays required by the underlying database + * managing a Uint8Array to Uint8Array kv database + * It translates typed keys and values to Uint8Arrays required by the underlying database * * By default, SSZ-encoded values, * indexed by root @@ -100,7 +100,7 @@ export abstract class Repository { await this.delete(this.getId(value)); } - async batchPut(items: ArrayLike>): Promise { + async batchPut(items: IKeyValue[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchPut( Array.from({length: items.length}, (_, i) => ({ @@ -111,7 +111,7 @@ export abstract class Repository { } // Similar to batchPut but we support value as Uint8Array - async batchPutBinary(items: ArrayLike>): Promise { + async batchPutBinary(items: IKeyValue[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchPut( Array.from({length: items.length}, (_, i) => ({ @@ -121,12 +121,12 @@ export abstract class Repository { ); } - async batchDelete(ids: ArrayLike): Promise { + async batchDelete(ids: I[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchDelete(Array.from({length: ids.length}, (_, i) => this.encodeKey(ids[i]))); } - async batchAdd(values: ArrayLike): Promise { + async batchAdd(values: T[]): Promise { await this.batchPut( Array.from({length: values.length}, (_, i) => ({ key: this.getId(values[i]), @@ -135,7 +135,7 @@ export abstract class Repository { ); } - async batchRemove(values: ArrayLike): Promise { + async batchRemove(values: T[]): Promise { await this.batchDelete(Array.from({length: values.length}, (ignored, i) => this.getId(values[i]))); } diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 51752e88cc3b..638d336420e4 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -42,7 +42,7 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" }, "keywords": [ "ethereum", diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 108bbead3da1..882073c99021 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,4 +1,4 @@ -import {isTreeBacked, readonlyValues, toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SAFE_SLOTS_TO_UPDATE_JUSTIFIED, SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {Slot, ValidatorIndex, phase0, allForks, ssz, RootHex, Epoch, Root} from "@chainsafe/lodestar-types"; import { @@ -9,6 +9,8 @@ import { ZERO_HASH, bellatrix, EffectiveBalanceIncrements, + BeaconStateBellatrix, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; @@ -270,7 +272,7 @@ export class ForkChoice implements IForkChoice { * `justifiedBalances` balances of justified state which is updated synchronously. * This ensures that the forkchoice is never out of sync. */ - onBlock(block: allForks.BeaconBlock, state: allForks.BeaconState, preCachedData?: OnBlockPrecachedData): void { + onBlock(block: allForks.BeaconBlock, state: BeaconStateAllForks, preCachedData?: OnBlockPrecachedData): void { const {parentRoot, slot} = block; const parentRootHex = toHexString(parentRoot); // Parent block must be known @@ -329,9 +331,9 @@ export class ForkChoice implements IForkChoice { if ( preCachedData?.isMergeTransitionBlock || - (bellatrix.isBellatrixStateType(state) && + (bellatrix.isBellatrixStateType(state as BeaconStateBellatrix) && bellatrix.isBellatrixBlockBodyType(block.body) && - bellatrix.isMergeTransitionBlock(state, block.body)) + bellatrix.isMergeTransitionBlock(state as BeaconStateBellatrix, block.body)) ) assertValidTerminalPowBlock(this.config, (block as unknown) as bellatrix.BeaconBlock, preCachedData); @@ -388,7 +390,7 @@ export class ForkChoice implements IForkChoice { throw Error("Missing blockDelaySec info for proposerBoost"); } - const proposerInterval = getCurrentInterval(this.config, state.genesisTime, blockDelaySec); + const proposerInterval = getCurrentInterval(this.config, blockDelaySec); if (proposerInterval < 1) { this.proposerBoostRoot = blockRootHex; this.synced = false; @@ -396,7 +398,7 @@ export class ForkChoice implements IForkChoice { } const targetSlot = computeStartSlotAtEpoch(computeEpochAtSlot(slot)); - const targetRoot = slot === targetSlot ? blockRoot : state.blockRoots[targetSlot % SLOTS_PER_HISTORICAL_ROOT]; + const targetRoot = slot === targetSlot ? blockRoot : state.blockRoots.get(targetSlot % SLOTS_PER_HISTORICAL_ROOT); // This does not apply a vote to the block, it just makes fork choice aware of the block so // it can still be identified as the head even if it doesn't have any votes. @@ -413,8 +415,8 @@ export class ForkChoice implements IForkChoice { finalizedRoot: toHexString(state.finalizedCheckpoint.root), ...(bellatrix.isBellatrixBlockBodyType(block.body) && - bellatrix.isBellatrixStateType(state) && - bellatrix.isExecutionEnabled(state, block.body) + bellatrix.isBellatrixStateType(state as BeaconStateBellatrix) && + bellatrix.isExecutionEnabled(state as BeaconStateBellatrix, block.body) ? { executionPayloadBlockHash: toHexString(block.body.executionPayload.blockHash), executionStatus: this.getPostMergeExecStatus(preCachedData), @@ -466,7 +468,7 @@ export class ForkChoice implements IForkChoice { this.validateOnAttestation(attestation, slot, blockRootHex, targetEpoch); if (slot < this.fcStore.currentSlot) { - for (const validatorIndex of readonlyValues(attestation.attestingIndices)) { + for (const validatorIndex of attestation.attestingIndices) { this.addLatestMessage(validatorIndex, targetEpoch, blockRootHex); } } else { @@ -478,7 +480,7 @@ export class ForkChoice implements IForkChoice { // ``` this.queuedAttestations.add({ slot: slot, - attestingIndices: Array.from(readonlyValues(attestation.attestingIndices)), + attestingIndices: attestation.attestingIndices, blockRoot: blockRootHex, targetEpoch, }); @@ -729,7 +731,7 @@ export class ForkChoice implements IForkChoice { * * https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/fork-choice.md#should_update_justified_checkpoint */ - private shouldUpdateJustifiedCheckpoint(state: allForks.BeaconState): boolean { + private shouldUpdateJustifiedCheckpoint(state: BeaconStateAllForks): boolean { const {slot, currentJustifiedCheckpoint} = state; const newJustifiedCheckpoint = currentJustifiedCheckpoint; @@ -799,24 +801,20 @@ export class ForkChoice implements IForkChoice { } const attestationData = indexedAttestation.data; - // Only cache attestation data root hex if it's tree backed since it's available. - if ( - isTreeBacked(attestationData) && - this.validatedAttestationDatas.has( - toHexString(((attestationData as unknown) as TreeBacked).tree.root) - ) - ) { - return; - } + // AttestationData is expected to internally cache its root to make this hashTreeRoot() call free + const attestationCacheKey = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestationData)); - this.validateAttestationData(indexedAttestation.data, slot, blockRootHex, targetEpoch); + if (!this.validatedAttestationDatas.has(attestationCacheKey)) { + this.validateAttestationData(indexedAttestation.data, slot, blockRootHex, targetEpoch, attestationCacheKey); + } } private validateAttestationData( attestationData: phase0.AttestationData, slot: Slot, beaconBlockRootHex: string, - targetEpoch: Epoch + targetEpoch: Epoch, + attestationCacheKey: string ): void { const epochNow = computeEpochAtSlot(this.fcStore.currentSlot); const targetRootHex = toHexString(attestationData.target.root); @@ -916,12 +914,7 @@ export class ForkChoice implements IForkChoice { }); } - // Only cache attestation data root hex if it's tree backed since it's available. - if (isTreeBacked(attestationData)) { - this.validatedAttestationDatas.add( - toHexString(((attestationData as unknown) as TreeBacked).tree.root) - ); - } + this.validatedAttestationDatas.add(attestationCacheKey); } /** diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 126dab0162ea..4b5dc6810290 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -1,4 +1,5 @@ import {EffectiveBalanceIncrements} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {Epoch, Slot, ValidatorIndex, phase0, allForks, Root, RootHex} from "@chainsafe/lodestar-types"; import {IProtoBlock, ExecutionStatus} from "../protoArray/interface"; import {CheckpointWithHex} from "./store"; @@ -57,7 +58,7 @@ export interface IForkChoice { * `preCachedData` includes data necessary for validation included in the spec but some data is * pre-fetched in advance to keep the fork-choice fully syncronous */ - onBlock(block: allForks.BeaconBlock, state: allForks.BeaconState, preCachedData?: OnBlockPrecachedData): void; + onBlock(block: allForks.BeaconBlock, state: BeaconStateAllForks, preCachedData?: OnBlockPrecachedData): void; /** * Register `attestation` with the fork choice DAG so that it may influence future calls to `getHead`. * diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index 9759d0f01f37..c4cd2b73c200 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -75,7 +75,7 @@ export class ForkChoiceStore implements IForkChoiceStore { export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWithHex { // `valueOf` coerses the checkpoint, which may be tree-backed, into a javascript object // See https://github.com/ChainSafe/lodestar/issues/2258 - const root = checkpoint.root.valueOf() as Uint8Array; + const root = checkpoint.root; return { epoch: checkpoint.epoch, root, diff --git a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts index fcc6e6c15842..9d5370f0d0c5 100644 --- a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts @@ -122,12 +122,14 @@ describe("ForkChoice", () => { const aggregatedAttestations: IndexedAttestation[] = []; const averageAggregatorsPerSlot = 11; for (let committeeIndex = 0; committeeIndex < ATTESTATION_SUBNET_COUNT; committeeIndex++) { - const tbAttestationData = ssz.phase0.AttestationData.createTreeBackedFromStruct({ + const tbAttestationData = { ...attestationDataOmitIndex, index: committeeIndex, - }); + }; + // cache the root - tbAttestationData.hashTreeRoot(); + ssz.phase0.AttestationData.hashTreeRoot(tbAttestationData); + for (let aggregator = 0; aggregator < averageAggregatorsPerSlot; aggregator++) { // same data, different signatures aggregatedAttestations.push({ diff --git a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts index fe61839037ea..4c6747ec16c3 100644 --- a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts +++ b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts @@ -1,7 +1,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {expect} from "chai"; import { - CachedBeaconStateAllForks, + CachedBeaconStateAltair, computeStartSlotAtEpoch, EffectiveBalanceIncrements, getEffectiveBalanceIncrementsZeroed, @@ -19,7 +19,7 @@ function flagIsTimelySource(flag: number): boolean { } describe("computeDeltas", () => { - let originalState: CachedBeaconStateAllForks; + let originalState: CachedBeaconStateAltair; const indices: Map = new Map(); let oldBalances: EffectiveBalanceIncrements; let newBalances: EffectiveBalanceIncrements; @@ -32,13 +32,13 @@ describe("computeDeltas", () => { originalState = (generatePerfTestCachedStateAltair({ goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAllForks; - const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; - const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; + }) as unknown) as CachedBeaconStateAltair; + + const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); + const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); + + const numPreviousEpochParticipation = previousEpochParticipationArr.filter(flagIsTimelySource).length; + const numCurrentEpochParticipation = currentEpochParticipationArr.filter(flagIsTimelySource).length; expect(numPreviousEpochParticipation).to.equal(250000, "Wrong numPreviousEpochParticipation"); expect(numCurrentEpochParticipation).to.equal(250000, "Wrong numCurrentEpochParticipation"); @@ -66,8 +66,8 @@ describe("computeDeltas", () => { id: "computeDeltas", beforeEach: () => { const votes: IVoteTracker[] = []; - const epoch = originalState.currentShuffling.epoch; - const committee = originalState.getBeaconCommittee(computeStartSlotAtEpoch(epoch), 0); + const epoch = originalState.epochCtx.currentShuffling.epoch; + const committee = originalState.epochCtx.getBeaconCommittee(computeStartSlotAtEpoch(epoch), 0); for (let i = 0; i < 250000; i++) { if (committee.includes(i)) { votes.push({ diff --git a/packages/light-client/package.json b/packages/light-client/package.json index f02046d56441..8a2ee00a3036 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -42,8 +42,8 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "cross-fetch": "^3.1.4", "mitt": "^3.0.0" }, diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index 36901f9dfccf..bd2c1c721eb3 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -6,7 +6,7 @@ import {altair, phase0, RootHex, ssz, SyncPeriod} from "@chainsafe/lodestar-type import {createIBeaconConfig, IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import {TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; import {isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; -import {fromHexString, Path, toHexString} from "@chainsafe/ssz"; +import {fromHexString, JsonPath, toHexString} from "@chainsafe/ssz"; import {getCurrentSlot, slotWithFutureTolerance, timeUntilNextEpoch} from "./utils/clock"; import {isBetterUpdate, LightclientUpdateStats} from "./utils/update"; import {deserializeSyncCommittee, isEmptyHeader, sumBits} from "./utils/utils"; @@ -229,7 +229,7 @@ export class Lightclient { } /** Returns header since head may change during request */ - async getHeadStateProof(paths: Path[]): Promise<{proof: TreeOffsetProof; header: phase0.BeaconBlockHeader}> { + async getHeadStateProof(paths: JsonPath[]): Promise<{proof: TreeOffsetProof; header: phase0.BeaconBlockHeader}> { const header = this.head.header; const stateId = toHexString(header.stateRoot); const res = await this.api.lightclient.getStateProof(stateId, paths); diff --git a/packages/light-client/src/utils/domain.ts b/packages/light-client/src/utils/domain.ts index 0d7ea0fd757f..2450637691ba 100644 --- a/packages/light-client/src/utils/domain.ts +++ b/packages/light-client/src/utils/domain.ts @@ -1,6 +1,6 @@ // Only used by processDeposit + lightclient -import {Epoch, Version, Root, DomainType, allForks, phase0, ssz, Domain} from "@chainsafe/lodestar-types"; +import {Epoch, Version, Root, DomainType, phase0, ssz, Domain} from "@chainsafe/lodestar-types"; import {Type} from "@chainsafe/ssz"; /** @@ -17,7 +17,7 @@ export function computeDomain(domainType: DomainType, forkVersion: Version, gene /** * Return the ForkVersion at an epoch from a Fork type */ -export function getForkVersion(fork: allForks.BeaconState["fork"], epoch: Epoch): Version { +export function getForkVersion(fork: phase0.Fork, epoch: Epoch): Version { return epoch < fork.epoch ? fork.previousVersion : fork.currentVersion; } diff --git a/packages/light-client/src/utils/utils.ts b/packages/light-client/src/utils/utils.ts index f74172409b32..f3ef85f520ab 100644 --- a/packages/light-client/src/utils/utils.ts +++ b/packages/light-client/src/utils/utils.ts @@ -1,17 +1,17 @@ import {PublicKey} from "@chainsafe/bls"; import {altair, Root, ssz} from "@chainsafe/lodestar-types"; import {BeaconBlockHeader} from "@chainsafe/lodestar-types/phase0"; -import {ArrayLike, BitVector} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {SyncCommitteeFast} from "../types"; -export function sumBits(bits: ArrayLike): number { - let sum = 0; - for (const bit of bits) { - if (bit) { - sum++; - } +export function sumBits(bits: BitArray): number { + // TODO: Optimize + const indexes: number[] = []; + for (let i = 0; i < bits.bitLen; i++) { + indexes.push(0); } - return sum; + + return bits.intersectValues(indexes).length; } export function isZeroHash(root: Root): boolean { @@ -23,7 +23,7 @@ export function isZeroHash(root: Root): boolean { return true; } -export function assertZeroHashes(rootArray: ArrayLike, expectedLength: number, errorMessage: string): void { +export function assertZeroHashes(rootArray: Root[], expectedLength: number, errorMessage: string): void { if (rootArray.length !== expectedLength) { throw Error(`Wrong length ${errorMessage}`); } @@ -38,16 +38,12 @@ export function assertZeroHashes(rootArray: ArrayLike, expectedLength: num /** * Util to guarantee that all bits have a corresponding pubkey */ -export function getParticipantPubkeys(pubkeys: ArrayLike, bits: BitVector): T[] { - const participantPubkeys: T[] = []; - for (let i = 0; i < bits.length; i++) { - if (bits[i]) { - if (pubkeys[i] === undefined) throw Error(`No pubkey ${i} in syncCommittee`); - participantPubkeys.push(pubkeys[i]); - } +export function getParticipantPubkeys(pubkeys: T[], bits: BitArray): T[] { + if (bits.bitLen > pubkeys.length) { + throw Error(`syncCommittee bitLen ${bits.bitLen} > pubkeys.length ${pubkeys.length}`); } - return participantPubkeys; + return bits.intersectValues(pubkeys); } export function toBlockHeader(block: altair.BeaconBlock): BeaconBlockHeader { @@ -61,7 +57,7 @@ export function toBlockHeader(block: altair.BeaconBlock): BeaconBlockHeader { } function deserializePubkeys(pubkeys: altair.LightClientUpdate["nextSyncCommittee"]["pubkeys"]): PublicKey[] { - return Array.from(pubkeys).map((pk) => PublicKey.fromBytes(pk.valueOf() as Uint8Array)); + return Array.from(pubkeys).map((pk) => PublicKey.fromBytes(pk)); } function serializePubkeys(pubkeys: PublicKey[]): altair.LightClientUpdate["nextSyncCommittee"]["pubkeys"] { @@ -71,7 +67,7 @@ function serializePubkeys(pubkeys: PublicKey[]): altair.LightClientUpdate["nextS export function deserializeSyncCommittee(syncCommittee: altair.SyncCommittee): SyncCommitteeFast { return { pubkeys: deserializePubkeys(syncCommittee.pubkeys), - aggregatePubkey: PublicKey.fromBytes(syncCommittee.aggregatePubkey.valueOf() as Uint8Array), + aggregatePubkey: PublicKey.fromBytes(syncCommittee.aggregatePubkey), }; } @@ -83,6 +79,6 @@ export function serializeSyncCommittee(syncCommittee: SyncCommitteeFast): altair } export function isEmptyHeader(header: BeaconBlockHeader): boolean { - const emptyValue = ssz.phase0.BeaconBlockHeader.defaultValue(); + const emptyValue = ssz.phase0.BeaconBlockHeader.defaultValue; return ssz.phase0.BeaconBlockHeader.equals(emptyValue, header); } diff --git a/packages/light-client/src/validation.ts b/packages/light-client/src/validation.ts index 813e5e3fee2a..35bdf1405f2d 100644 --- a/packages/light-client/src/validation.ts +++ b/packages/light-client/src/validation.ts @@ -66,10 +66,10 @@ export function assertValidFinalityProof(update: altair.LightClientUpdate): void if ( !isValidMerkleBranch( ssz.phase0.BeaconBlockHeader.hashTreeRoot(update.finalizedHeader), - Array.from(update.finalityBranch).map((i) => i.valueOf() as Uint8Array), + update.finalityBranch, FINALIZED_ROOT_DEPTH, FINALIZED_ROOT_INDEX, - update.attestedHeader.stateRoot.valueOf() as Uint8Array + update.attestedHeader.stateRoot ) ) { throw Error("Invalid finality header merkle branch"); @@ -96,10 +96,10 @@ export function assertValidSyncCommitteeProof(update: altair.LightClientUpdate): if ( !isValidMerkleBranch( ssz.altair.SyncCommittee.hashTreeRoot(update.nextSyncCommittee), - Array.from(update.nextSyncCommitteeBranch).map((i) => i.valueOf() as Uint8Array), + update.nextSyncCommitteeBranch, NEXT_SYNC_COMMITTEE_DEPTH, NEXT_SYNC_COMMITTEE_INDEX, - activeHeader(update).stateRoot.valueOf() as Uint8Array + activeHeader(update).stateRoot ) ) { throw Error("Invalid next sync committee merkle branch"); @@ -155,9 +155,7 @@ export function assertValidSignedHeader( domain: config.getDomain(DOMAIN_SYNC_COMMITTEE, signedHeaderSlot), }); - if ( - !isValidBlsAggregate(participantPubkeys, signingRoot, syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array) - ) { + if (!isValidBlsAggregate(participantPubkeys, signingRoot, syncAggregate.syncCommitteeSignature)) { throw Error("Invalid aggregate signature"); } } diff --git a/packages/light-client/test/getGenesisData.ts b/packages/light-client/test/getGenesisData.ts index 2679932e901b..7c812420174e 100644 --- a/packages/light-client/test/getGenesisData.ts +++ b/packages/light-client/test/getGenesisData.ts @@ -19,7 +19,7 @@ async function getGenesisData(): Promise { const api = getClient(config, {baseUrl}); const {data: genesis} = await api.beacon.getGenesis(); console.log(network, { - genesisTime: Number(genesis.genesisTime), + genesisTime: genesis.genesisTime, genesisValidatorsRoot: "0x" + Buffer.from(genesis.genesisValidatorsRoot as Uint8Array).toString("hex"), }); } diff --git a/packages/light-client/test/lightclientApiServer.ts b/packages/light-client/test/lightclientApiServer.ts index aa4cde975618..1696512c977e 100644 --- a/packages/light-client/test/lightclientApiServer.ts +++ b/packages/light-client/test/lightclientApiServer.ts @@ -4,14 +4,15 @@ import {registerRoutes} from "@chainsafe/lodestar-api/server"; import fastifyCors from "fastify-cors"; import querystring from "querystring"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Path, TreeBacked} from "@chainsafe/ssz"; -import {allForks, altair, RootHex, SyncPeriod} from "@chainsafe/lodestar-types"; +import {JsonPath} from "@chainsafe/ssz"; +import {altair, RootHex, SyncPeriod} from "@chainsafe/lodestar-types"; import {Proof} from "@chainsafe/persistent-merkle-tree"; +import {BeaconStateAltair} from "./types"; /* eslint-disable @typescript-eslint/no-unsafe-member-access */ export type IStateRegen = { - getStateByRoot(stateRoot: string): Promise>; + getStateByRoot(stateRoot: string): Promise; }; export type IBlockCache = { @@ -39,12 +40,12 @@ export async function startServer(opts: ServerOpts, config: IChainForkConfig, ap } export class LightclientServerApi implements routes.lightclient.Api { - readonly states = new Map>(); + readonly states = new Map(); readonly updates = new Map(); readonly snapshots = new Map(); latestHeadUpdate: routes.lightclient.LightclientHeaderUpdate | null = null; - async getStateProof(stateId: string, paths: Path[]): Promise<{data: Proof}> { + async getStateProof(stateId: string, paths: JsonPath[]): Promise<{data: Proof}> { const state = this.states.get(stateId); if (!state) throw Error(`stateId ${stateId} not available`); return {data: state.createProof(paths)}; diff --git a/packages/light-client/test/naive/update.ts b/packages/light-client/test/naive/update.ts index a13d52981588..45e26690dc98 100644 --- a/packages/light-client/test/naive/update.ts +++ b/packages/light-client/test/naive/update.ts @@ -71,7 +71,7 @@ export function processLightClientUpdate( // It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. if ( sumBits(update.syncCommitteeAggregate.syncCommitteeBits) * 3 >= - update.syncCommitteeAggregate.syncCommitteeBits.length * 2 && + update.syncCommitteeAggregate.syncCommitteeBits.bitLen * 2 && !isEmptyHeader(update.finalizedHeader) ) { applyLightClientUpdate(store.snapshot, update); diff --git a/packages/light-client/test/prepareUpdateNaive.ts b/packages/light-client/test/prepareUpdateNaive.ts index 64f4a7f76a1f..d0724ca434a1 100644 --- a/packages/light-client/test/prepareUpdateNaive.ts +++ b/packages/light-client/test/prepareUpdateNaive.ts @@ -1,16 +1,17 @@ -import {altair, phase0, Root} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {altair, phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {CompositeViewDU} from "@chainsafe/ssz"; import {FINALIZED_ROOT_GINDEX, NEXT_SYNC_COMMITTEE_GINDEX, SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; import {computeEpochAtSlot} from "../src/utils/clock"; import {getForkVersion} from "../src/utils/domain"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; export interface IBeaconChainLc { getBlockHeaderByRoot(blockRoot: Root): Promise; - getStateByRoot(stateRoot: Root): Promise>; + getStateByRoot(stateRoot: Root): Promise>; } /** - * From a TreeBacked state, return an update to be consumed by a light client + * From a TreeView state, return an update to be consumed by a light client * Spec v1.0.1 */ export async function prepareUpdateNaive( @@ -83,7 +84,7 @@ export async function prepareUpdateNaive( // Get the finality block root that sync committees have signed in blockA const syncAttestedSlot = stateWithSyncAggregate.slot - 1; // Inlined `getBlockRootAtSlot()` - const syncAttestedBlockRoot = stateWithSyncAggregate.blockRoots[syncAttestedSlot % SLOTS_PER_HISTORICAL_ROOT]; + const syncAttestedBlockRoot = stateWithSyncAggregate.blockRoots.get(syncAttestedSlot % SLOTS_PER_HISTORICAL_ROOT); const syncAttestedBlockHeader = await chain.getBlockHeaderByRoot(syncAttestedBlockRoot); // Get the ForkVersion used in the syncAggregate, as verified in the state transition fn @@ -93,17 +94,22 @@ export async function prepareUpdateNaive( // Get the finalized state defined in the block "attested" by the current sync committee const syncAttestedState = await chain.getStateByRoot(syncAttestedBlockHeader.stateRoot); const finalizedCheckpointBlockHeader = await chain.getBlockHeaderByRoot(syncAttestedState.finalizedCheckpoint.root); + // Prove that the `finalizedCheckpointRoot` belongs in that block - const finalityBranch = syncAttestedState.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); + syncAttestedState.commit(); + const syncAttestedStateTree = new Tree(syncAttestedState.node); + const finalityBranch = syncAttestedStateTree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); // Get `nextSyncCommittee` from a finalized state so the lightclient can safely transition to the next committee const finalizedCheckpointState = await chain.getStateByRoot(finalizedCheckpointBlockHeader.stateRoot); // Prove that the `nextSyncCommittee` is included in a finalized state "attested" by the current sync committee - const nextSyncCommitteeBranch = finalizedCheckpointState.tree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); + finalizedCheckpointState.commit(); + const finalizedCheckpointStateTree = new Tree(finalizedCheckpointState.node); + const nextSyncCommitteeBranch = finalizedCheckpointStateTree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); return { attestedHeader: syncAttestedBlockHeader, - nextSyncCommittee: finalizedCheckpointState.nextSyncCommittee, + nextSyncCommittee: finalizedCheckpointState.nextSyncCommittee.toValue(), nextSyncCommitteeBranch: nextSyncCommitteeBranch, finalizedHeader: finalizedCheckpointBlockHeader, finalityBranch: finalityBranch, diff --git a/packages/light-client/test/types.ts b/packages/light-client/test/types.ts new file mode 100644 index 000000000000..3f10cd32b1c4 --- /dev/null +++ b/packages/light-client/test/types.ts @@ -0,0 +1,4 @@ +import {CompositeViewDU} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; + +export type BeaconStateAltair = CompositeViewDU; diff --git a/packages/light-client/test/unit/sync.test.ts b/packages/light-client/test/unit/sync.test.ts index 88dca4e4671d..5ef93ed40b20 100644 --- a/packages/light-client/test/unit/sync.test.ts +++ b/packages/light-client/test/unit/sync.test.ts @@ -1,5 +1,6 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks, BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {routes, Api} from "@chainsafe/lodestar-api"; import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config"; @@ -13,7 +14,7 @@ import { committeeUpdateToHeadUpdate, lastInMap, } from "../utils"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {expect} from "chai"; const SOME_HASH = Buffer.alloc(32, 0xff); @@ -94,7 +95,7 @@ describe("Lightclient sync", () => { // Test fetching a proof // First create a state with some known data const executionStateRoot = Buffer.alloc(32, 0xee); - const state = ssz.bellatrix.BeaconState.defaultTreeBacked(); + const state = ssz.bellatrix.BeaconState.defaultViewDU; state.latestExecutionPayloadHeader.stateRoot = executionStateRoot; // Track head + reference states with some known data @@ -113,7 +114,7 @@ describe("Lightclient sync", () => { // Provide the state to the lightclient server impl. Only the last one to test proof fetching if (slot === targetSlot) { - lightclientServerApi.states.set(toHexString(stateRoot), state as TreeBacked); + lightclientServerApi.states.set(toHexString(stateRoot), (state as BeaconStateAllForks) as BeaconStateAltair); } // Emit a new head update with the custom state root @@ -140,7 +141,7 @@ describe("Lightclient sync", () => { // Fetch proof of "latestExecutionPayloadHeader.stateRoot" const {proof, header} = await lightclient.getHeadStateProof([["latestExecutionPayloadHeader", "stateRoot"]]); - const recoveredState = ssz.bellatrix.BeaconState.createTreeBackedFromProof(header.stateRoot as Uint8Array, proof); + const recoveredState = ssz.bellatrix.BeaconState.createFromProof(proof, header.stateRoot); expect(toHexString(recoveredState.latestExecutionPayloadHeader.stateRoot)).to.equal( toHexString(executionStateRoot), "Recovered executionStateRoot from getHeadStateProof() not correct" diff --git a/packages/light-client/test/unit/syncNaive.test.ts b/packages/light-client/test/unit/syncNaive.test.ts index a99f1a9f1170..3fc009610d32 100644 --- a/packages/light-client/test/unit/syncNaive.test.ts +++ b/packages/light-client/test/unit/syncNaive.test.ts @@ -1,7 +1,8 @@ import {expect} from "chai"; import {SecretKey} from "@chainsafe/bls"; import {altair, phase0, Root, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; +import {toHexString} from "@chainsafe/ssz"; import {chainConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {processLightClientUpdate} from "../naive/update"; @@ -42,30 +43,30 @@ describe("Lightclient flow", () => { const finalizedBlockSlot = SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD + 1; const headerBlockSlot = finalizedBlockSlot + 1; - const finalizedState = ssz.altair.BeaconState.defaultValue(); + const finalizedState = ssz.altair.BeaconState.defaultValue; finalizedState.nextSyncCommittee = getSyncCommittee(0).syncCommittee; - const finalizedBlockHeader = ssz.phase0.BeaconBlockHeader.defaultValue(); + const finalizedBlockHeader = ssz.phase0.BeaconBlockHeader.defaultValue; finalizedBlockHeader.slot = finalizedBlockSlot; finalizedBlockHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(finalizedState); // Create a state that has the finalizedState as finalized checkpoint - const syncAttestedState = ssz.altair.BeaconState.defaultValue(); + const syncAttestedState = ssz.altair.BeaconState.defaultValue; syncAttestedState.finalizedCheckpoint = { epoch: 0, // Checkpoint { epoch, blockRoot } root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(finalizedBlockHeader), }; - const syncAttestedBlockHeader = ssz.phase0.BeaconBlockHeader.defaultValue(); + const syncAttestedBlockHeader = ssz.phase0.BeaconBlockHeader.defaultValue; syncAttestedBlockHeader.slot = headerBlockSlot; syncAttestedBlockHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(syncAttestedState); // Create a state with the block blockWithSyncAggregate - const stateWithSyncAggregate = ssz.altair.BeaconState.defaultValue(); + const stateWithSyncAggregate = ssz.altair.BeaconState.defaultValue; stateWithSyncAggregate.slot = 1; stateWithSyncAggregate.blockRoots[0] = ssz.phase0.BeaconBlockHeader.hashTreeRoot(syncAttestedBlockHeader); // Create a signature from current committee to "attest" syncAttestedBlockHeader const signingRoot = getSyncAggregateSigningRoot(config, syncAttestedBlockHeader); - const blockWithSyncAggregate = ssz.altair.BeaconBlock.defaultValue(); + const blockWithSyncAggregate = ssz.altair.BeaconBlock.defaultValue; blockWithSyncAggregate.body.syncAggregate = getSyncCommittee(0).signAndAggregate(signingRoot); blockWithSyncAggregate.stateRoot = ssz.altair.BeaconState.hashTreeRoot(stateWithSyncAggregate); @@ -90,7 +91,7 @@ describe("Lightclient flow", () => { const store: LightClientStoreFast = { bestUpdates: new Map(), snapshot: { - header: ssz.phase0.BeaconBlockHeader.defaultValue(), + header: ssz.phase0.BeaconBlockHeader.defaultValue, currentSyncCommittee: getSyncCommittee(0).syncCommitteeFast, nextSyncCommittee: getSyncCommittee(0).syncCommitteeFast, }, @@ -121,10 +122,10 @@ class MockBeaconChainLc implements IBeaconChainLc { return blockHeader; } - async getStateByRoot(stateRoot: Root): Promise> { + async getStateByRoot(stateRoot: Root): Promise { const rootHex = toHexString(stateRoot); const state = this.states.get(rootHex); if (!state) throw Error(`No state for ${rootHex}`); - return ssz.altair.BeaconState.createTreeBackedFromStruct(state); + return ssz.altair.BeaconState.toViewDU(state); } } diff --git a/packages/light-client/test/unit/validation.test.ts b/packages/light-client/test/unit/validation.test.ts index d88c14c0f7ea..d3cfa7319a91 100644 --- a/packages/light-client/test/unit/validation.test.ts +++ b/packages/light-client/test/unit/validation.test.ts @@ -1,4 +1,5 @@ import {aggregatePublicKeys, PublicKey, SecretKey} from "@chainsafe/bls"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {altair, ssz} from "@chainsafe/lodestar-types"; import {chainConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; @@ -44,29 +45,31 @@ describe("validateLightClientUpdate", () => { }; // finalizedCheckpointState must have `nextSyncCommittee` - const finalizedCheckpointState = ssz.altair.BeaconState.defaultTreeBacked(); - finalizedCheckpointState.nextSyncCommittee = nextSyncCommittee; + const finalizedCheckpointState = ssz.altair.BeaconState.defaultViewDU; + finalizedCheckpointState.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(nextSyncCommittee); // Prove it - const nextSyncCommitteeBranch = finalizedCheckpointState.tree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); + const nextSyncCommitteeBranch = new Tree(finalizedCheckpointState.node).getSingleProof( + BigInt(NEXT_SYNC_COMMITTEE_GINDEX) + ); // update.header must have stateRoot to finalizedCheckpointState const finalizedHeader = defaultBeaconBlockHeader(updateHeaderSlot); - finalizedHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(finalizedCheckpointState); + finalizedHeader.stateRoot = finalizedCheckpointState.hashTreeRoot(); // syncAttestedState must have `header` as finalizedCheckpoint - const syncAttestedState = ssz.altair.BeaconState.defaultTreeBacked(); - syncAttestedState.finalizedCheckpoint = { + const syncAttestedState = ssz.altair.BeaconState.defaultViewDU; + syncAttestedState.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: 0, root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(finalizedHeader), - }; + }); // Prove it - const finalityBranch = syncAttestedState.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); + const finalityBranch = new Tree(syncAttestedState.node).getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); // finalityHeader must have stateRoot to syncAttestedState const syncAttestedBlockHeader = defaultBeaconBlockHeader(attestedHeaderSlot); - syncAttestedBlockHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(syncAttestedState); + syncAttestedBlockHeader.stateRoot = syncAttestedState.hashTreeRoot(); - const forkVersion = ssz.Bytes4.defaultValue(); + const forkVersion = ssz.Bytes4.defaultValue; const signingRoot = getSyncAggregateSigningRoot(config, syncAttestedBlockHeader); const syncAggregate = signAndAggregate(signingRoot, sks); diff --git a/packages/light-client/test/utils.ts b/packages/light-client/test/utils.ts index c28536ccb397..22cecdf1e561 100644 --- a/packages/light-client/test/utils.ts +++ b/packages/light-client/test/utils.ts @@ -13,7 +13,7 @@ import { } from "@chainsafe/lodestar-params"; import {altair, phase0, Slot, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; import {hash} from "@chainsafe/persistent-merkle-tree"; -import {fromHexString, List} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import {SyncCommitteeFast} from "../src/types"; import {computeSigningRoot} from "../src/utils/domain"; import {getLcLoggerConsole} from "../src/utils/logger"; @@ -38,7 +38,7 @@ export function signAndAggregate(message: Uint8Array, sks: SecretKey[]): altair. const sigs = sks.map((sk) => sk.sign(message)); const aggSig = Signature.aggregate(sigs).toBytes(); return { - syncCommitteeBits: sks.map(() => true), + syncCommitteeBits: BitArray.fromBoolArray(sks.map(() => true)), syncCommitteeSignature: aggSig, }; } @@ -52,7 +52,7 @@ export function getSyncAggregateSigningRoot( } export function defaultBeaconBlockHeader(slot: Slot): phase0.BeaconBlockHeader { - const header = ssz.phase0.BeaconBlockHeader.defaultValue(); + const header = ssz.phase0.BeaconBlockHeader.defaultValue; header.slot = slot; return header; } @@ -85,7 +85,7 @@ export function getInteropSyncCommittee(period: SyncPeriod): SyncCommitteeKeys { const sigs = Array.from({length: SYNC_COMMITTEE_SIZE}, () => sig); const aggSig = Signature.aggregate(sigs).toBytes(); return { - syncCommitteeBits: Array.from({length: SYNC_COMMITTEE_SIZE}, () => true), + syncCommitteeBits: BitArray.fromBoolArray(sigs.map(() => true)), syncCommitteeSignature: aggSig, }; } @@ -219,12 +219,12 @@ export function generateValidator(opts: Partial = {}): phase0. /** * Generates n number of validators, for tests purposes only. */ -export function generateValidators(n: number, opts?: Partial): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: Partial): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } -export function generateBalances(n: number): List { - return Array.from({length: n}, () => 32e9) as List; +export function generateBalances(n: number): number[] { + return Array.from({length: n}, () => 32e9); } /** diff --git a/packages/lodestar/package.json b/packages/lodestar/package.json index c49257daad2a..a6fa856e0096 100644 --- a/packages/lodestar/package.json +++ b/packages/lodestar/package.json @@ -74,9 +74,9 @@ "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/lodestar-validator": "^0.35.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", + "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", "@chainsafe/snappy-stream": "5.0.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "@ethersproject/abi": "^5.0.0", "@types/datastore-level": "^3.0.0", "bl": "^5.0.0", diff --git a/packages/lodestar/src/api/impl/beacon/blocks/index.ts b/packages/lodestar/src/api/impl/beacon/blocks/index.ts index fdd541ac9ad7..90ca8ae2a28a 100644 --- a/packages/lodestar/src/api/impl/beacon/blocks/index.ts +++ b/packages/lodestar/src/api/impl/beacon/blocks/index.ts @@ -125,7 +125,7 @@ export function getBeaconBlockApi({ if (slot < head.slot && head.slot <= slot + SLOTS_PER_HISTORICAL_ROOT) { const state = chain.getHeadState(); - return {data: state.blockRoots[slot % SLOTS_PER_HISTORICAL_ROOT]}; + return {data: state.blockRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT)}; } } else if (blockId === "head") { const head = chain.forkChoice.getHead(); diff --git a/packages/lodestar/src/api/impl/beacon/index.ts b/packages/lodestar/src/api/impl/beacon/index.ts index 445f6748bac1..d3fed6541a5a 100644 --- a/packages/lodestar/src/api/impl/beacon/index.ts +++ b/packages/lodestar/src/api/impl/beacon/index.ts @@ -24,7 +24,7 @@ export function getBeaconApi( return { data: { genesisForkVersion, - genesisTime: BigInt(chain.genesisTime), + genesisTime: chain.genesisTime, genesisValidatorsRoot: chain.genesisValidatorsRoot, }, }; diff --git a/packages/lodestar/src/api/impl/beacon/state/index.ts b/packages/lodestar/src/api/impl/beacon/state/index.ts index 33b27e1a92e0..db0ffbdd8f7b 100644 --- a/packages/lodestar/src/api/impl/beacon/state/index.ts +++ b/packages/lodestar/src/api/impl/beacon/state/index.ts @@ -1,9 +1,9 @@ import {routes} from "@chainsafe/lodestar-api"; // eslint-disable-next-line no-restricted-imports import {Api as IBeaconStateApi} from "@chainsafe/lodestar-api/lib/routes/beacon/state"; -import {allForks, altair} from "@chainsafe/lodestar-types"; -import {readonlyValues} from "@chainsafe/ssz"; import { + BeaconStateAllForks, + CachedBeaconStateAllForks, CachedBeaconStateAltair, computeEpochAtSlot, getCurrentEpoch, @@ -19,14 +19,14 @@ import { } from "./utils"; export function getBeaconStateApi({chain, config, db}: Pick): IBeaconStateApi { - async function getState(stateId: routes.beacon.StateId): Promise { + async function getState(stateId: routes.beacon.StateId): Promise { return await resolveStateId(config, chain, db, stateId); } return { async getStateRoot(stateId) { const state = await getState(stateId); - return {data: config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state)}; + return {data: state.hashTreeRoot()}; }, async getStateFork(stateId) { @@ -49,21 +49,21 @@ export function getBeaconStateApi({chain, config, db}: Pick { - return { - index, - balance, - }; - }); - return {data: balances}; + // TODO: This loops over the entire state, it's a DOS vector + const balancesArr = state.balances.getAll(); + const resp: routes.beacon.ValidatorBalance[] = []; + for (let i = 0; i < balancesArr.length; i++) { + resp.push({index: i, balance: balancesArr[i]}); + } + return {data: resp}; }, async getEpochCommittees(stateId, filters) { @@ -171,7 +173,7 @@ export function getBeaconStateApi({chain, config, db}: Pick { +): Promise { const state = await resolveStateIdOrNull(config, chain, db, stateId, opts); if (!state) { throw new ApiError(404, `No state found for id '${stateId}'`); @@ -48,7 +50,7 @@ async function resolveStateIdOrNull( db: IBeaconDb, stateId: routes.beacon.StateId, opts?: ResolveStateIdOpts -): Promise { +): Promise { stateId = stateId.toLowerCase(); if (stateId === "head" || stateId === "genesis" || stateId === "finalized" || stateId === "justified") { return await stateByName(db, chain.stateCache, chain.forkChoice, stateId); @@ -117,7 +119,7 @@ async function stateByName( stateCache: StateContextCache, forkChoice: IForkChoice, stateId: routes.beacon.StateId -): Promise { +): Promise { switch (stateId) { case "head": return stateCache.get(forkChoice.getHead().stateRoot) ?? null; @@ -136,7 +138,7 @@ async function stateByRoot( db: IBeaconDb, stateCache: StateContextCache, stateId: routes.beacon.StateId -): Promise { +): Promise { if (stateId.startsWith("0x")) { const stateRoot = stateId; const cachedStateCtx = stateCache.get(stateRoot); @@ -154,7 +156,7 @@ async function stateBySlot( forkChoice: IForkChoice, slot: Slot, opts?: ResolveStateIdOpts -): Promise { +): Promise { const blockSummary = forkChoice.getCanonicalBlockAtSlot(slot); if (blockSummary) { const state = stateCache.get(blockSummary.stateRoot); @@ -172,17 +174,20 @@ async function stateBySlot( export function filterStateValidatorsByStatuses( statuses: string[], - state: allForks.BeaconState, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap, currentEpoch: Epoch ): routes.beacon.ValidatorResponse[] { const responses: routes.beacon.ValidatorResponse[] = []; - const validators = Array.from(state.validators); - const filteredValidators = validators.filter((v) => statuses.includes(getValidatorStatus(v, currentEpoch))); - for (const validator of readonlyValues(filteredValidators)) { + const validatorsArr = state.validators.getAllReadonlyValues(); + const statusSet = new Set(statuses); + + for (const validator of validatorsArr) { + const validatorStatus = getValidatorStatus(validator, currentEpoch); + const validatorIndex = getStateValidatorIndex(validator.pubkey, state, pubkey2index); - if (validatorIndex !== undefined && statuses?.includes(getValidatorStatus(validator, currentEpoch))) { - responses.push(toValidatorResponse(validatorIndex, validator, state.balances[validatorIndex], currentEpoch)); + if (validatorIndex !== undefined && statusSet.has(validatorStatus)) { + responses.push(toValidatorResponse(validatorIndex, validator, state.balances.get(validatorIndex), currentEpoch)); } } return responses; @@ -197,8 +202,10 @@ async function getNearestArchivedState( slot: Slot ): Promise { const states = db.stateArchive.valuesStream({lte: slot, reverse: true}); - const state = (await states[Symbol.asyncIterator]().next()).value as TreeBacked; - return createCachedBeaconState(config, state); + const state = (await states[Symbol.asyncIterator]().next()).value as BeaconStateAllForks; + // TODO - PENDING: Don't create new immutable caches here + // see https://github.com/ChainSafe/lodestar/issues/3683 + return createCachedBeaconState(state, createEmptyEpochContextImmutableData(config, state)); } async function getFinalizedState( @@ -228,8 +235,8 @@ async function getFinalizedState( } export function getStateValidatorIndex( - id: routes.beacon.ValidatorId | ByteVector, - state: allForks.BeaconState, + id: routes.beacon.ValidatorId | BLSPubkey, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap ): number | undefined { let validatorIndex: ValidatorIndex | undefined; diff --git a/packages/lodestar/src/api/impl/debug/index.ts b/packages/lodestar/src/api/impl/debug/index.ts index a8f24b7f635b..4596f709a8dd 100644 --- a/packages/lodestar/src/api/impl/debug/index.ts +++ b/packages/lodestar/src/api/impl/debug/index.ts @@ -23,7 +23,7 @@ export function getDebugApi({ if (format === "ssz") { // Casting to any otherwise Typescript doesn't like the multi-type return // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any - return config.getForkTypes(state.slot).BeaconState.serialize(state) as any; + return state.serialize() as any; } else { return {data: state}; } @@ -34,7 +34,7 @@ export function getDebugApi({ if (format === "ssz") { // Casting to any otherwise Typescript doesn't like the multi-type return // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any - return config.getForkTypes(state.slot).BeaconState.serialize(state) as any; + return state.serialize() as any; } else { return {data: state, version: config.getForkName(state.slot)}; } diff --git a/packages/lodestar/src/api/impl/events/index.ts b/packages/lodestar/src/api/impl/events/index.ts index 2a58c98c0b48..69f4b0d5a5e1 100644 --- a/packages/lodestar/src/api/impl/events/index.ts +++ b/packages/lodestar/src/api/impl/events/index.ts @@ -37,7 +37,7 @@ export function getEventsApi({chain, config}: Pick toHexString(getBlockRootAtSlot(state, Math.max(computeStartSlotAtEpoch(epoch) - 1, 0))) ); diff --git a/packages/lodestar/src/api/impl/lightclient/index.ts b/packages/lodestar/src/api/impl/lightclient/index.ts index 6cb1192e8e23..392eab3882ea 100644 --- a/packages/lodestar/src/api/impl/lightclient/index.ts +++ b/packages/lodestar/src/api/impl/lightclient/index.ts @@ -2,8 +2,8 @@ import {ApiModules} from "../types"; import {resolveStateId} from "../beacon/state/utils"; import {routes} from "@chainsafe/lodestar-api"; import {linspace} from "../../../util/numpy"; -import {fromHexString, isCompositeType} from "@chainsafe/ssz"; -import {ProofType} from "@chainsafe/persistent-merkle-tree"; +import {fromHexString} from "@chainsafe/ssz"; +import {ProofType, Tree} from "@chainsafe/persistent-merkle-tree"; import {IApiOptions} from "../../options"; // TODO: Import from lightclient/server package @@ -17,31 +17,18 @@ export function getLightclientApi( const maxGindicesInProof = opts.maxGindicesInProof ?? 512; return { - async getStateProof(stateId, paths) { + async getStateProof(stateId, jsonPaths) { const state = await resolveStateId(config, chain, db, stateId); - // eslint-disable-next-line @typescript-eslint/naming-convention - const BeaconState = config.getForkTypes(state.slot).BeaconState; - const stateTreeBacked = BeaconState.createTreeBackedFromStruct(state); - const tree = stateTreeBacked.tree; - const gindicesSet = new Set(); + // Commit any changes before computing the state root. In normal cases the state should have no changes here + state.commit(); + const stateNode = state.node; + const tree = new Tree(stateNode); - for (const path of paths) { - // Logic from TreeBacked#createProof is (mostly) copied here to expose the # of gindices in the proof - const {type, gindex} = BeaconState.getPathInfo(path); - if (!isCompositeType(type)) { - gindicesSet.add(gindex); - } else { - // if the path subtype is composite, include the gindices of all the leaves - const gindexes = type.tree_getLeafGindices( - type.hasVariableSerializedLength() ? tree.getSubtree(gindex) : undefined, - gindex - ); - for (const gindex of gindexes) { - gindicesSet.add(gindex); - } - } - } + const gindexes = state.type.tree_createProofGindexes(stateNode, jsonPaths); + // TODO: Is it necessary to de-duplicate? + // It's not a problem if we overcount gindexes + const gindicesSet = new Set(gindexes); if (gindicesSet.size > maxGindicesInProof) { throw new Error("Requested proof is too large."); diff --git a/packages/lodestar/src/api/impl/lodestar/index.ts b/packages/lodestar/src/api/impl/lodestar/index.ts index a91d609f4acc..587c051907e3 100644 --- a/packages/lodestar/src/api/impl/lodestar/index.ts +++ b/packages/lodestar/src/api/impl/lodestar/index.ts @@ -2,7 +2,7 @@ import PeerId from "peer-id"; import {Multiaddr} from "multiaddr"; import {routes} from "@chainsafe/lodestar-api"; import {getLatestWeakSubjectivityCheckpointEpoch} from "@chainsafe/lodestar-beacon-state-transition"; -import {Json, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ssz} from "@chainsafe/lodestar-types"; import {BeaconChain} from "../../../chain"; @@ -159,7 +159,7 @@ export function getLodestarApi({ }; } -function regenRequestToJson(config: IChainForkConfig, regenRequest: RegenRequest): Json { +function regenRequestToJson(config: IChainForkConfig, regenRequest: RegenRequest): unknown { switch (regenRequest.key) { case "getBlockSlotState": return { diff --git a/packages/lodestar/src/api/impl/validator/index.ts b/packages/lodestar/src/api/impl/validator/index.ts index 93e4651748f7..0d87aff49142 100644 --- a/packages/lodestar/src/api/impl/validator/index.ts +++ b/packages/lodestar/src/api/impl/validator/index.ts @@ -64,7 +64,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: if (!genesisBlockRoot) { // Close to genesis the genesis block may not be available in the DB if (state.slot < SLOTS_PER_HISTORICAL_ROOT) { - genesisBlockRoot = state.blockRoots[0]; + genesisBlockRoot = state.blockRoots.get(0); } const genesisBlock = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT); @@ -285,7 +285,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: // Gather indexes to get pubkeys in batch (performance optimization) for (let i = 0; i < SLOTS_PER_EPOCH; i++) { // getBeaconProposer ensures the requested epoch is correct - const validatorIndex = state.getBeaconProposer(startSlot + i); + const validatorIndex = state.epochCtx.getBeaconProposer(startSlot + i); indexes.push(validatorIndex); } @@ -446,7 +446,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: await Promise.all([ chain.aggregatedAttestationPool.add( signedAggregateAndProof.message.aggregate, - indexedAttestation.attestingIndices.valueOf() as ValidatorIndex[], + indexedAttestation.attestingIndices, committeeIndices ), network.gossip.publishBeaconAggregateAndProof(signedAggregateAndProof), diff --git a/packages/lodestar/src/api/impl/validator/utils.ts b/packages/lodestar/src/api/impl/validator/utils.ts index f24d936d64ce..4d66c4471b87 100644 --- a/packages/lodestar/src/api/impl/validator/utils.ts +++ b/packages/lodestar/src/api/impl/validator/utils.ts @@ -1,7 +1,6 @@ -import {computeSlotsSinceEpochStart} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, computeSlotsSinceEpochStart} from "@chainsafe/lodestar-beacon-state-transition"; import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {allForks, BLSPubkey, CommitteeIndex, phase0, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {BranchNodeStruct, TreeValue, List} from "@chainsafe/ssz"; +import {BLSPubkey, CommitteeIndex, Slot, ValidatorIndex} from "@chainsafe/lodestar-types"; export function computeSubnetForCommitteesAtSlot( slot: Slot, @@ -23,11 +22,10 @@ export function computeSubnetForCommitteesAtSlot( * See benchmark -> packages/lodestar/test/perf/api/impl/validator/attester.test.ts */ export function getPubkeysForIndices( - validators: allForks.BeaconState["validators"], + validators: BeaconStateAllForks["validators"], indexes: ValidatorIndex[] ): BLSPubkey[] { const validatorsLen = validators.length; // Get once, it's expensive - const validatorsTree = ((validators as unknown) as TreeValue>).tree; const pubkeys: BLSPubkey[] = []; for (let i = 0, len = indexes.length; i < len; i++) { @@ -37,26 +35,9 @@ export function getPubkeysForIndices( } // NOTE: This could be optimized further by traversing the tree optimally with .getNodes() - const gindex = ssz.phase0.Validators.getGindexBitStringAtChunkIndex(index); - const node = validatorsTree.getNode(gindex) as BranchNodeStruct; - pubkeys.push(node.value.pubkey); + const validator = validators.getReadonly(index); + pubkeys.push(validator.pubkey); } return pubkeys; } - -/** - * Uses special BranchNodeStruct state.validators data structure to optimize getting pubkeys. - * Type-unsafe: assumes state.validators[i] is of BranchNodeStruct type. - */ -export function getPubkeysForIndex(validators: allForks.BeaconState["validators"], index: ValidatorIndex): BLSPubkey { - const validatorsLen = validators.length; - if (index >= validatorsLen) { - throw Error(`validatorIndex ${index} too high. Current validator count ${validatorsLen}`); - } - - const validatorsTree = ((validators as unknown) as TreeValue>).tree; - const gindex = ssz.phase0.Validators.getGindexBitStringAtChunkIndex(index); - const node = validatorsTree.getNode(gindex) as BranchNodeStruct; - return node.value.pubkey; -} diff --git a/packages/lodestar/src/chain/blocks/importBlock.ts b/packages/lodestar/src/chain/blocks/importBlock.ts index a6379ef5d86d..be179434c41c 100644 --- a/packages/lodestar/src/chain/blocks/importBlock.ts +++ b/packages/lodestar/src/chain/blocks/importBlock.ts @@ -1,5 +1,5 @@ import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {allForks} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, @@ -128,7 +128,7 @@ export async function importBlock(chain: ImportBlockModules, fullyVerifiedBlock: // - Register attestation with validator monitor (only after sync) // Only process attestations in response to an non-prefinalized block if (!skipImportingAttestations) { - const attestations = Array.from(readonlyValues(block.message.body.attestations)); + const attestations = block.message.body.attestations; const rootCache = new altair.RootCache(postState); const parentSlot = chain.forkChoice.getBlock(block.message.parentRoot)?.slot; const invalidAttestationErrorsByCode = new Map(); diff --git a/packages/lodestar/src/chain/blocks/utils/checkpoint.ts b/packages/lodestar/src/chain/blocks/utils/checkpoint.ts index 8b3d30f8220b..732dc480f366 100644 --- a/packages/lodestar/src/chain/blocks/utils/checkpoint.ts +++ b/packages/lodestar/src/chain/blocks/utils/checkpoint.ts @@ -7,7 +7,6 @@ import {ZERO_HASH} from "../../../constants"; * Compute a Checkpoint type from `state.latestBlockHeader` */ export function getCheckpointFromState(checkpointState: CachedBeaconStateAllForks): phase0.Checkpoint { - const config = checkpointState.config; const slot = checkpointState.slot; if (slot % SLOTS_PER_EPOCH !== 0) { @@ -16,7 +15,7 @@ export function getCheckpointFromState(checkpointState: CachedBeaconStateAllFork const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = config.getForkTypes(slot).BeaconState.hashTreeRoot(checkpointState); + blockHeader.stateRoot = checkpointState.hashTreeRoot(); } return { diff --git a/packages/lodestar/src/chain/blocks/verifyBlock.ts b/packages/lodestar/src/chain/blocks/verifyBlock.ts index 6a6bd1e3e452..daf6fabe1ba3 100644 --- a/packages/lodestar/src/chain/blocks/verifyBlock.ts +++ b/packages/lodestar/src/chain/blocks/verifyBlock.ts @@ -1,4 +1,3 @@ -import {ssz} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, @@ -19,6 +18,7 @@ import {IStateRegenerator, RegenCaller} from "../regen"; import {IBlsVerifier} from "../bls"; import {FullyVerifiedBlock, PartiallyVerifiedBlock} from "./types"; import {ExecutePayloadStatus} from "../../executionEngine/interface"; +import {byteArrayEquals} from "../../util/bytes"; export type VerifyBlockModules = { bls: IBlsVerifier; @@ -171,13 +171,7 @@ export async function verifyBlockStateTransition( let executionStatus: ExecutionStatus; if (executionPayloadEnabled) { // TODO: Handle better notifyNewPayload() returning error is syncing - const execResult = await chain.executionEngine.notifyNewPayload( - // executionPayload must be serialized as JSON and the TreeBacked structure breaks the baseFeePerGas serializer - // For clarity and since it's needed anyway, just send the struct representation at this level such that - // notifyNewPayload() can expect a regular JS object. - // TODO: If blocks are no longer TreeBacked, remove. - executionPayloadEnabled.valueOf() as typeof executionPayloadEnabled - ); + const execResult = await chain.executionEngine.notifyNewPayload(executionPayloadEnabled); switch (execResult.status) { case ExecutePayloadStatus.VALID: @@ -272,7 +266,7 @@ export async function verifyBlockStateTransition( } // Check state root matches - if (!ssz.Root.equals(block.message.stateRoot, postState.tree.root)) { + if (!byteArrayEquals(block.message.stateRoot, postState.hashTreeRoot())) { throw new BlockError(block, {code: BlockErrorCode.INVALID_STATE_ROOT, preState, postState}); } diff --git a/packages/lodestar/src/chain/bls/multithread/index.ts b/packages/lodestar/src/chain/bls/multithread/index.ts index 595bf15171d3..8490760f9490 100644 --- a/packages/lodestar/src/chain/bls/multithread/index.ts +++ b/packages/lodestar/src/chain/bls/multithread/index.ts @@ -144,7 +144,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { opts, sets: setsWorker.map((s) => ({ publicKey: getAggregatedPubkey(s).toBytes(this.format), - message: s.signingRoot.valueOf() as Uint8Array, + message: s.signingRoot, signature: s.signature, })), }) diff --git a/packages/lodestar/src/chain/bls/singleThread.ts b/packages/lodestar/src/chain/bls/singleThread.ts index b3b88e7be8b7..bb7e08261f9e 100644 --- a/packages/lodestar/src/chain/bls/singleThread.ts +++ b/packages/lodestar/src/chain/bls/singleThread.ts @@ -8,7 +8,7 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { return verifySignatureSetsMaybeBatch( sets.map((set) => ({ publicKey: getAggregatedPubkey(set), - message: set.signingRoot.valueOf() as Uint8Array, + message: set.signingRoot, signature: set.signature, })) ); diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index 92f27ce2d5d1..a240deb653dc 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -3,12 +3,19 @@ */ import fs from "node:fs"; -import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + computeStartSlotAtEpoch, + createCachedBeaconState, + Index2PubkeyCache, + PubkeyIndexMap, +} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; -import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types"; +import {allForks, UintNum64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {fromHexString, TreeBacked} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {AbortController} from "@chainsafe/abort-controller"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; @@ -22,7 +29,7 @@ import {IBeaconChain, SSZObjectType} from "./interface"; import {IChainOptions} from "./options"; import {IStateRegenerator, QueuedStateRegenerator, RegenCaller} from "./regen"; import {initializeForkChoice} from "./forkChoice"; -import {restoreStateCaches} from "./initState"; +import {computeAnchorCheckpoint} from "./initState"; import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls"; import { SeenAttesters, @@ -46,7 +53,7 @@ import {PrecomputeNextEpochTransitionScheduler} from "./precomputeNextEpochTrans import {ReprocessController} from "./reprocess"; export class BeaconChain implements IBeaconChain { - readonly genesisTime: Number64; + readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; readonly eth1: IEth1ForBlockProduction; readonly executionEngine: IExecutionEngine; @@ -78,6 +85,10 @@ export class BeaconChain implements IBeaconChain { readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof = new SeenContributionAndProof(); + // Global state caches + readonly pubkey2index: PubkeyIndexMap; + readonly index2pubkey: Index2PubkeyCache; + protected readonly blockProcessor: BlockProcessor; protected readonly db: IBeaconDb; protected readonly logger: ILogger; @@ -101,7 +112,7 @@ export class BeaconChain implements IBeaconChain { db: IBeaconDb; logger: ILogger; metrics: IMetrics | null; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; eth1: IEth1ForBlockProduction; executionEngine: IExecutionEngine; } @@ -113,7 +124,7 @@ export class BeaconChain implements IBeaconChain { this.metrics = metrics; this.genesisTime = anchorState.genesisTime; this.anchorStateLatestBlockSlot = anchorState.latestBlockHeader.slot; - this.genesisValidatorsRoot = anchorState.genesisValidatorsRoot.valueOf() as Uint8Array; + this.genesisValidatorsRoot = anchorState.genesisValidatorsRoot; this.eth1 = eth1; this.executionEngine = executionEngine; @@ -126,7 +137,21 @@ export class BeaconChain implements IBeaconChain { const clock = new LocalClock({config, emitter, genesisTime: this.genesisTime, signal}); const stateCache = new StateContextCache({metrics}); const checkpointStateCache = new CheckpointStateCache({metrics}); - const cachedState = restoreStateCaches(config, stateCache, checkpointStateCache, anchorState); + + // Initialize single global instance of state caches + this.pubkey2index = new PubkeyIndexMap(); + this.index2pubkey = []; + + // Restore state caches + const cachedState = createCachedBeaconState(anchorState, { + config, + pubkey2index: this.pubkey2index, + index2pubkey: this.index2pubkey, + }); + const {checkpoint} = computeAnchorCheckpoint(config, anchorState); + stateCache.add(cachedState); + checkpointStateCache.add(checkpoint, cachedState); + const forkChoice = initializeForkChoice( config, emitter, @@ -206,10 +231,6 @@ export class BeaconChain implements IBeaconChain { await this.opPool.toPersisted(this.db); } - getGenesisTime(): Number64 { - return this.genesisTime; - } - getHeadState(): CachedBeaconStateAllForks { // head state should always exist const head = this.forkChoice.getHead(); diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 3bd9c18112d2..1e968dc90e22 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -1,9 +1,13 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import {toHexString} from "@chainsafe/ssz"; import {allForks, Epoch, phase0, Slot, ssz, Version} from "@chainsafe/lodestar-types"; -import {ILogger} from "@chainsafe/lodestar-utils"; +import {Context, ILogger} from "@chainsafe/lodestar-utils"; import {CheckpointWithHex, IProtoBlock} from "@chainsafe/lodestar-fork-choice"; -import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + computeStartSlotAtEpoch, +} from "@chainsafe/lodestar-beacon-state-transition"; import {AttestationError, BlockError, BlockErrorCode} from "./errors"; import {ChainEvent, IChainEvents} from "./emitter"; import {BeaconChain} from "./chain"; @@ -96,13 +100,13 @@ export function onClockEpoch(this: BeaconChain, currentEpoch: Epoch): void { } export function onForkVersion(this: BeaconChain, version: Version): void { - this.logger.verbose("New fork version", ssz.Version.toJson(version)); + this.logger.verbose("New fork version", {version: ssz.Version.toJson(version) as string}); } export function onCheckpoint(this: BeaconChain, cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { - this.logger.verbose("Checkpoint processed", ssz.phase0.Checkpoint.toJson(cp)); + this.logger.verbose("Checkpoint processed", ssz.phase0.Checkpoint.toJson(cp) as Context); - this.metrics?.currentValidators.set({status: "active"}, state.currentShuffling.activeIndices.length); + this.metrics?.currentValidators.set({status: "active"}, state.epochCtx.currentShuffling.activeIndices.length); const parentBlockSummary = this.forkChoice.getBlock(state.latestBlockHeader.parentRoot); if (parentBlockSummary) { @@ -122,13 +126,13 @@ export function onCheckpoint(this: BeaconChain, cp: phase0.Checkpoint, state: Ca } export function onJustified(this: BeaconChain, cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { - this.logger.verbose("Checkpoint justified", ssz.phase0.Checkpoint.toJson(cp)); + this.logger.verbose("Checkpoint justified", ssz.phase0.Checkpoint.toJson(cp) as Context); this.metrics?.previousJustifiedEpoch.set(state.previousJustifiedCheckpoint.epoch); this.metrics?.currentJustifiedEpoch.set(cp.epoch); } export async function onFinalized(this: BeaconChain, cp: phase0.Checkpoint): Promise { - this.logger.verbose("Checkpoint finalized", ssz.phase0.Checkpoint.toJson(cp)); + this.logger.verbose("Checkpoint finalized", ssz.phase0.Checkpoint.toJson(cp) as Context); this.metrics?.finalizedEpoch.set(cp.epoch); } @@ -143,7 +147,7 @@ export async function onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWit // TODO: Improve using regen here const headState = this.stateCache.get(this.forkChoice.getHead().stateRoot); if (headState) { - this.opPool.pruneAll(headState); + this.opPool.pruneAll(headState as BeaconStateAllForks); } } @@ -167,7 +171,7 @@ export function onAttestation(this: BeaconChain, attestation: phase0.Attestation index: attestation.data.index, targetRoot: toHexString(attestation.data.target.root), aggregationBits: ssz.phase0.CommitteeBits.toJson(attestation.aggregationBits), - }); + } as Context); } export async function onBlock( diff --git a/packages/lodestar/src/chain/factory/block/body.ts b/packages/lodestar/src/chain/factory/block/body.ts index 440c9585843c..e54bf90fe35a 100644 --- a/packages/lodestar/src/chain/factory/block/body.ts +++ b/packages/lodestar/src/chain/factory/block/body.ts @@ -2,7 +2,6 @@ * @module chain/blockAssembly */ -import {List} from "@chainsafe/ssz"; import { Bytes96, Bytes32, @@ -68,11 +67,11 @@ export async function assembleBody( randaoReveal, graffiti, eth1Data, - proposerSlashings: proposerSlashings as List, - attesterSlashings: attesterSlashings as List, - attestations: attestations as List, - deposits: deposits as List, - voluntaryExits: voluntaryExits as List, + proposerSlashings: proposerSlashings, + attesterSlashings: attesterSlashings, + attestations: attestations, + deposits: deposits, + voluntaryExits: voluntaryExits, }; const blockEpoch = computeEpochAtSlot(blockSlot); @@ -102,7 +101,7 @@ export async function assembleBody( if (payloadId === null) { // Pre-merge, propose a pre-merge block with empty execution and keep the chain going - (blockBody as bellatrix.BeaconBlockBody).executionPayload = ssz.bellatrix.ExecutionPayload.defaultValue(); + (blockBody as bellatrix.BeaconBlockBody).executionPayload = ssz.bellatrix.ExecutionPayload.defaultValue; } else { (blockBody as bellatrix.BeaconBlockBody).executionPayload = await chain.executionEngine.getPayload(payloadId); } @@ -151,7 +150,7 @@ async function prepareExecutionPayload( } const timestamp = computeTimeAtSlot(chain.config, state.slot, state.genesisTime); - const random = getRandaoMix(state, state.currentShuffling.epoch); + const random = getRandaoMix(state, state.epochCtx.currentShuffling.epoch); const payloadId = await chain.executionEngine.notifyForkchoiceUpdate(parentHash, finalizedBlockHash, { timestamp, random, diff --git a/packages/lodestar/src/chain/factory/block/index.ts b/packages/lodestar/src/chain/factory/block/index.ts index 9d1252935342..210b81b894af 100644 --- a/packages/lodestar/src/chain/factory/block/index.ts +++ b/packages/lodestar/src/chain/factory/block/index.ts @@ -3,7 +3,6 @@ */ import {CachedBeaconStateAllForks, allForks} from "@chainsafe/lodestar-beacon-state-transition"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Bytes32, Bytes96, ExecutionAddress, Root, Slot} from "@chainsafe/lodestar-types"; import {fromHexString} from "@chainsafe/ssz"; @@ -38,7 +37,7 @@ export async function assembleBlock( const block: allForks.BeaconBlock = { slot, - proposerIndex: state.getBeaconProposer(slot), + proposerIndex: state.epochCtx.getBeaconProposer(slot), parentRoot: parentBlockRoot, stateRoot: ZERO_HASH, body: await assembleBody(chain, state, { @@ -51,7 +50,7 @@ export async function assembleBlock( }), }; - block.stateRoot = computeNewStateRoot({config: chain.config, metrics}, state, block); + block.stateRoot = computeNewStateRoot({metrics}, state, block); return block; } @@ -62,7 +61,7 @@ export async function assembleBlock( * epoch transition which happen at slot % 32 === 0) */ function computeNewStateRoot( - {config, metrics}: {config: IChainForkConfig; metrics: IMetrics | null}, + {metrics}: {metrics: IMetrics | null}, state: CachedBeaconStateAllForks, block: allForks.BeaconBlock ): Root { @@ -70,5 +69,5 @@ function computeNewStateRoot( // verifySignatures = false since the data to assemble the block is trusted allForks.processBlock(postState, block, {verifySignatures: false}, metrics); - return config.getForkTypes(state.slot).BeaconState.hashTreeRoot(postState); + return postState.hashTreeRoot(); } diff --git a/packages/lodestar/src/chain/genesis/genesis.ts b/packages/lodestar/src/chain/genesis/genesis.ts index 79938eab3bf7..dabb77b7627b 100644 --- a/packages/lodestar/src/chain/genesis/genesis.ts +++ b/packages/lodestar/src/chain/genesis/genesis.ts @@ -2,10 +2,10 @@ * @module chain/genesis */ -import {TreeBacked, List} from "@chainsafe/ssz"; -import {GENESIS_SLOT} from "@chainsafe/lodestar-params"; -import {Root, phase0, allForks, ssz} from "@chainsafe/lodestar-types"; +import {GENESIS_EPOCH, GENESIS_SLOT} from "@chainsafe/lodestar-params"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {AbortSignal} from "@chainsafe/abort-controller"; import { getTemporaryBlockHeader, @@ -13,16 +13,18 @@ import { applyDeposits, applyTimestamp, applyEth1BlockHash, - isValidGenesisState, - isValidGenesisValidators, CachedBeaconStateAllForks, createCachedBeaconState, + BeaconStateAllForks, + createEmptyEpochContextImmutableData, + getActiveValidatorIndices, } from "@chainsafe/lodestar-beacon-state-transition"; import {ILogger} from "@chainsafe/lodestar-utils"; import {IEth1Provider} from "../../eth1"; import {IEth1StreamParams} from "../../eth1/interface"; import {getDepositsAndBlockStreamForGenesis, getDepositsStream} from "../../eth1/stream"; import {IGenesisBuilder, IGenesisResult} from "./interface"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; export interface IGenesisBuilderKwargs { config: IChainForkConfig; @@ -31,8 +33,8 @@ export interface IGenesisBuilderKwargs { /** Use to restore pending progress */ pendingStatus?: { - state: TreeBacked; - depositTree: TreeBacked>; + state: BeaconStateAllForks; + depositTree: DepositTree; lastProcessedBlockNumber: number; }; @@ -42,8 +44,8 @@ export interface IGenesisBuilderKwargs { export class GenesisBuilder implements IGenesisBuilder { // Expose state to persist on error - state: CachedBeaconStateAllForks; - depositTree: TreeBacked>; + readonly state: CachedBeaconStateAllForks; + readonly depositTree: DepositTree; /** Is null if no block has been processed yet */ lastProcessedBlockNumber: number | null = null; @@ -56,12 +58,13 @@ export class GenesisBuilder implements IGenesisBuilder { private readonly fromBlock: number; private readonly logEvery = 30 * 1000; private lastLog = 0; + /** Current count of active validators in the state */ + private activatedValidatorCount: number; constructor({config, eth1Provider, logger, signal, pendingStatus, maxBlocksPerPoll}: IGenesisBuilderKwargs) { // at genesis builder, there is no genesis validator so we don't have a real IBeaconConfig // but we need IBeaconConfig to temporarily create CachedBeaconState, the cast here is safe since we don't use any getDomain here - // the use of state as CachedBeaconState is just for convenient, IGenesisResult returns TreeBacked anyway - this.config = config as IBeaconConfig; + // the use of state as CachedBeaconState is just for convenient, IGenesisResult returns TreeView anyway this.eth1Provider = eth1Provider; this.logger = logger; this.signal = signal; @@ -70,20 +73,27 @@ export class GenesisBuilder implements IGenesisBuilder { maxBlocksPerPoll: maxBlocksPerPoll ?? 10000, }; + let stateView: BeaconStateAllForks; + if (pendingStatus) { this.logger.info("Restoring pending genesis state", {block: pendingStatus.lastProcessedBlockNumber}); - this.state = createCachedBeaconState(this.config, pendingStatus.state); + stateView = pendingStatus.state; this.depositTree = pendingStatus.depositTree; this.fromBlock = Math.max(pendingStatus.lastProcessedBlockNumber + 1, this.eth1Provider.deployBlock); } else { - this.state = getGenesisBeaconState( - this.config, - ssz.phase0.Eth1Data.defaultValue(), - getTemporaryBlockHeader(this.config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) + stateView = getGenesisBeaconState( + config, + ssz.phase0.Eth1Data.defaultValue, + getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue) ); - this.depositTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + this.depositTree = ssz.phase0.DepositDataRootList.toViewDU(ssz.phase0.DepositDataRootList.defaultValue); this.fromBlock = this.eth1Provider.deployBlock; } + + // TODO - PENDING: Ensure EpochContextImmutableData is created only once + this.state = createCachedBeaconState(stateView, createEmptyEpochContextImmutableData(config, stateView)); + this.config = this.state.config; + this.activatedValidatorCount = getActiveValidatorIndices(stateView, GENESIS_EPOCH).length; } /** @@ -109,7 +119,10 @@ export class GenesisBuilder implements IGenesisBuilder { applyEth1BlockHash(this.state, block.blockHash); this.lastProcessedBlockNumber = block.blockNumber; - if (isValidGenesisState(this.config, this.state)) { + if ( + this.state.genesisTime >= this.config.MIN_GENESIS_TIME && + this.activatedValidatorCount >= this.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + ) { this.logger.info("Found genesis state", {blockNumber: block.blockNumber}); return { state: this.state, @@ -136,7 +149,7 @@ export class GenesisBuilder implements IGenesisBuilder { this.applyDeposits(depositEvents); this.lastProcessedBlockNumber = blockNumber; - if (isValidGenesisValidators(this.config, this.state)) { + if (this.activatedValidatorCount >= this.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT) { this.logger.info("Found enough genesis validators", {blockNumber}); return blockNumber; } else { @@ -155,13 +168,19 @@ export class GenesisBuilder implements IGenesisBuilder { .map((depositEvent) => { this.depositCache.add(depositEvent.index); this.depositTree.push(ssz.phase0.DepositData.hashTreeRoot(depositEvent.depositData)); + const gindex = toGindex(this.depositTree.type.depth, BigInt(depositEvent.index)); + + // Apply changes from the push above + this.depositTree.commit(); + const depositTreeNode = this.depositTree.node; return { - proof: this.depositTree.tree.getSingleProof(this.depositTree.type.getPropertyGindex(depositEvent.index)), + proof: new Tree(depositTreeNode).getSingleProof(gindex), data: depositEvent.depositData, }; }); - applyDeposits(this.config, this.state, newDeposits, this.depositTree); + const {activatedValidatorCount} = applyDeposits(this.config, this.state, newDeposits, this.depositTree); + this.activatedValidatorCount += activatedValidatorCount; // TODO: If necessary persist deposits here to this.db.depositData, this.db.depositDataRoot } diff --git a/packages/lodestar/src/chain/genesis/interface.ts b/packages/lodestar/src/chain/genesis/interface.ts index 5f3cf17bc157..35fb8f1d1cbf 100644 --- a/packages/lodestar/src/chain/genesis/interface.ts +++ b/packages/lodestar/src/chain/genesis/interface.ts @@ -1,10 +1,11 @@ -import {TreeBacked, List} from "@chainsafe/ssz"; -import {allForks, Root} from "@chainsafe/lodestar-types"; +import {ssz} from "@chainsafe/lodestar-types"; +import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {CompositeViewDU, VectorCompositeType} from "@chainsafe/ssz"; import {Eth1Block} from "../../eth1/interface"; export interface IGenesisResult { - state: TreeBacked; - depositTree: TreeBacked>; + state: CachedBeaconStateAllForks; + depositTree: CompositeViewDU>; block: Eth1Block; } diff --git a/packages/lodestar/src/chain/initState.ts b/packages/lodestar/src/chain/initState.ts index 6a373614fab9..d5d6c9d1892d 100644 --- a/packages/lodestar/src/chain/initState.ts +++ b/packages/lodestar/src/chain/initState.ts @@ -6,14 +6,14 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import { blockToHeader, computeEpochAtSlot, - createCachedBeaconState, phase0, + BeaconStateAllForks, CachedBeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {allForks, ssz} from "@chainsafe/lodestar-types"; -import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {GENESIS_SLOT, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; @@ -21,7 +21,6 @@ import {Eth1Provider} from "../eth1"; import {IMetrics} from "../metrics"; import {GenesisBuilder} from "./genesis/genesis"; import {IGenesisResult} from "./genesis/interface"; -import {CheckpointStateCache, StateContextCache} from "./stateCache"; import {Eth1Options} from "../eth1/options"; export async function persistGenesisResult( @@ -32,7 +31,7 @@ export async function persistGenesisResult( await Promise.all([ db.stateArchive.add(genesisResult.state), db.blockArchive.add(genesisBlock), - db.depositDataRoot.putList(genesisResult.depositTree), + db.depositDataRoot.putList(genesisResult.depositTree.getAllReadonlyValues()), db.eth1Data.put(genesisResult.block.timestamp, { ...genesisResult.block, depositCount: genesisResult.depositTree.length, @@ -44,7 +43,7 @@ export async function persistGenesisResult( export async function persistAnchorState( config: IChainForkConfig, db: IBeaconDb, - anchorState: TreeBacked + anchorState: BeaconStateAllForks ): Promise { if (anchorState.slot === GENESIS_SLOT) { const genesisBlock = createGenesisBlock(config, anchorState); @@ -60,11 +59,11 @@ export async function persistAnchorState( export function createGenesisBlock( config: IChainForkConfig, - genesisState: allForks.BeaconState + genesisState: BeaconStateAllForks ): allForks.SignedBeaconBlock { const types = config.getForkTypes(GENESIS_SLOT); - const genesisBlock = types.SignedBeaconBlock.defaultValue(); - const stateRoot = types.BeaconState.hashTreeRoot(genesisState); + const genesisBlock = types.SignedBeaconBlock.defaultValue; + const stateRoot = genesisState.hashTreeRoot(); genesisBlock.message.stateRoot = stateRoot; return genesisBlock; } @@ -84,7 +83,7 @@ export async function initStateFromEth1({ logger: ILogger; opts: Eth1Options; signal: AbortSignal; -}): Promise> { +}): Promise { logger.info("Listening to eth1 for genesis state"); const statePreGenesis = await db.preGenesisState.get(); @@ -104,9 +103,11 @@ export async function initStateFromEth1({ try { const genesisResult = await builder.waitForGenesis(); + + // Note: .hashTreeRoot() automatically commits() const genesisBlock = createGenesisBlock(config, genesisResult.state); const types = config.getForkTypes(GENESIS_SLOT); - const stateRoot = types.BeaconState.hashTreeRoot(genesisResult.state); + const stateRoot = genesisResult.state.hashTreeRoot(); const blockRoot = types.BeaconBlock.hashTreeRoot(genesisBlock.message); logger.info("Initializing genesis state", { @@ -125,8 +126,12 @@ export async function initStateFromEth1({ } catch (e) { if (builder.lastProcessedBlockNumber != null) { logger.info("Persisting genesis state", {block: builder.lastProcessedBlockNumber}); + + // Commit changed before serializing + builder.state.commit(); + await db.preGenesisState.put(builder.state); - await db.depositDataRoot.putList(builder.depositTree); + await db.depositDataRoot.putList(builder.depositTree.getAllReadonlyValues()); await db.preGenesisStateLastProcessedBlock.put(builder.lastProcessedBlockNumber); } throw e; @@ -140,7 +145,7 @@ export async function initStateFromDb( config: IChainForkConfig, db: IBeaconDb, logger: ILogger -): Promise> { +): Promise { const state = await db.stateArchive.lastValue(); if (!state) { throw new Error("No state exists in database"); @@ -149,10 +154,10 @@ export async function initStateFromDb( logger.info("Initializing beacon state from db", { slot: state.slot, epoch: computeEpochAtSlot(state.slot), - stateRoot: toHexString(config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state)), + stateRoot: toHexString(state.hashTreeRoot()), }); - return state as TreeBacked; + return state as BeaconStateAllForks; } /** @@ -162,12 +167,12 @@ export async function initStateFromAnchorState( config: IChainForkConfig, db: IBeaconDb, logger: ILogger, - anchorState: TreeBacked -): Promise> { + anchorState: BeaconStateAllForks +): Promise { logger.info("Initializing beacon state from anchor state", { slot: anchorState.slot, epoch: computeEpochAtSlot(anchorState.slot), - stateRoot: toHexString(config.getForkTypes(anchorState.slot).BeaconState.hashTreeRoot(anchorState)), + stateRoot: toHexString(anchorState.hashTreeRoot()), }); await persistAnchorState(config, db, anchorState); @@ -175,26 +180,7 @@ export async function initStateFromAnchorState( return anchorState; } -/** - * Restore a beacon state to the state cache. - */ -export function restoreStateCaches( - config: IBeaconConfig, - stateCache: StateContextCache, - checkpointStateCache: CheckpointStateCache, - state: TreeBacked -): CachedBeaconStateAllForks { - const {checkpoint} = computeAnchorCheckpoint(config, state); - - const cachedBeaconState = createCachedBeaconState(config, state); - - // store state in state caches - void stateCache.add(cachedBeaconState); - checkpointStateCache.add(checkpoint, cachedBeaconState); - return cachedBeaconState; -} - -export function initBeaconMetrics(metrics: IMetrics, state: TreeBacked): void { +export function initBeaconMetrics(metrics: IMetrics, state: BeaconStateAllForks): void { metrics.headSlot.set(state.slot); metrics.previousJustifiedEpoch.set(state.previousJustifiedCheckpoint.epoch); metrics.currentJustifiedEpoch.set(state.currentJustifiedCheckpoint.epoch); @@ -203,21 +189,21 @@ export function initBeaconMetrics(metrics: IMetrics, state: TreeBacked; /** Persist in-memory data to the DB. Call at least once before stopping the process */ persistToDisk(): Promise; - getGenesisTime(): Number64; getHeadState(): CachedBeaconStateAllForks; getHeadStateAtCurrentEpoch(): Promise; diff --git a/packages/lodestar/src/chain/lightClient/index.ts b/packages/lodestar/src/chain/lightClient/index.ts index 1f0022b41166..4656f8a6195d 100644 --- a/packages/lodestar/src/chain/lightClient/index.ts +++ b/packages/lodestar/src/chain/lightClient/index.ts @@ -7,7 +7,7 @@ import { } from "@chainsafe/lodestar-beacon-state-transition"; import {ILogger} from "@chainsafe/lodestar-utils"; import {routes} from "@chainsafe/lodestar-api"; -import {BitVector, toHexString} from "@chainsafe/ssz"; +import {BitArray, CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {IBeaconDb} from "../../db"; import {MapDef, pruneSetToMax} from "../../util/map"; import {ChainEvent, ChainEventEmitter} from "../emitter"; @@ -179,10 +179,8 @@ export class LightClientServer { this.logger = modules.logger; this.zero = { - finalizedHeader: ssz.phase0.BeaconBlockHeader.defaultValue(), - finalityBranch: ssz.altair.LightClientUpdate.getPropertyType( - "finalityBranch" - ).defaultValue() as altair.LightClientUpdate["finalityBranch"], + finalizedHeader: ssz.phase0.BeaconBlockHeader.defaultValue, + finalityBranch: ssz.altair.LightClientUpdate.fields["finalityBranch"].defaultValue, }; } @@ -487,10 +485,13 @@ export class LightClientServer { }); } - private async storeSyncCommittee(syncCommittee: altair.SyncCommittee, syncCommitteeRoot: Uint8Array): Promise { + private async storeSyncCommittee( + syncCommittee: CompositeViewDU, + syncCommitteeRoot: Uint8Array + ): Promise { const isKnown = await this.db.syncCommittee.has(syncCommitteeRoot); if (!isKnown) { - await this.db.syncCommittee.put(syncCommitteeRoot, syncCommittee); + await this.db.syncCommittee.putBinary(syncCommitteeRoot, syncCommittee.serialize()); } } @@ -545,12 +546,6 @@ export function isBetterUpdate( return prevUpdate.attestedHeader.slot > nextSyncAttestedData.attestedHeader.slot; } -export function sumBits(bits: BitVector): number { - let sum = 0; - for (const bit of bits) { - if (bit) { - sum++; - } - } - return sum; +export function sumBits(bits: BitArray): number { + return bits.getTrueBitIndexes().length; } diff --git a/packages/lodestar/src/chain/lightClient/proofs.ts b/packages/lodestar/src/chain/lightClient/proofs.ts index 544f8414eeb8..5d41242c9f78 100644 --- a/packages/lodestar/src/chain/lightClient/proofs.ts +++ b/packages/lodestar/src/chain/lightClient/proofs.ts @@ -1,10 +1,11 @@ -import {altair} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {FINALIZED_ROOT_GINDEX} from "@chainsafe/lodestar-params"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {SyncCommitteeWitness} from "./types"; -export function getSyncCommitteesWitness(state: TreeBacked): SyncCommitteeWitness { - const n1 = state.tree.rootNode; +export function getSyncCommitteesWitness(state: BeaconStateAllForks): SyncCommitteeWitness { + state.commit(); + const n1 = state.node; const n3 = n1.right; // [1]0110 const n6 = n3.left; // 1[0]110 const n13 = n6.right; // 10[1]10 @@ -37,6 +38,7 @@ export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitt return [syncCommitteesWitness.nextSyncCommitteeRoot, ...syncCommitteesWitness.witness]; } -export function getFinalizedRootProof(state: TreeBacked): Uint8Array[] { - return state.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); +export function getFinalizedRootProof(state: BeaconStateAllForks): Uint8Array[] { + state.commit(); + return new Tree(state.node).getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); } diff --git a/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts b/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts index edf79c496a78..8ecd286c036d 100644 --- a/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts @@ -6,16 +6,15 @@ import { SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX, } from "@chainsafe/lodestar-params"; -import {Epoch, ParticipationFlags, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {Epoch, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair, computeEpochAtSlot, phase0, - zipIndexesCommitteeBits, } from "@chainsafe/lodestar-beacon-state-transition"; -import {BitList, List, readonlyValues, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {pruneBySlot} from "./utils"; import {InsertOutcome} from "./types"; @@ -89,7 +88,7 @@ export class AggregatedAttestationPool { */ getAttestationsForBlock(state: CachedBeaconStateAllForks): phase0.Attestation[] { const stateSlot = state.slot; - const stateEpoch = state.currentShuffling.epoch; + const stateEpoch = state.epochCtx.epoch; const statePrevEpoch = stateEpoch - 1; const forkName = state.config.getForkName(stateSlot); @@ -195,8 +194,11 @@ export class AggregatedAttestationPool { const phase0State = state as CachedBeaconStatePhase0; const stateEpoch = computeEpochAtSlot(state.slot); - const previousEpochParticipants = extractParticipation(phase0State.previousEpochAttestations, state); - const currentEpochParticipants = extractParticipation(phase0State.currentEpochAttestations, state); + const previousEpochParticipants = extractParticipation( + phase0State.previousEpochAttestations.getAllReadonly(), + state + ); + const currentEpochParticipants = extractParticipation(phase0State.currentEpochAttestations.getAllReadonly(), state); return (epoch: Epoch) => { return epoch === stateEpoch @@ -216,8 +218,8 @@ export class AggregatedAttestationPool { // check for altair block already const altairState = state as CachedBeaconStateAltair; const stateEpoch = computeEpochAtSlot(state.slot); - const previousParticipation = altairState.previousEpochParticipation.persistent.toArray(); - const currentParticipation = altairState.currentEpochParticipation.persistent.toArray(); + const previousParticipation = altairState.previousEpochParticipation.getAll(); + const currentParticipation = altairState.currentEpochParticipation.getAll(); return (epoch: Epoch, committee: number[]) => { const participationStatus = @@ -270,6 +272,7 @@ export class MatchingDataAttestationGroup { for (const [i, existingAttestation] of this.attestations.entries()) { const existingAttestingIndices = existingAttestation.attestingIndices; const numIntersection = + // TODO: Intersect the uint8arrays from BitArray directly, it's probably much faster existingAttestingIndices.size >= attestingIndices.size ? intersection(existingAttestingIndices, attestingIndices) : intersection(attestingIndices, existingAttestingIndices); @@ -335,34 +338,26 @@ export function aggregateInto(attestation1: AttestationWithIndex, attestation2: } // Merge bits of attestation2 into attestation1 - bitArrayMergeOrWith(attestation1.attestation.aggregationBits, attestation2.attestation.aggregationBits); + attestation1.attestation.aggregationBits.mergeOrWith(attestation2.attestation.aggregationBits); - const signature1 = bls.Signature.fromBytes( - attestation1.attestation.signature.valueOf() as Uint8Array, - undefined, - true - ); - const signature2 = bls.Signature.fromBytes( - attestation2.attestation.signature.valueOf() as Uint8Array, - undefined, - true - ); + const signature1 = bls.Signature.fromBytes(attestation1.attestation.signature, undefined, true); + const signature2 = bls.Signature.fromBytes(attestation2.attestation.signature, undefined, true); attestation1.attestation.signature = bls.Signature.aggregate([signature1, signature2]).toBytes(); } export function extractParticipation( - attestations: List, + attestations: phase0.PendingAttestation[], state: CachedBeaconStateAllForks ): Set { const {epochCtx} = state; const allParticipants = new Set(); - for (const att of readonlyValues(attestations)) { + for (const att of attestations) { const aggregationBits = att.aggregationBits; const attData = att.data; const attSlot = attData.slot; const committeeIndex = attData.index; const committee = epochCtx.getBeaconCommittee(attSlot, committeeIndex); - const participants = zipIndexesCommitteeBits(committee, aggregationBits); + const participants = aggregationBits.intersectValues(committee); for (const participant of participants) { allParticipants.add(participant); } @@ -398,17 +393,6 @@ export function isValidAttestationData( return ssz.phase0.Checkpoint.equals(data.source, justifiedCheckpoint); } -/** - * Returns true if the `TIMELY_SOURCE` bit in a `ParticipationFlags` is set - */ -export function flagIsTimelySource(flag: ParticipationFlags): boolean { +function flagIsTimelySource(flag: number): boolean { return (flag & TIMELY_SOURCE) === TIMELY_SOURCE; } - -function bitArrayMergeOrWith(bits1: BitList, bits2: BitList): void { - for (let i = 0; i < bits2.length; i++) { - if (bits2[i]) { - bits1[i] = true; - } - } -} diff --git a/packages/lodestar/src/chain/opPools/attestationPool.ts b/packages/lodestar/src/chain/opPools/attestationPool.ts index ef687175554d..b46dba69874c 100644 --- a/packages/lodestar/src/chain/opPools/attestationPool.ts +++ b/packages/lodestar/src/chain/opPools/attestationPool.ts @@ -1,6 +1,6 @@ import {phase0, Slot, Root, ssz} from "@chainsafe/lodestar-types"; import bls, {PointFormat, Signature} from "@chainsafe/bls"; -import {BitList, readonlyValues, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; import {MapDef} from "../../util/map"; @@ -23,7 +23,7 @@ const MAX_ATTESTATIONS_PER_SLOT = 16_384; type AggregateFast = { data: phase0.Attestation["data"]; - aggregationBits: boolean[]; + aggregationBits: BitArray; signature: Signature; }; @@ -161,21 +161,21 @@ export class AttestationPool { * Aggregate a new contribution into `aggregate` mutating it */ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0.Attestation): InsertOutcome { - const bitIndex = bitArrayGetSingleTrueBit(attestation.aggregationBits); + const bitIndex = attestation.aggregationBits.getSingleTrueBit(); // Should never happen, attestations are verified against this exact condition before if (bitIndex === null) { throw Error("Invalid attestation not exactly one bit set"); } - if (aggregate.aggregationBits[bitIndex] === true) { + if (aggregate.aggregationBits.get(bitIndex) === true) { return InsertOutcome.AlreadyKnown; } - aggregate.aggregationBits[bitIndex] = true; + aggregate.aggregationBits.set(bitIndex, true); aggregate.signature = Signature.aggregate([ aggregate.signature, - bls.Signature.fromBytes(attestation.signature.valueOf() as Uint8Array, undefined, true), + bls.Signature.fromBytes(attestation.signature, undefined, true), ]); return InsertOutcome.Aggregated; } @@ -186,8 +186,9 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0. function attestationToAggregate(attestation: phase0.Attestation): AggregateFast { return { data: attestation.data, - aggregationBits: Array.from(readonlyValues(attestation.aggregationBits)), - signature: bls.Signature.fromBytes(attestation.signature.valueOf() as Uint8Array, undefined, true), + // clone because it will be mutated + aggregationBits: attestation.aggregationBits.clone(), + signature: bls.Signature.fromBytes(attestation.signature, undefined, true), }; } @@ -197,16 +198,7 @@ function attestationToAggregate(attestation: phase0.Attestation): AggregateFast function fastToAttestation(aggFast: AggregateFast): phase0.Attestation { return { data: aggFast.data, - aggregationBits: aggFast.aggregationBits as BitList, + aggregationBits: aggFast.aggregationBits, signature: aggFast.signature.toBytes(PointFormat.compressed), }; } - -function bitArrayGetSingleTrueBit(bits: BitList): number | null { - for (const [index, participated] of Array.from(readonlyValues(bits)).entries()) { - if (participated) { - return index; - } - } - return null; -} diff --git a/packages/lodestar/src/chain/opPools/opPool.ts b/packages/lodestar/src/chain/opPools/opPool.ts index 460f9797993f..e427d955f88d 100644 --- a/packages/lodestar/src/chain/opPools/opPool.ts +++ b/packages/lodestar/src/chain/opPools/opPool.ts @@ -3,6 +3,7 @@ import { computeEpochAtSlot, allForks, getAttesterSlashableIndices, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {Repository, Id} from "@chainsafe/lodestar-db"; import {MAX_PROPOSER_SLASHINGS, MAX_VOLUNTARY_EXITS} from "@chainsafe/lodestar-params"; @@ -126,7 +127,7 @@ export class OpPool { for (const proposerSlashing of this.proposerSlashings.values()) { const index = proposerSlashing.signedHeader1.message.proposerIndex; - const validator = state.validators[index]; + const validator = state.validators.get(index); if (!validator.slashed && validator.activationEpoch <= stateEpoch && stateEpoch < validator.withdrawableEpoch) { proposerSlashings.push(proposerSlashing); // Set of validators to be slashed, so we don't attempt to construct invalid attester slashings. @@ -143,7 +144,7 @@ export class OpPool { const slashableIndices = new Set(); for (let i = 0; i < attesterSlashing.intersectingIndices.length; i++) { const index = attesterSlashing.intersectingIndices[i]; - const validator = state.validators[index]; + const validator = state.validators.get(index); // If we already have a slashing for this index, we can continue on to the next slashing if (toBeSlashedIndices.has(index)) { @@ -198,7 +199,7 @@ export class OpPool { /** * Prune all types of transactions given the latest head state */ - pruneAll(headState: allForks.BeaconState): void { + pruneAll(headState: BeaconStateAllForks): void { this.pruneAttesterSlashings(headState); this.pruneProposerSlashings(headState); this.pruneVoluntaryExits(headState); @@ -207,7 +208,7 @@ export class OpPool { /** * Prune attester slashings for all slashed or withdrawn validators. */ - private pruneAttesterSlashings(headState: allForks.BeaconState): void { + private pruneAttesterSlashings(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; attesterSlashing: for (const [key, attesterSlashing] of this.attesterSlashings.entries()) { // Slashings that don't slash any validators can be dropped @@ -219,7 +220,7 @@ export class OpPool { // // We cannot check the `slashed` field since the `head` is not finalized and // a fork could un-slash someone. - if (headState.validators[index].exitEpoch > finalizedEpoch) { + if (headState.validators.get(index).exitEpoch > finalizedEpoch) { continue attesterSlashing; } } @@ -232,11 +233,11 @@ export class OpPool { /** * Prune proposer slashings for validators which are exited in the finalized epoch. */ - private pruneProposerSlashings(headState: allForks.BeaconState): void { + private pruneProposerSlashings(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; for (const [key, proposerSlashing] of this.proposerSlashings.entries()) { const index = proposerSlashing.signedHeader1.message.proposerIndex; - if (headState.validators[index].exitEpoch <= finalizedEpoch) { + if (headState.validators.get(index).exitEpoch <= finalizedEpoch) { this.proposerSlashings.delete(key); } } @@ -246,7 +247,7 @@ export class OpPool { * Call after finalizing * Prune if validator has already exited at or before the finalized checkpoint of the head. */ - private pruneVoluntaryExits(headState: allForks.BeaconState): void { + private pruneVoluntaryExits(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; for (const [key, voluntaryExit] of this.voluntaryExits.entries()) { // TODO: Improve this simplistic condition diff --git a/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts b/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts index d0fdf8f4440d..a95953b000a8 100644 --- a/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts +++ b/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts @@ -1,8 +1,7 @@ import bls, {PointFormat, Signature} from "@chainsafe/bls"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; import {altair, Root, Slot, SubcommitteeIndex} from "@chainsafe/lodestar-types"; -import {BitList, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; @@ -20,7 +19,7 @@ const SLOTS_RETAINED = 3; const MAX_ITEMS_PER_SLOT = 512; type ContributionFast = Omit & { - aggregationBits: boolean[]; + aggregationBits: BitArray; signature: Signature; }; @@ -84,7 +83,7 @@ export class SyncCommitteeMessagePool { return { ...contribution, - aggregationBits: contribution.aggregationBits as BitList, + aggregationBits: contribution.aggregationBits, signature: contribution.signature.toBytes(PointFormat.compressed), }; } @@ -107,15 +106,14 @@ function aggregateSignatureInto( signature: altair.SyncCommitteeMessage, indexInSubcommittee: number ): InsertOutcome { - if (contribution.aggregationBits[indexInSubcommittee] === true) { + if (contribution.aggregationBits.get(indexInSubcommittee) === true) { return InsertOutcome.AlreadyKnown; } - contribution.aggregationBits[indexInSubcommittee] = true; - + contribution.aggregationBits.set(indexInSubcommittee, true); contribution.signature = Signature.aggregate([ contribution.signature, - bls.Signature.fromBytes(signature.signature.valueOf() as Uint8Array, undefined, true), + bls.Signature.fromBytes(signature.signature, undefined, true), ]); return InsertOutcome.Aggregated; } @@ -129,14 +127,13 @@ function signatureToAggregate( indexInSubcommittee: number ): ContributionFast { const indexesPerSubnet = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); - const aggregationBits = newFilledArray(indexesPerSubnet, false); - aggregationBits[indexInSubcommittee] = true; + const aggregationBits = BitArray.fromSingleBit(indexesPerSubnet, indexInSubcommittee); return { slot: signature.slot, beaconBlockRoot: signature.beaconBlockRoot, subcommitteeIndex: subnet, aggregationBits, - signature: bls.Signature.fromBytes(signature.signature.valueOf() as Uint8Array, undefined, true), + signature: bls.Signature.fromBytes(signature.signature, undefined, true), }; } diff --git a/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts b/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts index 1b8cfcd34d5c..9e5e8029b757 100644 --- a/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts +++ b/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts @@ -1,8 +1,8 @@ import bls, {Signature} from "@chainsafe/bls"; -import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; +import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {altair, Slot, Root, ssz} from "@chainsafe/lodestar-types"; -import {newFilledArray, G2_POINT_AT_INFINITY} from "@chainsafe/lodestar-beacon-state-transition"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {G2_POINT_AT_INFINITY} from "@chainsafe/lodestar-beacon-state-transition"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; @@ -19,11 +19,14 @@ const SLOTS_RETAINED = 8; */ const MAX_ITEMS_PER_SLOT = 512; +// SyncContributionAndProofPool constructor ensures SYNC_COMMITTEE_SUBNET_SIZE is multiple of 8 +const SYNC_COMMITTEE_SUBNET_BYTES = SYNC_COMMITTEE_SUBNET_SIZE / 8; + /** * A one-one mapping to SyncContribution with fast data structure to help speed up the aggregation. */ export type SyncContributionFast = { - syncSubcommitteeBits: boolean[]; + syncSubcommitteeBits: BitArray; numParticipants: number; syncSubcommitteeSignature: Uint8Array; }; @@ -44,6 +47,13 @@ export class SyncContributionAndProofPool { private lowestPermissibleSlot = 0; + constructor() { + // Param guarantee for optimizations below that merge syncSubcommitteeBits as bytes + if (SYNC_COMMITTEE_SUBNET_SIZE % 8 !== 0) { + throw Error("SYNC_COMMITTEE_SUBNET_SIZE must be multiple of 8"); + } + } + /** * Only call this once we pass all validation. */ @@ -55,7 +65,7 @@ export class SyncContributionAndProofPool { // Reject if too old. if (slot < lowestPermissibleSlot) { - throw new OpPoolError({code: OpPoolErrorCode.SLOT_TOO_LOW, slot, lowestPermissibleSlot}); + return InsertOutcome.Old; } // Limit object per slot @@ -85,7 +95,7 @@ export class SyncContributionAndProofPool { // Must return signature as G2_POINT_AT_INFINITY when participating bits are empty // https://github.com/ethereum/eth2.0-specs/blob/30f2a076377264677e27324a8c3c78c590ae5e20/specs/altair/bls.md#eth2_fast_aggregate_verify return { - syncCommitteeBits: ssz.altair.SyncCommitteeBits.defaultValue(), + syncCommitteeBits: ssz.altair.SyncCommitteeBits.defaultValue, syncCommitteeSignature: G2_POINT_AT_INFINITY, }; } @@ -118,9 +128,9 @@ export function replaceIfBetter( return InsertOutcome.NotBetterThan; } - bestContribution.syncSubcommitteeBits = Array.from(readonlyValues(newContribution.aggregationBits)); + bestContribution.syncSubcommitteeBits = newContribution.aggregationBits; bestContribution.numParticipants = newNumParticipants; - bestContribution.syncSubcommitteeSignature = newContribution.signature as Uint8Array; + bestContribution.syncSubcommitteeSignature = newContribution.signature; return InsertOutcome.NewData; } @@ -133,10 +143,10 @@ export function contributionToFast( ): SyncContributionFast { return { // No need to clone, aggregationBits are not mutated, only replaced - syncSubcommitteeBits: Array.from(readonlyValues(contribution.aggregationBits)), + syncSubcommitteeBits: contribution.aggregationBits, numParticipants, // No need to deserialize, signatures are not aggregated until when calling .getAggregate() - syncSubcommitteeSignature: contribution.signature as Uint8Array, + syncSubcommitteeSignature: contribution.signature, }; } @@ -146,14 +156,14 @@ export function contributionToFast( */ export function aggregate(bestContributionBySubnet: Map): altair.SyncAggregate { // check for empty/undefined bestContributionBySubnet earlier - const syncCommitteeBits = newFilledArray(SYNC_COMMITTEE_SIZE, false); - const subnetSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); + const syncCommitteeBits = BitArray.fromBitLen(SYNC_COMMITTEE_SIZE); + const signatures: Signature[] = []; for (const [subnet, bestContribution] of bestContributionBySubnet.entries()) { - const indexOffset = subnet * subnetSize; + const byteOffset = subnet * SYNC_COMMITTEE_SUBNET_BYTES; - for (const [index, participated] of bestContribution.syncSubcommitteeBits.entries()) { - if (participated) syncCommitteeBits[indexOffset + index] = true; + for (let i = 0; i < SYNC_COMMITTEE_SUBNET_BYTES; i++) { + syncCommitteeBits.uint8Array[byteOffset + i] = bestContribution.syncSubcommitteeBits.uint8Array[i]; } signatures.push(bls.Signature.fromBytes(bestContribution.syncSubcommitteeSignature, undefined, true)); diff --git a/packages/lodestar/src/chain/regen/regen.ts b/packages/lodestar/src/chain/regen/regen.ts index 5bcc31873f35..f0ef414901fa 100644 --- a/packages/lodestar/src/chain/regen/regen.ts +++ b/packages/lodestar/src/chain/regen/regen.ts @@ -157,12 +157,8 @@ export class StateRegenerator implements IStateRegenerator { } for (const b of blocksToReplay.reverse()) { - const structBlock = await this.modules.db.block.get(fromHexString(b.blockRoot)); - if (!structBlock) { - throw Error(`No block found for ${b.blockRoot}`); - } - const block = this.modules.config.getForkTypes(b.slot).SignedBeaconBlock.createTreeBackedFromStruct(structBlock); - if (block === undefined) { + const block = await this.modules.db.block.get(fromHexString(b.blockRoot)); + if (!block) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_DB, blockRoot: b.blockRoot, diff --git a/packages/lodestar/src/chain/stateCache/stateContextCache.ts b/packages/lodestar/src/chain/stateCache/stateContextCache.ts index 9f45d044c368..0e2b576a6db0 100644 --- a/packages/lodestar/src/chain/stateCache/stateContextCache.ts +++ b/packages/lodestar/src/chain/stateCache/stateContextCache.ts @@ -1,5 +1,5 @@ -import {ByteVector, toHexString} from "@chainsafe/ssz"; -import {Epoch, RootHex} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/ssz"; +import {Epoch, Root, RootHex} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {routes} from "@chainsafe/lodestar-api"; import {IMetrics} from "../../metrics"; @@ -58,7 +58,7 @@ export class StateContextCache { } } - delete(root: ByteVector): void { + delete(root: Root): void { const key = toHexString(root); const item = this.cache.get(key); if (!item) return; @@ -66,7 +66,7 @@ export class StateContextCache { this.cache.delete(key); } - batchDelete(roots: ByteVector[]): void { + batchDelete(roots: Root[]): void { roots.map((root) => this.delete(root)); } diff --git a/packages/lodestar/src/chain/validation/aggregateAndProof.ts b/packages/lodestar/src/chain/validation/aggregateAndProof.ts index de3bd166b282..264eecc2a062 100644 --- a/packages/lodestar/src/chain/validation/aggregateAndProof.ts +++ b/packages/lodestar/src/chain/validation/aggregateAndProof.ts @@ -1,11 +1,9 @@ import {ValidatorIndex} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; import { phase0, allForks, computeEpochAtSlot, isAggregatorFromCommitteeLength, - zipIndexesCommitteeBits, } from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconChain} from ".."; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets"; @@ -71,9 +69,9 @@ export async function validateGossipAggregateAndProof( }); const committeeIndices = getCommitteeIndices(attHeadState, attSlot, attData.index); - const attestingIndices = zipIndexesCommitteeBits(committeeIndices, aggregate.aggregationBits); + const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: attestingIndices as List, + attestingIndices, data: attData, signature: aggregate.signature, }; @@ -102,7 +100,7 @@ export async function validateGossipAggregateAndProof( // by the validator with index aggregate_and_proof.aggregator_index. // [REJECT] The aggregator signature, signed_aggregate_and_proof.signature, is valid. // [REJECT] The signature of aggregate is valid. - const aggregator = attHeadState.index2pubkey[aggregateAndProof.aggregatorIndex]; + const aggregator = attHeadState.epochCtx.index2pubkey[aggregateAndProof.aggregatorIndex]; const signatureSets = [ getSelectionProofSignatureSet(attHeadState, attSlot, aggregator, signedAggregateAndProof), getAggregateAndProofSignatureSet(attHeadState, attEpoch, aggregator, signedAggregateAndProof), diff --git a/packages/lodestar/src/chain/validation/attestation.ts b/packages/lodestar/src/chain/validation/attestation.ts index ee7a2b66c0af..965aba7855ca 100644 --- a/packages/lodestar/src/chain/validation/attestation.ts +++ b/packages/lodestar/src/chain/validation/attestation.ts @@ -1,14 +1,11 @@ import {Epoch, Root, Slot} from "@chainsafe/lodestar-types"; import {IProtoBlock} from "@chainsafe/lodestar-fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import { allForks, phase0, computeEpochAtSlot, - getSingleBitIndex, - AggregationBitsError, - AggregationBitsErrorCode, CachedBeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconChain} from ".."; @@ -55,15 +52,9 @@ export async function validateGossipAttestation( // (len([bit for bit in attestation.aggregation_bits if bit]) == 1, i.e. exactly 1 bit is set). // > TODO: Do this check **before** getting the target state but don't recompute zipIndexes const aggregationBits = attestation.aggregationBits; - let bitIndex: number; - try { - bitIndex = getSingleBitIndex(aggregationBits); - } catch (e) { - if (e instanceof AggregationBitsError && e.type.code === AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET}); - } else { - throw e; - } + const bitIndex = aggregationBits.getSingleTrueBit(); + if (bitIndex === null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET}); } // Attestations must be for a known block. If the block is unknown, we simply drop the @@ -104,7 +95,7 @@ export async function validateGossipAttestation( // [REJECT] The number of aggregation bits matches the committee size // -- i.e. len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index)). // > TODO: Is this necessary? Lighthouse does not do this check - if (aggregationBits.length !== committeeIndices.length) { + if (aggregationBits.bitLen !== committeeIndices.length) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS}); } @@ -117,7 +108,7 @@ export async function validateGossipAttestation( // -- i.e. compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index) == subnet_id, // where committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch), // which may be pre-computed along with the committee information for the signature check. - const expectedSubnet = computeSubnetForSlot(attHeadState, attSlot, attIndex); + const expectedSubnet = attHeadState.epochCtx.computeSubnetForSlot(attSlot, attIndex); if (subnet !== null && subnet !== expectedSubnet) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SUBNET_ID, @@ -138,7 +129,7 @@ export async function validateGossipAttestation( // [REJECT] The signature of attestation is valid. const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: [validatorIndex] as List, + attestingIndices: [validatorIndex], data: attData, signature: attestation.signature, }; @@ -232,7 +223,7 @@ function verifyHeadBlockIsKnown(chain: IBeaconChain, beaconBlockRoot: Root): IPr if (headBlock === null) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.UNKNOWN_BEACON_BLOCK_ROOT, - root: toHexString(beaconBlockRoot.valueOf() as typeof beaconBlockRoot), + root: toHexString(beaconBlockRoot as typeof beaconBlockRoot), }); } @@ -292,7 +283,7 @@ export function getCommitteeIndices( attestationSlot: Slot, attestationIndex: number ): number[] { - const {committees} = attestationTargetState.getShufflingAtSlot(attestationSlot); + const {committees} = attestationTargetState.epochCtx.getShufflingAtSlot(attestationSlot); const slotCommittees = committees[attestationSlot % SLOTS_PER_EPOCH]; if (attestationIndex >= slotCommittees.length) { @@ -309,7 +300,7 @@ export function getCommitteeIndices( */ export function computeSubnetForSlot(state: CachedBeaconStateAllForks, slot: number, committeeIndex: number): number { const slotsSinceEpochStart = slot % SLOTS_PER_EPOCH; - const committeesPerSlot = state.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); + const committeesPerSlot = state.epochCtx.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); const committeesSinceEpochStart = committeesPerSlot * slotsSinceEpochStart; return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts index 16f6e1f29683..416bed8420ab 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -24,6 +24,6 @@ export function getAggregateAndProofSignatureSet( type: SignatureSetType.single, pubkey: aggregator, signingRoot, - signature: aggregateAndProof.signature.valueOf() as Uint8Array, + signature: aggregateAndProof.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts b/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts index 1740490bc5d5..a8fd97e54634 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts @@ -21,6 +21,6 @@ export function getContributionAndProofSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[signedContributionAndProof.message.aggregatorIndex], signingRoot: computeSigningRoot(ssz.altair.ContributionAndProof, signingData, domain), - signature: signedContributionAndProof.signature.valueOf() as Uint8Array, + signature: signedContributionAndProof.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts b/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts index fde527984810..dbcf2a90bc12 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts @@ -20,6 +20,6 @@ export function getSelectionProofSignatureSet( type: SignatureSetType.single, pubkey: aggregator, signingRoot: computeSigningRoot(ssz.Slot, slot, selectionProofDomain), - signature: aggregateAndProof.message.selectionProof.valueOf() as Uint8Array, + signature: aggregateAndProof.message.selectionProof, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts index c12dd4c8de88..1c5d02732c36 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts @@ -17,6 +17,6 @@ export function getSyncCommitteeSignatureSet( type: SignatureSetType.single, pubkey: state.epochCtx.index2pubkey[syncCommittee.validatorIndex], signingRoot: computeSigningRoot(ssz.Root, syncCommittee.beaconBlockRoot, domain), - signature: syncCommittee.signature.valueOf() as Uint8Array, + signature: syncCommittee.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts index 2184bb019063..ac323996b108 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts @@ -1,7 +1,6 @@ import {PublicKey} from "@chainsafe/bls"; import {altair, ssz} from "@chainsafe/lodestar-types"; -import {DOMAIN_SYNC_COMMITTEE, SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; +import {DOMAIN_SYNC_COMMITTEE} from "@chainsafe/lodestar-params"; import { CachedBeaconStateAltair, computeSigningRoot, @@ -19,33 +18,6 @@ export function getSyncCommitteeContributionSignatureSet( type: SignatureSetType.aggregate, pubkeys, signingRoot: computeSigningRoot(ssz.Root, contribution.beaconBlockRoot, domain), - signature: contribution.signature.valueOf() as Uint8Array, + signature: contribution.signature, }; } - -/** - * Retrieve pubkeys in contribution aggregate using epochCtx: - * - currSyncCommitteeIndexes cache - * - index2pubkey cache - */ -export function getContributionPubkeys( - state: CachedBeaconStateAltair, - contribution: altair.SyncCommitteeContribution -): PublicKey[] { - const pubkeys: PublicKey[] = []; - - const subcommitteeSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); - const startIndex = contribution.subcommitteeIndex * subcommitteeSize; - const aggBits = Array.from(readonlyValues(contribution.aggregationBits)); - const syncCommittee = state.epochCtx.getIndexedSyncCommittee(contribution.slot); - for (const [i, bit] of aggBits.entries()) { - if (bit) { - const indexInCommittee = startIndex + i; - const validatorIndex = syncCommittee.validatorIndices[indexInCommittee]; - const pubkey = state.index2pubkey[validatorIndex]; - pubkeys.push(pubkey); - } - } - - return pubkeys; -} diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts index 5a5bffc3eda5..ca7428b6ea4c 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts @@ -22,6 +22,6 @@ export function getSyncCommitteeSelectionProofSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[contributionAndProof.aggregatorIndex], signingRoot: computeSigningRoot(ssz.altair.SyncAggregatorSelectionData, signingData, domain), - signature: contributionAndProof.selectionProof.valueOf() as Uint8Array, + signature: contributionAndProof.selectionProof, }; } diff --git a/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts b/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts index 3aa141347b18..9a160a017618 100644 --- a/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts +++ b/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts @@ -1,5 +1,6 @@ import {CachedBeaconStateAltair, isSyncCommitteeAggregator} from "@chainsafe/lodestar-beacon-state-transition"; -import {altair} from "@chainsafe/lodestar-types"; +import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {GossipAction, SyncCommitteeError, SyncCommitteeErrorCode} from "../errors"; import {IBeaconChain} from "../interface"; import {validateGossipSyncCommitteeExceptSig} from "./syncCommittee"; @@ -7,7 +8,6 @@ import { getSyncCommitteeSelectionProofSignatureSet, getContributionAndProofSignatureSet, getSyncCommitteeContributionSignatureSet, - getContributionPubkeys, } from "./signatureSets"; /** @@ -44,8 +44,8 @@ export async function validateSyncCommitteeGossipContributionAndProof( } // [REJECT] The contribution has participants -- that is, any(contribution.aggregation_bits) - const pubkeys = getContributionPubkeys(headState as CachedBeaconStateAltair, contribution); - if (!pubkeys.length) { + const syncCommitteeIndices = getContributionIndices(headState as CachedBeaconStateAltair, contribution); + if (!syncCommitteeIndices.length) { throw new SyncCommitteeError(GossipAction.REJECT, { code: SyncCommitteeErrorCode.NO_PARTICIPANT, }); @@ -64,6 +64,7 @@ export async function validateSyncCommitteeGossipContributionAndProof( // i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in get_sync_subcommittee_pubkeys(state, contribution.subcommittee_index). // > Checked in validateGossipSyncCommitteeExceptSig() + const pubkeys = syncCommitteeIndices.map((validatorIndex) => headState.epochCtx.index2pubkey[validatorIndex]); const signatureSets = [ // [REJECT] The contribution_and_proof.selection_proof is a valid signature of the SyncAggregatorSelectionData // derived from the contribution by the validator with index contribution_and_proof.aggregator_index. @@ -86,5 +87,22 @@ export async function validateSyncCommitteeGossipContributionAndProof( // no need to add to seenSyncCommittteeContributionCache here, gossip handler will do that chain.seenContributionAndProof.add(slot, subcommitteeIndex, aggregatorIndex); - return {syncCommitteeParticipants: pubkeys.length}; + return {syncCommitteeParticipants: syncCommitteeIndices.length}; +} + +/** + * Retrieve pubkeys in contribution aggregate using epochCtx: + * - currSyncCommitteeIndexes cache + * - index2pubkey cache + */ +function getContributionIndices( + state: CachedBeaconStateAltair, + contribution: altair.SyncCommitteeContribution +): ValidatorIndex[] { + const startIndex = contribution.subcommitteeIndex * SYNC_COMMITTEE_SUBNET_SIZE; + + const syncCommittee = state.epochCtx.getIndexedSyncCommittee(contribution.slot); + // The bits in contribution.aggregationBits select validatorIndexes in the subcommittee starting at startIndex + const subcommitteeIndices = syncCommittee.validatorIndices.slice(startIndex, startIndex + SYNC_COMMITTEE_SUBNET_SIZE); + return contribution.aggregationBits.intersectValues(subcommitteeIndices); } diff --git a/packages/lodestar/src/db/repositories/blockArchive.ts b/packages/lodestar/src/db/repositories/blockArchive.ts index 163ddba855d6..5aa9d3fbb697 100644 --- a/packages/lodestar/src/db/repositories/blockArchive.ts +++ b/packages/lodestar/src/db/repositories/blockArchive.ts @@ -1,5 +1,4 @@ import all from "it-all"; -import {ArrayLike} from "@chainsafe/ssz"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Db, Repository, IKeyValue, IFilterOptions, Bucket, IDbMetrics} from "@chainsafe/lodestar-db"; import {Slot, Root, allForks, ssz} from "@chainsafe/lodestar-types"; @@ -59,7 +58,7 @@ export class BlockArchiveRepository extends Repository>): Promise { + async batchPut(items: IKeyValue[]): Promise { await Promise.all([ super.batchPut(items), Array.from(items).map((item) => { @@ -75,7 +74,7 @@ export class BlockArchiveRepository extends Repository): Promise { + async batchPutBinary(items: BlockArchiveBatchPutBinaryItem[]): Promise { await Promise.all([ super.batchPutBinary(items), Array.from(items).map((item) => storeRootIndex(this.db, item.slot, item.blockRoot)), @@ -91,7 +90,7 @@ export class BlockArchiveRepository extends Repository): Promise { + async batchRemove(values: allForks.SignedBeaconBlock[]): Promise { await Promise.all([ super.batchRemove(values), Array.from(values).map((value) => diff --git a/packages/lodestar/src/db/repositories/blockArchiveIndex.ts b/packages/lodestar/src/db/repositories/blockArchiveIndex.ts index a8dc4aeda35c..3587eea954fd 100644 --- a/packages/lodestar/src/db/repositories/blockArchiveIndex.ts +++ b/packages/lodestar/src/db/repositories/blockArchiveIndex.ts @@ -1,7 +1,6 @@ import {Db, encodeKey, Bucket} from "@chainsafe/lodestar-db"; -import {Slot, Root, allForks} from "@chainsafe/lodestar-types"; +import {Slot, Root, allForks, ssz} from "@chainsafe/lodestar-types"; import {intToBytes} from "@chainsafe/lodestar-utils"; -import {ContainerType} from "@chainsafe/ssz"; export async function storeRootIndex(db: Db, slot: Slot, blockRoot: Root): Promise { return db.put(getRootIndexKey(blockRoot), intToBytes(slot, 8, "be")); @@ -13,10 +12,11 @@ export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root) export async function deleteRootIndex( db: Db, - blockType: ContainerType, + signedBeaconBlockType: allForks.AllForksSSZTypes["SignedBeaconBlock"], block: allForks.SignedBeaconBlock ): Promise { - return db.delete(getRootIndexKey(blockType.fields["message"].hashTreeRoot(block.message))); + const beaconBlockType = (signedBeaconBlockType as typeof ssz.phase0.SignedBeaconBlock).fields["message"]; + return db.delete(getRootIndexKey(beaconBlockType.hashTreeRoot(block.message))); } export async function deleteParentRootIndex(db: Db, block: allForks.SignedBeaconBlock): Promise { @@ -24,9 +24,9 @@ export async function deleteParentRootIndex(db: Db, block: allForks.SignedBeacon } export function getParentRootIndexKey(parentRoot: Root): Uint8Array { - return encodeKey(Bucket.index_blockArchiveParentRootIndex, parentRoot.valueOf() as Uint8Array); + return encodeKey(Bucket.index_blockArchiveParentRootIndex, parentRoot); } export function getRootIndexKey(root: Root): Uint8Array { - return encodeKey(Bucket.index_blockArchiveRootIndex, root.valueOf() as Uint8Array); + return encodeKey(Bucket.index_blockArchiveRootIndex, root); } diff --git a/packages/lodestar/src/db/repositories/depositDataRoot.ts b/packages/lodestar/src/db/repositories/depositDataRoot.ts index a75252aa4c3e..70700d100ee3 100644 --- a/packages/lodestar/src/db/repositories/depositDataRoot.ts +++ b/packages/lodestar/src/db/repositories/depositDataRoot.ts @@ -1,11 +1,14 @@ -import {List, readonlyValues, TreeBacked, Vector} from "@chainsafe/ssz"; +import {ByteVectorType, CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; import {Root, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {bytesToInt} from "@chainsafe/lodestar-utils"; import {Db, Bucket, Repository, IKeyValue, IDbMetrics} from "@chainsafe/lodestar-db"; +// TODO: Review where is best to put this type +export type DepositTree = CompositeViewDU>; + export class DepositDataRootRepository extends Repository { - private depositRootTree?: TreeBacked>; + private depositRootTree?: DepositTree; constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { super(config, db, Bucket.index_depositDataRoot, ssz.Root, metrics); @@ -21,22 +24,20 @@ export class DepositDataRootRepository extends Repository { throw new Error("Unable to create depositIndex from root"); } - async put(id: number, value: Root): Promise { - const depositRootTree = await this.getDepositRootTree(); - await super.put(id, value); - depositRootTree[id] = value as TreeBacked; + async put(index: number, value: Root): Promise { + await super.put(index, value); + await this.depositRootTreeSet(index, value); } async batchPut(items: IKeyValue[]): Promise { - const depositRootTree = await this.getDepositRootTree(); await super.batchPut(items); for (const {key, value} of items) { - depositRootTree[key] = value as TreeBacked; + await this.depositRootTreeSet(key, value); } } - async putList(list: List): Promise { - await this.batchPut(Array.from(readonlyValues(list), (value, key) => ({key, value}))); + async putList(roots: Root[]): Promise { + await this.batchPut(roots.map((root, index) => ({key: index, value: root}))); } async batchPutValues(values: {index: number; root: Root}[]): Promise { @@ -48,25 +49,29 @@ export class DepositDataRootRepository extends Repository { ); } - async getTreeBacked(depositIndex: number): Promise>> { - const depositRootTree = await this.getDepositRootTree(); - const tree = depositRootTree.clone(); - let maxIndex = tree.length - 1; - if (depositIndex > maxIndex) { - throw new Error(`Cannot get tree for unseen deposits: requested ${depositIndex}, last seen ${maxIndex}`); - } - while (maxIndex > depositIndex) { - tree.pop(); - maxIndex = tree.length - 1; + async getDepositRootTree(): Promise { + if (!this.depositRootTree) { + const values = await this.values(); + this.depositRootTree = ssz.phase0.DepositDataRootList.toViewDU(values); } - return tree; + return this.depositRootTree; } - async getDepositRootTree(): Promise>> { - if (!this.depositRootTree) { - const values = (await this.values()) as List>; - this.depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct(values); + async getDepositRootTreeAtIndex(depositIndex: number): Promise { + const depositRootTree = await this.getDepositRootTree(); + return depositRootTree.sliceTo(depositIndex); + } + + private async depositRootTreeSet(index: number, value: Uint8Array): Promise { + const depositRootTree = await this.getDepositRootTree(); + + // TODO: Review and fix properly + if (index > depositRootTree.length) { + throw Error(`Error setting depositRootTree index ${index} > length ${depositRootTree.length}`); + } else if (index === depositRootTree.length) { + depositRootTree.push(value); + } else { + depositRootTree.set(index, value); } - return this.depositRootTree; } } diff --git a/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts b/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts index 2dc88bde8ec6..59fa6ded3482 100644 --- a/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts +++ b/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts @@ -2,12 +2,8 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Bucket, IDatabaseController, IDbMetrics, Repository} from "@chainsafe/lodestar-db"; import {FINALIZED_ROOT_DEPTH} from "@chainsafe/lodestar-params"; import {ssz, SyncPeriod} from "@chainsafe/lodestar-types"; -import {booleanType, ContainerType, VectorType} from "@chainsafe/ssz"; -import { - PartialLightClientUpdate, - PartialLightClientUpdateFinalized, - PartialLightClientUpdateNonFinalized, -} from "../../chain/lightClient/types"; +import {BooleanType, ContainerType, VectorCompositeType} from "@chainsafe/ssz"; +import {PartialLightClientUpdate} from "../../chain/lightClient/types"; /** * Best PartialLightClientUpdate in each SyncPeriod @@ -15,42 +11,23 @@ import { * Used to prepare light client updates */ export class BestPartialLightClientUpdateRepository extends Repository { - typeFinalized = new ContainerType({ - fields: { - // isFinalized: true - isFinalized: booleanType, - attestedHeader: ssz.phase0.BeaconBlockHeader, - blockRoot: ssz.Root, - finalityBranch: new VectorType({length: FINALIZED_ROOT_DEPTH, elementType: ssz.Root}), - finalizedCheckpoint: ssz.phase0.Checkpoint, - finalizedHeader: ssz.phase0.BeaconBlockHeader, - syncCommitteeAggregate: ssz.altair.SyncAggregate, - }, - casingMap: { - isFinalized: "is_finalized", - attestedHeader: "attested_header", - blockRoot: "block_root", - finalityBranch: "finality_branch", - finalizedCheckpoint: "finalized_checkpoint", - finalizedHeader: "finalized_header", - syncCommitteeAggregate: "sync_committee_aggregate", - }, + typeFinalized = new ContainerType({ + // isFinalized: true + isFinalized: new BooleanType(), + attestedHeader: ssz.phase0.BeaconBlockHeader, + blockRoot: ssz.Root, + finalityBranch: new VectorCompositeType(ssz.Root, FINALIZED_ROOT_DEPTH), + finalizedCheckpoint: ssz.phase0.Checkpoint, + finalizedHeader: ssz.phase0.BeaconBlockHeader, + syncCommitteeAggregate: ssz.altair.SyncAggregate, }); - typeNonFinalized = new ContainerType({ - fields: { - // isFinalized: false - isFinalized: booleanType, - attestedHeader: ssz.phase0.BeaconBlockHeader, - blockRoot: ssz.Root, - syncCommitteeAggregate: ssz.altair.SyncAggregate, - }, - casingMap: { - isFinalized: "is_finalized", - attestedHeader: "attested_header", - blockRoot: "block_root", - syncCommitteeAggregate: "sync_committee_aggregate", - }, + typeNonFinalized = new ContainerType({ + // isFinalized: false + isFinalized: new BooleanType(), + attestedHeader: ssz.phase0.BeaconBlockHeader, + blockRoot: ssz.Root, + syncCommitteeAggregate: ssz.altair.SyncAggregate, }); constructor(config: IChainForkConfig, db: IDatabaseController, metrics?: IDbMetrics) { @@ -72,7 +49,7 @@ export class BestPartialLightClientUpdateRepository extends Repository { constructor(config: IChainForkConfig, db: IDatabaseController, metrics?: IDbMetrics) { - const type = new ContainerType({ - fields: { - witness: new VectorType({length: 4, elementType: ssz.Root}), - currentSyncCommitteeRoot: ssz.Root, - nextSyncCommitteeRoot: ssz.Root, - }, + const type = new ContainerType({ + witness: new VectorCompositeType(ssz.Root, 4), + currentSyncCommitteeRoot: ssz.Root, + nextSyncCommitteeRoot: ssz.Root, }); super(config, db, Bucket.lightClient_syncCommitteeWitness, type, metrics); diff --git a/packages/lodestar/src/db/repositories/stateArchive.ts b/packages/lodestar/src/db/repositories/stateArchive.ts index 001ec7acb992..c624e0e7bcec 100644 --- a/packages/lodestar/src/db/repositories/stateArchive.ts +++ b/packages/lodestar/src/db/repositories/stateArchive.ts @@ -1,5 +1,5 @@ -import {ContainerType, TreeBacked} from "@chainsafe/ssz"; -import {Epoch, Root, Slot, allForks, ssz} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {Epoch, Root, Slot, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {bytesToInt} from "@chainsafe/lodestar-utils"; import {Db, Bucket, Repository, IDbMetrics} from "@chainsafe/lodestar-db"; @@ -8,40 +8,41 @@ import {getRootIndexKey, storeRootIndex} from "./stateArchiveIndex"; /* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call */ -export class StateArchiveRepository extends Repository> { +export class StateArchiveRepository extends Repository { constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { - // Pick some type but won't be used - const type = (ssz.phase0.BeaconState as unknown) as ContainerType>; + // Pick some type but won't be used. Casted to any because no type can match `BeaconStateAllForks` + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + const type = ssz.phase0.BeaconState as any; super(config, db, Bucket.allForks_stateArchive, type, metrics); } // Overrides for multi-fork - encodeValue(value: allForks.BeaconState): Buffer { - return this.config.getForkTypes(value.slot).BeaconState.serialize(value) as Buffer; + encodeValue(value: BeaconStateAllForks): Uint8Array { + return value.serialize(); } - decodeValue(data: Buffer): TreeBacked { - return getStateTypeFromBytes(this.config, data).createTreeBackedFromBytes(data); + decodeValue(data: Uint8Array): BeaconStateAllForks { + return getStateTypeFromBytes(this.config, data).deserializeToViewDU(data); } // Handle key as slot - async put(key: Slot, value: TreeBacked): Promise { + async put(key: Slot, value: BeaconStateAllForks): Promise { await Promise.all([super.put(key, value), storeRootIndex(this.db, key, value.hashTreeRoot())]); } - getId(state: TreeBacked): Epoch { + getId(state: BeaconStateAllForks): Epoch { return state.slot; } - decodeKey(data: Buffer): number { + decodeKey(data: Uint8Array): number { return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be"); } // Index Root -> Slot - async getByRoot(stateRoot: Root): Promise | null> { + async getByRoot(stateRoot: Root): Promise { const slot = await this.getSlotByRoot(stateRoot); if (slot !== null && Number.isInteger(slot)) { return this.get(slot); diff --git a/packages/lodestar/src/db/repositories/stateArchiveIndex.ts b/packages/lodestar/src/db/repositories/stateArchiveIndex.ts index adf0838c9b8e..62e585be79ce 100644 --- a/packages/lodestar/src/db/repositories/stateArchiveIndex.ts +++ b/packages/lodestar/src/db/repositories/stateArchiveIndex.ts @@ -7,5 +7,5 @@ export function storeRootIndex(db: Db, slot: Slot, stateRoot: Root): Promise): Promise { + async put(value: BeaconStateAllForks): Promise { this.metrics?.dbWrites.labels({bucket: "phase0_preGenesisState"}).inc(); - await this.db.put(this.key, this.type().serialize(value) as Buffer); + await this.db.put(this.key, value.serialize()); } - async get(): Promise | null> { + async get(): Promise { this.metrics?.dbReads.labels({bucket: "phase0_preGenesisState"}).inc(); const value = await this.db.get(this.key); - return value ? this.type().createTreeBackedFromBytes(value) : null; + return value ? this.type.deserializeToViewDU(value) : null; } async delete(): Promise { await this.db.delete(this.key); } - - private type(): ContainerType { - return this.config.getForkTypes(GENESIS_SLOT).BeaconState; - } } diff --git a/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts b/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts index 2f2af759eee0..1cd25a861dd0 100644 --- a/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts +++ b/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts @@ -1,26 +1,26 @@ -import {NumberUintType} from "@chainsafe/ssz"; +import {UintNumberType} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Db, Bucket, IDbMetrics} from "@chainsafe/lodestar-db"; export class PreGenesisStateLastProcessedBlock { private readonly bucket: Bucket; - private readonly type: NumberUintType; + private readonly type: UintNumberType; private readonly db: Db; - private readonly key: Buffer; + private readonly key: Uint8Array; private readonly metrics?: IDbMetrics; constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { this.db = db; - this.type = ssz.Number64; + this.type = ssz.UintNum64; this.bucket = Bucket.phase0_preGenesisStateLastProcessedBlock; - this.key = Buffer.from(new Uint8Array([this.bucket])); + this.key = new Uint8Array([this.bucket]); this.metrics = metrics; } async put(value: number): Promise { this.metrics?.dbWrites.labels({bucket: "phase0_preGenesisStateLastProcessedBlock"}).inc(); - await this.db.put(this.key, this.type.serialize(value) as Buffer); + await this.db.put(this.key, this.type.serialize(value)); } async get(): Promise { diff --git a/packages/lodestar/src/eth1/eth1DepositDataTracker.ts b/packages/lodestar/src/eth1/eth1DepositDataTracker.ts index d35be427f43b..6e8da33c3c44 100644 --- a/packages/lodestar/src/eth1/eth1DepositDataTracker.ts +++ b/packages/lodestar/src/eth1/eth1DepositDataTracker.ts @@ -1,6 +1,6 @@ import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, allForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {ErrorAborted, ILogger, isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; import {AbortSignal} from "@chainsafe/abort-controller"; import {IBeaconDb} from "../db"; @@ -72,7 +72,7 @@ export class Eth1DepositDataTracker { /** * Return eth1Data and deposits ready for block production for a given state */ - async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { + async getEth1DataAndDeposits(state: BeaconStateAllForks): Promise { const eth1Data = await this.getEth1Data(state); const deposits = await this.getDeposits(state, eth1Data); return {eth1Data, deposits}; @@ -82,7 +82,7 @@ export class Eth1DepositDataTracker { * Returns an eth1Data vote for a given state. * Requires internal caches to be updated regularly to return good results */ - private async getEth1Data(state: allForks.BeaconState): Promise { + private async getEth1Data(state: BeaconStateAllForks): Promise { try { const eth1VotesToConsider = await getEth1VotesToConsider( this.config, @@ -101,10 +101,7 @@ export class Eth1DepositDataTracker { * Returns deposits to be included for a given state and eth1Data vote. * Requires internal caches to be updated regularly to return good results */ - private async getDeposits( - state: CachedBeaconStateAllForks, - eth1DataVote: phase0.Eth1Data - ): Promise { + private async getDeposits(state: BeaconStateAllForks, eth1DataVote: phase0.Eth1Data): Promise { // No new deposits have to be included, continue if (eth1DataVote.depositCount === state.eth1DepositIndex) { return []; @@ -112,7 +109,7 @@ export class Eth1DepositDataTracker { // TODO: Review if this is optimal // Convert to view first to hash once and compare hashes - const eth1DataVoteView = ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataVote); + const eth1DataVoteView = ssz.phase0.Eth1Data.toViewDU(eth1DataVote); // Eth1 data may change due to the vote included in this block const newEth1Data = allForks.becomesNewEth1Data(state, eth1DataVoteView) ? eth1DataVoteView : state.eth1Data; diff --git a/packages/lodestar/src/eth1/index.ts b/packages/lodestar/src/eth1/index.ts index d297894becc1..96f0f4504e7e 100644 --- a/packages/lodestar/src/eth1/index.ts +++ b/packages/lodestar/src/eth1/index.ts @@ -1,9 +1,10 @@ import { + BeaconStateAllForks, CachedBeaconStateAllForks, computeEpochAtSlot, getCurrentSlot, } from "@chainsafe/lodestar-beacon-state-transition"; -import {allForks, Root} from "@chainsafe/lodestar-types"; +import {Root} from "@chainsafe/lodestar-types"; import {bellatrix} from "@chainsafe/lodestar-beacon-state-transition"; import {fromHexString} from "@chainsafe/ssz"; import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock} from "./interface"; @@ -48,7 +49,7 @@ export {IEth1ForBlockProduction, IEth1Provider, Eth1Provider}; export function initializeEth1ForBlockProduction( opts: Eth1Options, modules: Pick, - anchorState: allForks.BeaconState + anchorState: BeaconStateAllForks ): IEth1ForBlockProduction { if (opts.enabled) { return new Eth1ForBlockProduction(opts, { diff --git a/packages/lodestar/src/eth1/provider/utils.ts b/packages/lodestar/src/eth1/provider/utils.ts index d18c057122f1..829766114436 100644 --- a/packages/lodestar/src/eth1/provider/utils.ts +++ b/packages/lodestar/src/eth1/provider/utils.ts @@ -1,6 +1,6 @@ import {RootHex} from "@chainsafe/lodestar-types"; import {bytesToBigInt, bigIntToBytes} from "@chainsafe/lodestar-utils"; -import {ByteVector, fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ErrorParseJson} from "./jsonRpcHttpClient"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -25,7 +25,7 @@ export function isJsonRpcTruncatedError(error: Error): boolean { ); } -export function bytesToHex(bytes: Uint8Array | ByteVector): string { +export function bytesToHex(bytes: Uint8Array): string { // Handle special case in Ethereum hex formating where hex values may include a single letter // 0x0, 0x1 are valid values if (bytes.length === 1 && bytes[0] <= 0xf) { @@ -82,7 +82,7 @@ export function quantityToBytes(hex: QUANTITY): Uint8Array { * QUANTITY as defined in ethereum execution layer JSON RPC https://eth.wiki/json-rpc/API. * Compress a 32 ByteVector into a QUANTITY */ -export function bytesToQuantity(bytes: Uint8Array | ByteVector): QUANTITY { +export function bytesToQuantity(bytes: Uint8Array): QUANTITY { const bn = bytesToBigInt(bytes as Uint8Array, "le"); return numToQuantity(bn); } @@ -99,7 +99,7 @@ export function bytesToQuantity(bytes: Uint8Array | ByteVector): QUANTITY { * - WRONG: 0xf0f0f (must be even number of digits) * - WRONG: 004200 (must be prefixed 0x) */ -export function bytesToData(bytes: Uint8Array | ByteVector): DATA { +export function bytesToData(bytes: Uint8Array): DATA { return toHexString(bytes); } diff --git a/packages/lodestar/src/eth1/utils/depositContract.ts b/packages/lodestar/src/eth1/utils/depositContract.ts index 140e2c18e574..090727338319 100644 --- a/packages/lodestar/src/eth1/utils/depositContract.ts +++ b/packages/lodestar/src/eth1/utils/depositContract.ts @@ -25,12 +25,17 @@ export function parseDepositLog(log: {blockNumber: number; data: string; topics: if (values === undefined) throw Error(`DepositEvent at ${log.blockNumber} has no values`); return { blockNumber: log.blockNumber, - index: ssz.Number64.deserialize(fromHexString(values.index)), + index: parseHexNumLittleEndian(values.index), depositData: { pubkey: fromHexString(values.pubkey), withdrawalCredentials: fromHexString(values.withdrawal_credentials), - amount: ssz.Number64.deserialize(fromHexString(values.amount)), + amount: parseHexNumLittleEndian(values.amount), signature: fromHexString(values.signature), }, }; } + +function parseHexNumLittleEndian(hex: string): number { + // Can't use parseInt() because amount is a hex string in little endian + return ssz.UintNum64.deserialize(fromHexString(hex)); +} diff --git a/packages/lodestar/src/eth1/utils/deposits.ts b/packages/lodestar/src/eth1/utils/deposits.ts index 9cae58f883b9..2fb3821b3a0f 100644 --- a/packages/lodestar/src/eth1/utils/deposits.ts +++ b/packages/lodestar/src/eth1/utils/deposits.ts @@ -1,15 +1,17 @@ import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; -import {Root, phase0, allForks, ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked, List, toHexString} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {toHexString} from "@chainsafe/ssz"; import {IFilterOptions} from "@chainsafe/lodestar-db"; -import {getTreeAtIndex} from "../../util/tree"; import {Eth1Error, Eth1ErrorCode} from "../errors"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; export type DepositGetter = (indexRange: IFilterOptions, eth1Data: phase0.Eth1Data) => Promise; export async function getDeposits( // eth1_deposit_index represents the next deposit index to be added - state: allForks.BeaconState, + state: BeaconStateAllForks, eth1Data: phase0.Eth1Data, depositsGetter: DepositGetter ): Promise { @@ -38,13 +40,13 @@ export async function getDeposits( export function getDepositsWithProofs( depositEvents: phase0.DepositEvent[], - depositRootTree: TreeBacked>, + depositRootTree: DepositTree, eth1Data: phase0.Eth1Data ): phase0.Deposit[] { // Get tree at this particular depositCount to compute correct proofs - const treeAtDepositCount = getTreeAtIndex(depositRootTree, eth1Data.depositCount - 1); + const viewAtDepositCount = depositRootTree.sliceTo(eth1Data.depositCount - 1); - const depositRoot = treeAtDepositCount.hashTreeRoot(); + const depositRoot = viewAtDepositCount.hashTreeRoot(); if (!ssz.Root.equals(depositRoot, eth1Data.depositRoot)) { throw new Eth1Error({ @@ -54,8 +56,12 @@ export function getDepositsWithProofs( }); } + // Already commited for .hashTreeRoot() + const treeAtDepositCount = new Tree(viewAtDepositCount.node); + const depositTreeDepth = viewAtDepositCount.type.depth; + return depositEvents.map((log) => ({ - proof: treeAtDepositCount.tree.getSingleProof(treeAtDepositCount.type.getPropertyGindex(log.index)), + proof: treeAtDepositCount.getSingleProof(toGindex(depositTreeDepth, BigInt(log.index))), data: log.depositData, })); } diff --git a/packages/lodestar/src/eth1/utils/eth1Data.ts b/packages/lodestar/src/eth1/utils/eth1Data.ts index 5eb1088cf373..76b021745cb6 100644 --- a/packages/lodestar/src/eth1/utils/eth1Data.ts +++ b/packages/lodestar/src/eth1/utils/eth1Data.ts @@ -1,8 +1,7 @@ import {Root, phase0} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {getTreeAtIndex} from "../../util/tree"; import {binarySearchLte} from "../../util/binarySearch"; import {Eth1Error, Eth1ErrorCode} from "../errors"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; import {Eth1Block} from "../interface"; type BlockNumber = number; @@ -14,7 +13,7 @@ type BlockNumber = number; export async function getEth1DataForBlocks( blocks: Eth1Block[], depositDescendingStream: AsyncIterable, - depositRootTree: TreeBacked>, + depositRootTree: DepositTree, lastProcessedDepositBlockNumber: BlockNumber | null ): Promise<(phase0.Eth1Data & Eth1Block)[]> { // Exclude blocks for which there is no valid eth1 data deposit @@ -80,10 +79,7 @@ export async function getDepositsByBlockNumber( /** * Precompute a map of depositCount => depositRoot from a depositRootTree filled beforehand */ -export function getDepositRootByDepositCount( - depositCounts: number[], - depositRootTree: TreeBacked> -): Map { +export function getDepositRootByDepositCount(depositCounts: number[], depositRootTree: DepositTree): Map { // Unique + sort numerically in descending order depositCounts = [...new Set(depositCounts)].sort((a, b) => b - a); @@ -97,7 +93,7 @@ export function getDepositRootByDepositCount( const depositRootByDepositCount = new Map(); for (const depositCount of depositCounts) { - depositRootTree = getTreeAtIndex(depositRootTree, depositCount - 1); + depositRootTree = depositRootTree.sliceTo(depositCount - 1); depositRootByDepositCount.set(depositCount, depositRootTree.hashTreeRoot()); } return depositRootByDepositCount; diff --git a/packages/lodestar/src/eth1/utils/eth1Vote.ts b/packages/lodestar/src/eth1/utils/eth1Vote.ts index fd4ed82f661f..4fdb1b2993e8 100644 --- a/packages/lodestar/src/eth1/utils/eth1Vote.ts +++ b/packages/lodestar/src/eth1/utils/eth1Vote.ts @@ -1,8 +1,8 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0, RootHex} from "@chainsafe/lodestar-types"; -import {computeTimeAtSlot} from "@chainsafe/lodestar-beacon-state-transition"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {phase0, RootHex} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks, computeTimeAtSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {toHex} from "@chainsafe/lodestar-utils"; export type Eth1DataGetter = ({ timestampRange, @@ -12,7 +12,7 @@ export type Eth1DataGetter = ({ export async function getEth1VotesToConsider( config: IChainForkConfig, - state: allForks.BeaconState, + state: BeaconStateAllForks, eth1DataGetter: Eth1DataGetter ): Promise { const periodStart = votingPeriodStartTime(config, state); @@ -33,7 +33,7 @@ export async function getEth1VotesToConsider( ).filter((eth1Data) => eth1Data.depositCount >= state.eth1Data.depositCount); } -export function pickEth1Vote(state: allForks.BeaconState, votesToConsider: phase0.Eth1Data[]): phase0.Eth1Data { +export function pickEth1Vote(state: BeaconStateAllForks, votesToConsider: phase0.Eth1Data[]): phase0.Eth1Data { const votesToConsiderKeys = new Set(); for (const eth1Data of votesToConsider) { votesToConsiderKeys.add(getEth1DataKey(eth1Data)); @@ -48,7 +48,7 @@ export function pickEth1Vote(state: allForks.BeaconState, votesToConsider: phase // However `votesToConsider` is an array of values since those are read from DB. // TODO: Optimize cache of known votes, to prevent re-hashing stored values. // Note: for low validator counts it's not very important, since this runs once per proposal - const eth1DataVotes = Array.from(readonlyValues(state.eth1DataVotes)); + const eth1DataVotes = state.eth1DataVotes.getAllReadonly(); for (const eth1DataVote of eth1DataVotes) { const rootHex = getEth1DataKey(eth1DataVote); @@ -126,13 +126,13 @@ function getEth1DataKey(eth1Data: phase0.Eth1Data): string { } /** - * Serialize eth1Data types to a unique string ID. It is only used for comparison. + * Returns the array of keys with max value. May return 0, 1 or more keys */ export function fastSerializeEth1Data(eth1Data: phase0.Eth1Data): string { - return toHexString(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHexString(eth1Data.depositRoot); + return toHex(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHex(eth1Data.depositRoot); } -export function votingPeriodStartTime(config: IChainForkConfig, state: allForks.BeaconState): number { +export function votingPeriodStartTime(config: IChainForkConfig, state: BeaconStateAllForks): number { const eth1VotingPeriodStartSlot = state.slot - (state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)); return computeTimeAtSlot(config, eth1VotingPeriodStartSlot, state.genesisTime); } diff --git a/packages/lodestar/src/executionEngine/interface.ts b/packages/lodestar/src/executionEngine/interface.ts index 77859afe51f6..e50a64baa5a1 100644 --- a/packages/lodestar/src/executionEngine/interface.ts +++ b/packages/lodestar/src/executionEngine/interface.ts @@ -1,5 +1,4 @@ import {bellatrix, Root, RootHex} from "@chainsafe/lodestar-types"; -import {ByteVector} from "@chainsafe/ssz"; import {DATA, QUANTITY} from "../eth1/provider/utils"; // An execution engine can produce a payload id anywhere the the uint64 range @@ -50,8 +49,8 @@ export type ForkChoiceUpdateStatus = export type PayloadAttributes = { timestamp: number; - random: Uint8Array | ByteVector; - suggestedFeeRecipient: Uint8Array | ByteVector; + random: Uint8Array; + suggestedFeeRecipient: Uint8Array; }; export type ApiPayloadAttributes = { diff --git a/packages/lodestar/src/executionEngine/mock.ts b/packages/lodestar/src/executionEngine/mock.ts index c7d5ff8ba390..ede6b0bbf3f3 100644 --- a/packages/lodestar/src/executionEngine/mock.ts +++ b/packages/lodestar/src/executionEngine/mock.ts @@ -132,7 +132,7 @@ export class ExecutionEngineMock implements IExecutionEngine { * 3. Client software MAY stop the corresponding building process after serving this call. */ async getPayload(payloadId: PayloadId): Promise { - const payloadIdNbr = Number(payloadId); + const payloadIdNbr = parseInt(payloadId, 10); const payload = this.preparingPayloads.get(payloadIdNbr); if (!payload) { throw Error(`Unknown payloadId ${payloadId}`); diff --git a/packages/lodestar/src/metrics/metrics.ts b/packages/lodestar/src/metrics/metrics.ts index 7a5a1c7b60ed..6fbef215de82 100644 --- a/packages/lodestar/src/metrics/metrics.ts +++ b/packages/lodestar/src/metrics/metrics.ts @@ -1,9 +1,8 @@ /** * @module metrics */ -import {getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks} from "@chainsafe/lodestar-types"; import {collectDefaultMetrics, Counter, Registry} from "prom-client"; import gcStats from "prometheus-gc-stats"; import {DbMetricLabels, IDbMetrics} from "@chainsafe/lodestar-db"; @@ -18,7 +17,7 @@ export type IMetrics = IBeaconMetrics & ILodestarMetrics & IValidatorMonitor & { export function createMetrics( opts: IMetricsOptions, config: IChainForkConfig, - anchorState: allForks.BeaconState, + anchorState: BeaconStateAllForks, registries: Registry[] = [] ): IMetrics { const register = new RegistryMetricCreator(); diff --git a/packages/lodestar/src/metrics/metrics/lodestar.ts b/packages/lodestar/src/metrics/metrics/lodestar.ts index e02783316ec3..d5e4be15526d 100644 --- a/packages/lodestar/src/metrics/metrics/lodestar.ts +++ b/packages/lodestar/src/metrics/metrics/lodestar.ts @@ -1,4 +1,4 @@ -import {allForks} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {RegistryMetricCreator} from "../utils/registryMetricCreator"; import {IMetricsOptions} from "../options"; @@ -11,7 +11,7 @@ export type ILodestarMetrics = ReturnType; export function createLodestarMetrics( register: RegistryMetricCreator, metadata: IMetricsOptions["metadata"], - anchorState?: allForks.BeaconState + anchorState?: BeaconStateAllForks ) { if (metadata) { register.static<"semver" | "branch" | "commit" | "version" | "network">({ diff --git a/packages/lodestar/src/network/gossip/validation/index.ts b/packages/lodestar/src/network/gossip/validation/index.ts index 96b176b1531a..a04f30a75d81 100644 --- a/packages/lodestar/src/network/gossip/validation/index.ts +++ b/packages/lodestar/src/network/gossip/validation/index.ts @@ -1,8 +1,7 @@ import {ERR_TOPIC_VALIDATOR_IGNORE, ERR_TOPIC_VALIDATOR_REJECT} from "libp2p-gossipsub/src/constants"; import {AbortSignal} from "@chainsafe/abort-controller"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Json} from "@chainsafe/ssz"; -import {ILogger, mapValues} from "@chainsafe/lodestar-utils"; +import {Context, ILogger, mapValues} from "@chainsafe/lodestar-utils"; import {IMetrics} from "../../../metrics"; import {getGossipSSZType} from "../topic"; import { @@ -86,11 +85,7 @@ function getGossipValidatorFn( try { const sszType = getGossipSSZType(topic); const messageData = decodeMessageData(encoding, gossipMsg.data, uncompressCache); - gossipObject = - // TODO: Review if it's really necessary to deserialize this as TreeBacked - topic.type === GossipType.beacon_block || topic.type === GossipType.beacon_aggregate_and_proof - ? sszType.createTreeBackedFromBytes(messageData) - : sszType.deserialize(messageData); + gossipObject = sszType.deserialize(messageData); } catch (e) { // TODO: Log the error or do something better with it throw new GossipActionError(GossipAction.REJECT, {code: (e as Error).message}); @@ -109,7 +104,7 @@ function getGossipValidatorFn( // If the gossipObject was deserialized include its short metadata with the error data const metadata = gossipObject && getGossipObjectAcceptMetadata(config, gossipObject, topic); - const errorData = (typeof e.type === "object" && metadata ? {...metadata, ...e.type} : e.type) as Json; + const errorData = (typeof e.type === "object" && metadata ? {...metadata, ...e.type} : e.type) as Context; switch (e.action) { case GossipAction.IGNORE: diff --git a/packages/lodestar/src/network/metadata.ts b/packages/lodestar/src/network/metadata.ts index cd153ef6cef5..e90ef9268663 100644 --- a/packages/lodestar/src/network/metadata.ts +++ b/packages/lodestar/src/network/metadata.ts @@ -1,5 +1,5 @@ import {ENR} from "@chainsafe/discv5"; -import {BitVector, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {altair, Epoch, phase0, ssz} from "@chainsafe/lodestar-types"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; @@ -45,7 +45,7 @@ export class MetadataController { this.config = modules.config; this.chain = modules.chain; this.logger = modules.logger; - this._metadata = opts.metadata || ssz.altair.Metadata.defaultValue(); + this._metadata = opts.metadata || ssz.altair.Metadata.defaultValue; } start(enr: ENR | undefined, currentFork: ForkName): void { @@ -69,22 +69,22 @@ export class MetadataController { return this._metadata.seqNumber; } - get syncnets(): BitVector { + get syncnets(): BitArray { return this._metadata.syncnets; } - set syncnets(syncnets: BitVector) { + set syncnets(syncnets: BitArray) { if (this.enr) { this.enr.set(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(syncnets)); } this._metadata.syncnets = syncnets; } - get attnets(): BitVector { + get attnets(): BitArray { return this._metadata.attnets; } - set attnets(attnets: BitVector) { + set attnets(attnets: BitArray) { if (this.enr) { this.enr.set(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(attnets)); } diff --git a/packages/lodestar/src/network/network.ts b/packages/lodestar/src/network/network.ts index 20865a972e5f..3de72880dbef 100644 --- a/packages/lodestar/src/network/network.ts +++ b/packages/lodestar/src/network/network.ts @@ -95,7 +95,7 @@ export class Network implements INetwork { signal, gossipHandlers: gossipHandlers ?? getGossipHandlers({chain, config, logger, network: this, metrics}, opts), eth2Context: { - activeValidatorCount: chain.getHeadState().currentShuffling.activeIndices.length, + activeValidatorCount: chain.getHeadState().epochCtx.currentShuffling.activeIndices.length, currentSlot: this.clock.currentSlot, currentEpoch: this.clock.currentEpoch, }, diff --git a/packages/lodestar/src/network/peers/metastore.ts b/packages/lodestar/src/network/peers/metastore.ts index bcafc47c0124..bd537bfbba97 100644 --- a/packages/lodestar/src/network/peers/metastore.ts +++ b/packages/lodestar/src/network/peers/metastore.ts @@ -39,7 +39,7 @@ export class Libp2pPeerMetadataStore implements IPeerMetadataStore { constructor(metabook: MetadataBook) { this.metabook = metabook; - const number64Serdes = typeToSerdes(ssz.Number64); + const number64Serdes = typeToSerdes(ssz.UintNum64); const metadataV2Serdes = typeToSerdes(ssz.altair.Metadata); const stringSerdes: BucketSerdes = { serialize: (v) => Buffer.from(v, "utf8"), diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index 358b8456fd56..089e8f530248 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -22,6 +22,8 @@ import { renderIrrelevantPeerType, } from "./utils"; import {SubnetType} from "../metadata"; +import {BitArray} from "@chainsafe/ssz"; +import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; /** heartbeat performs regular updates such as updating reputations and performing discovery requests */ const HEARTBEAT_INTERVAL_MS = 30 * 1000; @@ -260,7 +262,7 @@ export class PeerManager { // Trust that the peer always sends the latest metadata (From Lighthouse) this.peerMetadata.metadata.set(peer, { ...metadata, - syncnets: (metadata as Partial).syncnets || [], + syncnets: (metadata as Partial).syncnets ?? BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT), }); } @@ -376,8 +378,8 @@ export class PeerManager { const {peersToDisconnect, peersToConnect, attnetQueries, syncnetQueries} = prioritizePeers( connectedHealthyPeers.map((peer) => ({ id: peer, - attnets: this.peerMetadata.metadata.get(peer)?.attnets ?? [], - syncnets: this.peerMetadata.metadata.get(peer)?.syncnets ?? [], + attnets: this.peerMetadata.metadata.get(peer)?.attnets ?? null, + syncnets: this.peerMetadata.metadata.get(peer)?.syncnets ?? null, score: this.peerRpcScores.getScore(peer), })), // Collect subnets which we need peers for in the current slot diff --git a/packages/lodestar/src/network/peers/utils/assertPeerRelevance.ts b/packages/lodestar/src/network/peers/utils/assertPeerRelevance.ts index d907e0ac35de..6a17a1abb5a2 100644 --- a/packages/lodestar/src/network/peers/utils/assertPeerRelevance.ts +++ b/packages/lodestar/src/network/peers/utils/assertPeerRelevance.ts @@ -83,7 +83,7 @@ export function assertPeerRelevance(remote: phase0.Status, chain: IBeaconChain): } export function isZeroRoot(root: Root): boolean { - const ZERO_ROOT = ssz.Root.defaultValue(); + const ZERO_ROOT = ssz.Root.defaultValue; return ssz.Root.equals(root, ZERO_ROOT); } diff --git a/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts b/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts index 75ed687af296..fc244f4e1a0e 100644 --- a/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts +++ b/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts @@ -1,4 +1,5 @@ -import {getUint8ByteToBitBooleanArray, newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; +import {getUint8ByteToBitBooleanArray} from "@chainsafe/ssz"; +import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; export const zeroAttnets = newFilledArray(ATTESTATION_SUBNET_COUNT, false); diff --git a/packages/lodestar/src/network/peers/utils/prioritizePeers.ts b/packages/lodestar/src/network/peers/utils/prioritizePeers.ts index c3bbf403e877..5cd43f5ff797 100644 --- a/packages/lodestar/src/network/peers/utils/prioritizePeers.ts +++ b/packages/lodestar/src/network/peers/utils/prioritizePeers.ts @@ -24,7 +24,12 @@ type SubnetDiscvQuery = {subnet: number; toSlot: number; maxPeersToDiscover: num * - Prioritize peers with good score */ export function prioritizePeers( - connectedPeers: {id: PeerId; attnets: phase0.AttestationSubnets; syncnets: altair.SyncSubnets; score: number}[], + connectedPeers: { + id: PeerId; + attnets: phase0.AttestationSubnets | null; + syncnets: altair.SyncSubnets | null; + score: number; + }[], activeAttnets: RequestedSubnet[], activeSyncnets: RequestedSubnet[], {targetPeers, maxPeers}: {targetPeers: number; maxPeers: number} @@ -56,7 +61,9 @@ export function prioritizePeers( for (const peer of connectedPeers) { let hasDuty = false; for (const {subnet} of subnets) { - if (peer[subnetKey][subnet]) { + const subnetBitArray = peer[subnetKey]; + // TODO: Review performance + if (subnetBitArray && subnetBitArray.get(subnet) === true) { hasDuty = true; peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0)); } diff --git a/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts b/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts index 2fe758be1b6a..51fe9a2e6b2a 100644 --- a/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts +++ b/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts @@ -8,7 +8,6 @@ import { Protocol, IncomingResponseBody, ContextBytesType, - deserializeToTreeByMethod, contextBytesTypeByProtocol, getResponseSzzTypeByMethod, CONTEXT_BYTES_FORK_DIGEST_LENGTH, @@ -34,7 +33,6 @@ export function responseDecode( protocol: Protocol ): (source: AsyncIterable) => AsyncGenerator { return async function* responseDecodeSink(source) { - const deserializeToTree = deserializeToTreeByMethod[protocol.method]; const contextBytesType = contextBytesTypeByProtocol(protocol); const bufferedSource = new BufferedSource(source as AsyncGenerator); @@ -58,7 +56,7 @@ export function responseDecode( const forkName = await readForkName(forkDigestContext, bufferedSource, contextBytesType); const type = getResponseSzzTypeByMethod(protocol, forkName); - yield await readEncodedPayload(bufferedSource, protocol.encoding, type, {deserializeToTree}); + yield await readEncodedPayload(bufferedSource, protocol.encoding, type); } }; } diff --git a/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts b/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts index 8505978e851c..2bc76647a67d 100644 --- a/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts +++ b/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts @@ -6,7 +6,7 @@ import { OutgoingSerializer, } from "../types"; import {BufferedSource} from "../utils"; -import {readSszSnappyPayload, ISszSnappyOptions} from "./sszSnappy/decode"; +import {readSszSnappyPayload} from "./sszSnappy/decode"; import {writeSszSnappyPayload} from "./sszSnappy/encode"; // For more info about eth2 request/response encoding strategies, see: @@ -23,12 +23,11 @@ import {writeSszSnappyPayload} from "./sszSnappy/encode"; export async function readEncodedPayload( bufferedSource: BufferedSource, encoding: Encoding, - type: RequestOrResponseType, - options?: ISszSnappyOptions + type: RequestOrResponseType ): Promise { switch (encoding) { case Encoding.SSZ_SNAPPY: - return await readSszSnappyPayload(bufferedSource, type, options); + return await readSszSnappyPayload(bufferedSource, type); default: throw Error("Unsupported encoding"); diff --git a/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts b/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts index cb42bc28d58a..7540a32fe088 100644 --- a/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts +++ b/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts @@ -1,6 +1,5 @@ import BufferList from "bl"; import varint from "varint"; -import {CompositeType} from "@chainsafe/ssz"; import {MAX_VARINT_BYTES} from "../../../../constants"; import {BufferedSource} from "../../utils"; import {RequestOrResponseType, RequestOrIncomingResponseBody} from "../../types"; @@ -8,10 +7,6 @@ import {SnappyFramesUncompress} from "./snappyFrames/uncompress"; import {maxEncodedLen} from "./utils"; import {SszSnappyError, SszSnappyErrorCode} from "./errors"; -export interface ISszSnappyOptions { - deserializeToTree?: boolean; -} - /** * ssz_snappy encoding strategy reader. * Consumes a stream source to read encoded header and payload as defined in the spec: @@ -21,13 +16,12 @@ export interface ISszSnappyOptions { */ export async function readSszSnappyPayload( bufferedSource: BufferedSource, - type: RequestOrResponseType, - options?: ISszSnappyOptions + type: RequestOrResponseType ): Promise { const sszDataLength = await readSszSnappyHeader(bufferedSource, type); const bytes = await readSszSnappyBody(bufferedSource, sszDataLength); - return deserializeSszBody(bytes, type, options); + return deserializeSszBody(bytes, type); } /** @@ -61,8 +55,8 @@ async function readSszSnappyHeader(bufferedSource: BufferedSource, type: Request buffer.consume(varintBytes); // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. - const minSize = type.getMinSerializedLength(); - const maxSize = type.getMaxSerializedLength(); + const minSize = type.minSize; + const maxSize = type.maxSize; if (sszDataLength < minSize) { throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); } @@ -130,18 +124,9 @@ async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: * Deseralizes SSZ body. * `isSszTree` option allows the SignedBeaconBlock type to be deserialized as a tree */ -function deserializeSszBody( - bytes: Buffer, - type: RequestOrResponseType, - options?: ISszSnappyOptions -): T { +function deserializeSszBody(bytes: Buffer, type: RequestOrResponseType): T { try { - if (options?.deserializeToTree) { - const typeTree = (type as unknown) as CompositeType>; - return (typeTree.createTreeBackedFromBytes(bytes) as unknown) as T; - } else { - return type.deserialize(bytes) as T; - } + return type.deserialize(bytes) as T; } catch (e) { throw new SszSnappyError({code: SszSnappyErrorCode.DESERIALIZE_ERROR, deserializeError: e as Error}); } diff --git a/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts b/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts index 315954cbe3e5..4091780464fa 100644 --- a/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts +++ b/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts @@ -10,7 +10,7 @@ export async function* onBeaconBlocksByRoot( db: IBeaconDb ): AsyncIterable { for (const blockRoot of requestBody) { - const root = blockRoot.valueOf() as Uint8Array; + const root = blockRoot; const summary = chain.forkChoice.getBlock(root); let blockBytes: Uint8Array | null = null; diff --git a/packages/lodestar/src/network/reqresp/types.ts b/packages/lodestar/src/network/reqresp/types.ts index 8d7b94cf887e..951e789eea30 100644 --- a/packages/lodestar/src/network/reqresp/types.ts +++ b/packages/lodestar/src/network/reqresp/types.ts @@ -54,16 +54,6 @@ export const isSingleResponseChunkByMethod: {[K in Method]: boolean} = { [Method.BeaconBlocksByRoot]: false, }; -/** Deserialize some types to TreeBacked directly for more efficient hashing */ -export const deserializeToTreeByMethod: {[K in Method]: boolean} = { - [Method.Status]: false, - [Method.Goodbye]: false, - [Method.Ping]: false, - [Method.Metadata]: false, - [Method.BeaconBlocksByRange]: true, - [Method.BeaconBlocksByRoot]: true, -}; - export const CONTEXT_BYTES_FORK_DIGEST_LENGTH = 4; export enum ContextBytesType { /** 0 bytes chunk, can be ignored */ diff --git a/packages/lodestar/src/network/subnets/attnetsService.ts b/packages/lodestar/src/network/subnets/attnetsService.ts index 07add024c880..7d19f4b77647 100644 --- a/packages/lodestar/src/network/subnets/attnetsService.ts +++ b/packages/lodestar/src/network/subnets/attnetsService.ts @@ -248,9 +248,9 @@ export class AttnetsService implements IAttnetsService { /** Update ENR */ private updateMetadata(): void { - const subnets = ssz.phase0.AttestationSubnets.defaultValue(); + const subnets = ssz.phase0.AttestationSubnets.defaultValue; for (const subnet of this.subscriptionsRandom.getAll()) { - subnets[subnet] = true; + subnets.set(subnet, true); } // Only update metadata if necessary, setting `metadata.[key]` triggers a write to disk diff --git a/packages/lodestar/src/network/subnets/syncnetsService.ts b/packages/lodestar/src/network/subnets/syncnetsService.ts index 83cfcb1b831c..121875cbe065 100644 --- a/packages/lodestar/src/network/subnets/syncnetsService.ts +++ b/packages/lodestar/src/network/subnets/syncnetsService.ts @@ -101,9 +101,9 @@ export class SyncnetsService implements ISubnetsService { /** Update ENR */ private updateMetadata(): void { - const subnets = ssz.altair.SyncSubnets.defaultValue(); + const subnets = ssz.altair.SyncSubnets.defaultValue; for (const subnet of this.subscriptionsCommittee.getAll()) { - subnets[subnet] = true; + subnets.set(subnet, true); } // Only update metadata if necessary, setting `metadata.[key]` triggers a write to disk diff --git a/packages/lodestar/src/node/nodejs.ts b/packages/lodestar/src/node/nodejs.ts index 3f7fe97c758a..129b25d34b2b 100644 --- a/packages/lodestar/src/node/nodejs.ts +++ b/packages/lodestar/src/node/nodejs.ts @@ -6,9 +6,8 @@ import {AbortController} from "@chainsafe/abort-controller"; import LibP2p from "libp2p"; import {Registry} from "prom-client"; -import {TreeBacked} from "@chainsafe/ssz"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0} from "@chainsafe/lodestar-types"; +import {phase0} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; import {Api} from "@chainsafe/lodestar-api"; @@ -23,6 +22,7 @@ import {initializeExecutionEngine} from "../executionEngine"; import {initializeEth1ForBlockProduction} from "../eth1"; import {IBeaconNodeOptions} from "./options"; import {runNodeNotifier} from "./notifier"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; export * from "./options"; @@ -47,7 +47,7 @@ export interface IBeaconNodeInitModules { db: IBeaconDb; logger: ILogger; libp2p: LibP2p; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; metricsRegistries?: Registry[]; } diff --git a/packages/lodestar/src/node/notifier.ts b/packages/lodestar/src/node/notifier.ts index a4416d7e2cd2..dec46148c12e 100644 --- a/packages/lodestar/src/node/notifier.ts +++ b/packages/lodestar/src/node/notifier.ts @@ -53,7 +53,7 @@ export async function runNodeNotifier({ const headInfo = chain.forkChoice.getHead(); const headState = chain.getHeadState(); const finalizedEpoch = headState.finalizedCheckpoint.epoch; - const finalizedRoot = headState.finalizedCheckpoint.root.valueOf() as Uint8Array; + const finalizedRoot = headState.finalizedCheckpoint.root; const headSlot = headInfo.slot; timeSeries.addPoint(headSlot, Date.now()); @@ -61,7 +61,7 @@ export async function runNodeNotifier({ const finalizedCheckpointRow = `finalized: ${prettyBytes(finalizedRoot)}:${finalizedEpoch}`; const headRow = `head: ${headInfo.slot} ${prettyBytes(headInfo.blockRoot)}`; const isMergeTransitionComplete = - bellatrix.isBellatrixStateType(headState) && bellatrix.isMergeTransitionComplete(headState); + bellatrix.isBellatrixCachedStateType(headState) && bellatrix.isMergeTransitionComplete(headState); const executionInfo = isMergeTransitionComplete ? [ `execution: ${headInfo.executionStatus.toLowerCase()}(${prettyBytes( @@ -132,7 +132,7 @@ export async function runNodeNotifier({ function timeToNextHalfSlot(config: IBeaconConfig, chain: IBeaconChain): number { const msPerSlot = config.SECONDS_PER_SLOT * 1000; - const msFromGenesis = Date.now() - chain.getGenesisTime() * 1000; + const msFromGenesis = Date.now() - chain.genesisTime * 1000; const msToNextSlot = msPerSlot - (msFromGenesis % msPerSlot); return msToNextSlot > msPerSlot / 2 ? msToNextSlot - msPerSlot / 2 : msToNextSlot + msPerSlot / 2; } diff --git a/packages/lodestar/src/node/utils/interop/deposits.ts b/packages/lodestar/src/node/utils/interop/deposits.ts index 8303e20e0744..08a2372ba2cb 100644 --- a/packages/lodestar/src/node/utils/interop/deposits.ts +++ b/packages/lodestar/src/node/utils/interop/deposits.ts @@ -1,5 +1,6 @@ -import {hash, TreeBacked, List} from "@chainsafe/ssz"; -import {phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {hash} from "@chainsafe/ssz"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import { computeDomain, @@ -8,16 +9,19 @@ import { ZERO_HASH, } from "@chainsafe/lodestar-beacon-state-transition"; import {BLS_WITHDRAWAL_PREFIX, DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; +import {DepositTree} from "../../../db/repositories/depositDataRoot"; /** * Compute and return deposit data from other validators. */ export function interopDeposits( config: IChainForkConfig, - depositDataRootList: TreeBacked>, + depositDataRootList: DepositTree, validatorCount: number ): phase0.Deposit[] { - const tree = depositDataRootList.tree; + depositDataRootList.commit(); + const depositTreeDepth = depositDataRootList.type.depth; + return interopSecretKeys(validatorCount).map((secretKey, i) => { const pubkey = secretKey.toPublicKey().toBytes(); // create DepositData @@ -32,8 +36,9 @@ export function interopDeposits( data.signature = secretKey.sign(signingRoot).toBytes(); // Add to merkle tree depositDataRootList.push(ssz.phase0.DepositData.hashTreeRoot(data)); + depositDataRootList.commit(); return { - proof: tree.getSingleProof(depositDataRootList.type.getPropertyGindex(i)), + proof: new Tree(depositDataRootList.node).getSingleProof(toGindex(depositTreeDepth, BigInt(i))), data, }; }); diff --git a/packages/lodestar/src/node/utils/interop/state.ts b/packages/lodestar/src/node/utils/interop/state.ts index ecb5f8ffc89b..c8c51dacc40a 100644 --- a/packages/lodestar/src/node/utils/interop/state.ts +++ b/packages/lodestar/src/node/utils/interop/state.ts @@ -1,8 +1,9 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; -import {allForks, Bytes32, Number64, phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {Bytes32, phase0, ssz, TimeSeconds} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {initializeBeaconStateFromEth1} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, initializeBeaconStateFromEth1} from "@chainsafe/lodestar-beacon-state-transition"; +import {createEmptyEpochContextImmutableData} from "@chainsafe/lodestar-beacon-state-transition"; import {GENESIS_BASE_FEE_PER_GAS, GENESIS_GAS_LIMIT} from "@chainsafe/lodestar-params"; +import {DepositTree} from "../../../db/repositories/depositDataRoot"; export const INTEROP_BLOCK_HASH = Buffer.alloc(32, "B"); export const INTEROP_TIMESTAMP = Math.pow(2, 40); @@ -10,7 +11,7 @@ export const INTEROP_TIMESTAMP = Math.pow(2, 40); export type InteropStateOpts = { genesisTime?: number; eth1BlockHash?: Bytes32; - eth1Timestamp?: Number64; + eth1Timestamp?: TimeSeconds; }; export function getInteropState( @@ -21,9 +22,9 @@ export function getInteropState( eth1Timestamp = INTEROP_TIMESTAMP, }: InteropStateOpts, deposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked> -): TreeBacked { - const latestPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked(); + fullDepositDataRootList?: DepositTree +): BeaconStateAllForks { + const latestPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU; // TODO: when having different test options, consider modifying these values latestPayloadHeader.blockHash = eth1BlockHash; latestPayloadHeader.timestamp = eth1Timestamp; @@ -32,6 +33,7 @@ export function getInteropState( latestPayloadHeader.baseFeePerGas = GENESIS_BASE_FEE_PER_GAS; const state = initializeBeaconStateFromEth1( config, + createEmptyEpochContextImmutableData(config, {genesisValidatorsRoot: Buffer.alloc(32, 0)}), eth1BlockHash, eth1Timestamp, deposits, diff --git a/packages/lodestar/src/node/utils/state.ts b/packages/lodestar/src/node/utils/state.ts index 1badbcbf4139..23f329e26f79 100644 --- a/packages/lodestar/src/node/utils/state.ts +++ b/packages/lodestar/src/node/utils/state.ts @@ -1,11 +1,9 @@ -import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {interopDeposits} from "./interop/deposits"; import {getInteropState, InteropStateOpts} from "./interop/state"; -import {mkdirSync, writeFileSync} from "node:fs"; -import {dirname} from "node:path"; import {IBeaconDb} from "../../db"; -import {TreeBacked} from "@chainsafe/ssz"; import {GENESIS_SLOT} from "../../constants"; export async function initDevState( @@ -13,27 +11,22 @@ export async function initDevState( db: IBeaconDb, validatorCount: number, interopStateOpts: InteropStateOpts -): Promise> { - const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultTreeBacked(), validatorCount); - await storeDeposits(config, db, deposits); +): Promise { + const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultViewDU, validatorCount); + await storeDeposits(db, deposits); const state = getInteropState( config, interopStateOpts, deposits, - await db.depositDataRoot.getTreeBacked(validatorCount - 1) + await db.depositDataRoot.getDepositRootTreeAtIndex(validatorCount - 1) ); - const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue(); - block.message.stateRoot = config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state); + const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue; + block.message.stateRoot = state.hashTreeRoot(); await db.blockArchive.add(block); return state; } -export function storeSSZState(config: IBeaconConfig, state: TreeBacked, path: string): void { - mkdirSync(dirname(path), {recursive: true}); - writeFileSync(path, config.getForkTypes(state.slot).BeaconState.serialize(state)); -} - -async function storeDeposits(config: IChainForkConfig, db: IBeaconDb, deposits: phase0.Deposit[]): Promise { +async function storeDeposits(db: IBeaconDb, deposits: phase0.Deposit[]): Promise { for (let i = 0; i < deposits.length; i++) { await Promise.all([ db.depositEvent.put(i, { diff --git a/packages/lodestar/src/sync/backfill/backfill.ts b/packages/lodestar/src/sync/backfill/backfill.ts index d22e2e14b536..789b833900fc 100644 --- a/packages/lodestar/src/sync/backfill/backfill.ts +++ b/packages/lodestar/src/sync/backfill/backfill.ts @@ -2,11 +2,11 @@ import {IMetrics} from "../../metrics/metrics"; import {EventEmitter} from "events"; import PeerId from "peer-id"; import {StrictEventEmitter} from "strict-event-emitter-types"; -import {blockToHeader} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, blockToHeader} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import {phase0, Root, Slot, allForks, ssz} from "@chainsafe/lodestar-types"; import {ErrorAborted, ILogger} from "@chainsafe/lodestar-utils"; -import {List, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {IBeaconChain} from "../../chain"; import {GENESIS_SLOT, ZERO_HASH} from "../../constants"; import {IBeaconDb} from "../../db"; @@ -18,7 +18,6 @@ import {BackfillSyncError, BackfillSyncErrorCode} from "./errors"; import {verifyBlockProposerSignature, verifyBlockSequence, BackfillBlockHeader, BackfillBlock} from "./verify"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {byteArrayEquals} from "../../util/bytes"; -import {TreeBacked} from "@chainsafe/ssz"; import {computeAnchorCheckpoint} from "../../chain/initState"; /** Default batch size. Same as range sync (2 epochs) */ @@ -31,7 +30,7 @@ export type BackfillSyncModules = { config: IBeaconConfig; logger: ILogger; metrics: IMetrics | null; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; }; @@ -727,7 +726,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} } private async syncBlockByRoot(peer: PeerId, anchorBlockRoot: Root): Promise { - const [anchorBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [anchorBlockRoot] as List); + const [anchorBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [anchorBlockRoot]); if (anchorBlock == null) throw new Error("InvalidBlockSyncedFromPeer"); // GENESIS_SLOT doesn't has valid signature diff --git a/packages/lodestar/src/sync/unknownBlock.ts b/packages/lodestar/src/sync/unknownBlock.ts index 863363a54388..2a5dd55ca21c 100644 --- a/packages/lodestar/src/sync/unknownBlock.ts +++ b/packages/lodestar/src/sync/unknownBlock.ts @@ -1,7 +1,7 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; import {allForks, Root, RootHex} from "@chainsafe/lodestar-types"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {INetwork, NetworkEvent, PeerAction} from "../network"; import {IBeaconChain} from "../chain"; import {IMetrics} from "../metrics"; @@ -231,7 +231,7 @@ export class UnknownBlockSync { for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) { const peer = shuffledPeers[i % shuffledPeers.length]; try { - const [signedBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [blockRoot] as List); + const [signedBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [blockRoot]); // Peer does not have the block, try with next peer if (signedBlock === undefined) { diff --git a/packages/lodestar/src/util/multifork.ts b/packages/lodestar/src/util/multifork.ts index 986a5ba5fc4d..167f152137ff 100644 --- a/packages/lodestar/src/util/multifork.ts +++ b/packages/lodestar/src/util/multifork.ts @@ -1,7 +1,6 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {allForks, Slot} from "@chainsafe/lodestar-types"; import {bytesToInt} from "@chainsafe/lodestar-utils"; -import {ContainerType} from "@chainsafe/ssz"; /** * Slot uint64 @@ -38,7 +37,7 @@ const SLOT_BYTES_POSITION_IN_STATE = 40; export function getSignedBlockTypeFromBytes( config: IChainForkConfig, bytes: Buffer | Uint8Array -): ContainerType { +): allForks.AllForksSSZTypes["SignedBeaconBlock"] { const slot = getSlotFromBytes(bytes); return config.getForkTypes(slot).SignedBeaconBlock; } @@ -50,7 +49,7 @@ export function getSlotFromBytes(bytes: Buffer | Uint8Array): Slot { export function getStateTypeFromBytes( config: IChainForkConfig, bytes: Buffer | Uint8Array -): ContainerType { +): allForks.AllForksSSZTypes["BeaconState"] { const slot = bytesToInt(bytes.slice(SLOT_BYTES_POSITION_IN_STATE, SLOT_BYTES_POSITION_IN_STATE + SLOT_BYTE_COUNT)); return config.getForkTypes(slot).BeaconState; } diff --git a/packages/lodestar/src/util/tree.ts b/packages/lodestar/src/util/tree.ts deleted file mode 100644 index 27d2b42b47ba..000000000000 --- a/packages/lodestar/src/util/tree.ts +++ /dev/null @@ -1,14 +0,0 @@ -import {TreeBacked, List} from "@chainsafe/ssz"; - -export function getTreeAtIndex(tree: TreeBacked>, index: number): TreeBacked> { - const newTree = tree.clone(); - let maxIndex = newTree.length - 1; - if (index > maxIndex) { - throw new Error(`Cannot get tree for index: ${index}, maxIndex: ${maxIndex}`); - } - while (maxIndex > index) { - newTree.pop(); - maxIndex = newTree.length - 1; - } - return newTree; -} diff --git a/packages/lodestar/test/e2e/chain/lightclient.test.ts b/packages/lodestar/test/e2e/chain/lightclient.test.ts index a63116de0db3..9c41954920a1 100644 --- a/packages/lodestar/test/e2e/chain/lightclient.test.ts +++ b/packages/lodestar/test/e2e/chain/lightclient.test.ts @@ -121,10 +121,7 @@ describe("chain / lightclient", function () { throw Error(`LC head state not in cache ${stateRootHex}`); } - const stateLcFromProof = ssz.altair.BeaconState.createTreeBackedFromProof( - header.stateRoot as Uint8Array, - proof - ); + const stateLcFromProof = ssz.altair.BeaconState.createFromProof(proof, header.stateRoot as Uint8Array); expect(toHexString(stateLcFromProof.latestBlockHeader.bodyRoot)).to.equal( toHexString(lcHeadState.latestBlockHeader.bodyRoot), `Recovered 'latestBlockHeader.bodyRoot' from state ${stateRootHex} not correct` diff --git a/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts b/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts index 22a2dea9cf24..bc5fe5661ae5 100644 --- a/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts +++ b/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts @@ -13,11 +13,11 @@ import {getTestnetConfig, medallaTestnetConfig} from "../../utils/testnet"; import {testLogger} from "../../utils/logger"; import {BeaconDb} from "../../../src/db"; import {generateState} from "../../utils/state"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; -import {Root, ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {Eth1Provider} from "../../../src/eth1/provider/eth1Provider"; import {getGoerliRpcUrl} from "../../testParams"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; const dbLocation = "./.__testdb"; @@ -103,8 +103,8 @@ describe("eth1 / Eth1Provider", function () { const periodStart = maxTimestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE; // Compute correct deposit root tree - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct( - pyrmontDepositsDataRoot.map((root) => fromHexString(root)) as List + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU( + pyrmontDepositsDataRoot.map((root) => fromHexString(root)) ); const tbState = generateState( @@ -125,7 +125,7 @@ describe("eth1 / Eth1Provider", function () { config ); - const state = createCachedBeaconState(config, tbState); + const state = createCachedBeaconStateTest(tbState, config); const result = await eth1ForBlockProduction.getEth1DataAndDeposits(state); expect(result.eth1Data).to.deep.equal(latestEth1Data, "Wrong eth1Data for block production"); diff --git a/packages/lodestar/test/e2e/interop/genesisState.test.ts b/packages/lodestar/test/e2e/interop/genesisState.test.ts new file mode 100644 index 000000000000..5755b7675d41 --- /dev/null +++ b/packages/lodestar/test/e2e/interop/genesisState.test.ts @@ -0,0 +1,92 @@ +import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; +import {LevelDbController} from "@chainsafe/lodestar-db"; +import {config} from "@chainsafe/lodestar-config/default"; +import {BeaconDb} from "../../../src"; +import {initDevState} from "../../../src/node/utils/state"; +import {testLogger} from "../../utils/logger"; +import {interopDeposits} from "../../../src/node/utils/interop/deposits"; +import {ssz} from "@chainsafe/lodestar-types"; + +describe("interop / initDevState", () => { + let db: BeaconDb; + const logger = testLogger(); + + before(async () => { + db = new BeaconDb({ + config, + controller: new LevelDbController({name: ".tmpdb"}, {logger}), + }); + await db.start(); + }); + + after(async () => { + await db.stop(); + }); + + it("Create interop deposits", () => { + const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultViewDU, 1); + + /* eslint-disable @typescript-eslint/naming-convention */ + expect(deposits.map((deposit) => ssz.phase0.Deposit.toJson(deposit))).to.deep.equal([ + { + proof: [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7", + "0x0100000000000000000000000000000000000000000000000000000000000000", + ], + data: { + pubkey: "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c", + withdrawal_credentials: "0x00fad2a6bfb0e7f1f0f45460944fbd8dfa7f37da06a4d13b3983cc90bb46963b", + amount: "32000000000", + signature: + "0xa95af8ff0f8c06af4d29aef05ce865f85f82df42b606008ec5b1bcb42b17ae47f4b78cdce1db31ce32d18f42a6b296b4014a2164981780e56b5a40d7723c27b8423173e58fa36f075078b177634f66351412b867c103f532aedd50bcd9b98446", + }, + }, + ]); + }); + + it("Create correct genesisState", async () => { + const validatorCount = 8; + const state = await initDevState(config, db, validatorCount, { + genesisTime: 1644000000, + eth1BlockHash: Buffer.alloc(32, 0xaa), + eth1Timestamp: 1644000000, + }); + + expect(toHexString(state.hashTreeRoot())).to.equal( + "0x3ef3bda2cee48ebdbb6f7a478046631bad3b5eeda3543e55d9dd39da230425bb", + "Wrong genesis state root" + ); + }); +}); diff --git a/packages/lodestar/test/e2e/network/gossipsub.test.ts b/packages/lodestar/test/e2e/network/gossipsub.test.ts index 95f665ea9ef5..6bcf9d048bc1 100644 --- a/packages/lodestar/test/e2e/network/gossipsub.test.ts +++ b/packages/lodestar/test/e2e/network/gossipsub.test.ts @@ -117,7 +117,7 @@ describe("network", function () { } } - const voluntaryExit = ssz.phase0.SignedVoluntaryExit.defaultValue(); + const voluntaryExit = ssz.phase0.SignedVoluntaryExit.defaultValue; await netA.gossip.publishVoluntaryExit(voluntaryExit); const receivedVoluntaryExit = await onVoluntaryExitPromise; @@ -153,7 +153,7 @@ describe("network", function () { const msgCount = 1000; for (let i = 0; i < msgCount; i++) { - const voluntaryExit = ssz.phase0.SignedVoluntaryExit.defaultValue(); + const voluntaryExit = ssz.phase0.SignedVoluntaryExit.defaultValue; voluntaryExit.message.epoch = i; netA.gossip.publishVoluntaryExit(voluntaryExit).catch((e: Error) => { logger.error("Error on publishVoluntaryExit", {}, e); diff --git a/packages/lodestar/test/e2e/network/network.test.ts b/packages/lodestar/test/e2e/network/network.test.ts index ceffccd96e3e..e12b75a03b3f 100644 --- a/packages/lodestar/test/e2e/network/network.test.ts +++ b/packages/lodestar/test/e2e/network/network.test.ts @@ -163,7 +163,7 @@ describe("network", function () { isAggregator: false, }; - netB.metadata.attnets[subscription.subnet] = true; + netB.metadata.attnets.set(subscription.subnet, true); const connected = Promise.all([onPeerConnect(netA), onPeerConnect(netB)]); // Add subnets to B ENR diff --git a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts index 128ccf049dbc..f51c4447bbe3 100644 --- a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts +++ b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts @@ -3,6 +3,7 @@ import {EventEmitter} from "events"; import sinon from "sinon"; import {expect} from "chai"; import {config} from "@chainsafe/lodestar-config/default"; +import {BitArray} from "@chainsafe/ssz"; import {IReqResp, ReqRespMethod} from "../../../../src/network/reqresp"; import {PeerRpcScoreStore, PeerManager, Libp2pPeerMetadataStore} from "../../../../src/network/peers"; import {NetworkEvent, NetworkEventBus} from "../../../../src/network"; @@ -114,7 +115,7 @@ describe("network / peers / PeerManager", function () { const {reqResp, networkEventBus} = await mockModules(); const seqNumber = BigInt(2); - const metadata: phase0.Metadata = {seqNumber, attnets: []}; + const metadata: phase0.Metadata = {seqNumber, attnets: BitArray.fromBitLen(0)}; // Simulate peer1 responding with its metadata reqResp.metadata.resolves(metadata); diff --git a/packages/lodestar/test/e2e/network/reqresp.test.ts b/packages/lodestar/test/e2e/network/reqresp.test.ts index 1b3ce44c4df7..b33b8b6e0acb 100644 --- a/packages/lodestar/test/e2e/network/reqresp.test.ts +++ b/packages/lodestar/test/e2e/network/reqresp.test.ts @@ -7,6 +7,7 @@ import {config} from "@chainsafe/lodestar-config/default"; import {sleep as _sleep} from "@chainsafe/lodestar-utils"; import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; import {createPeerId, IReqRespOptions, Network, prettyPrintPeerId} from "../../../src/network"; import {defaultNetworkOptions, INetworkOptions} from "../../../src/network/options"; import {Method, Encoding} from "../../../src/network/reqresp/types"; @@ -118,8 +119,8 @@ describe("network / ReqResp", function () { const [netA, netB] = await createAndConnectPeers(); // Modify the metadata to make the seqNumber non-zero - netB.metadata.attnets = []; - netB.metadata.attnets = []; + netB.metadata.attnets = BitArray.fromBitLen(0); + netB.metadata.attnets = BitArray.fromBitLen(0); const expectedPong = netB.metadata.seqNumber; expect(expectedPong.toString()).to.deep.equal("2", "seqNumber"); diff --git a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts index 3589239f989d..4d6ed8ca911e 100644 --- a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts +++ b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts @@ -37,7 +37,7 @@ describe("api / impl / validator", () => { noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { - const pubkey = state.index2pubkey[i]; + const pubkey = state.epochCtx.index2pubkey[i]; pubkey.toBytes(PointFormat.compressed); } }, diff --git a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index acb359bbee89..2735d9c83f4e 100644 --- a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,35 +1,41 @@ import {itBench} from "@dapplion/benchmark"; import {expect} from "chai"; import { + CachedBeaconStateAltair, CachedBeaconStateAllForks, computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, } from "@chainsafe/lodestar-beacon-state-transition"; -import {AggregatedAttestationPool, flagIsTimelySource} from "../../../../src/chain/opPools/aggregatedAttestationPool"; -import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List} from "@chainsafe/ssz"; +import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool"; +import {SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX} from "@chainsafe/lodestar-params"; import {generatePerfTestCachedStateAltair} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; -import {ssz} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; + +/** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ +const TIMELY_SOURCE = 1 << TIMELY_SOURCE_FLAG_INDEX; +function flagIsTimelySource(flag: number): boolean { + return (flag & TIMELY_SOURCE) === TIMELY_SOURCE; +} // Aug 11 2021 // getAttestationsForBlock // ✓ getAttestationsForBlock 4.410948 ops/s 226.7086 ms/op - 64 runs 51.8 s describe("getAttestationsForBlock", () => { - let originalState: CachedBeaconStateAllForks; + let originalState: CachedBeaconStateAltair; before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow originalState = (generatePerfTestCachedStateAltair({ goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAllForks; - const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; - const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; + }) as unknown) as CachedBeaconStateAltair; + + const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); + const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); + + const numPreviousEpochParticipation = previousEpochParticipationArr.filter(flagIsTimelySource).length; + const numCurrentEpochParticipation = currentEpochParticipationArr.filter(flagIsTimelySource).length; expect(numPreviousEpochParticipation).to.equal(250000, "Wrong numPreviousEpochParticipation"); expect(numCurrentEpochParticipation).to.equal(250000, "Wrong numCurrentEpochParticipation"); @@ -40,24 +46,24 @@ describe("getAttestationsForBlock", () => { beforeEach: () => getAggregatedAttestationPool(originalState), fn: (pool) => { // logger.info("Number of attestations in pool", pool.getAll().length); - pool.getAttestationsForBlock(originalState); + pool.getAttestationsForBlock(originalState as CachedBeaconStateAllForks); }, }); }); -function getAggregatedAttestationPool(state: CachedBeaconStateAllForks): AggregatedAttestationPool { +function getAggregatedAttestationPool(state: CachedBeaconStateAltair): AggregatedAttestationPool { const pool = new AggregatedAttestationPool(); for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { const slot = state.slot - 1 - epochSlot; const epoch = computeEpochAtSlot(slot); - const committeeCount = state.getCommitteeCountPerSlot(epoch); + const committeeCount = state.epochCtx.getCommitteeCountPerSlot(epoch); const sourceCheckpoint = { epoch: state.currentJustifiedCheckpoint.epoch, - root: state.currentJustifiedCheckpoint.root.valueOf() as Uint8Array, + root: state.currentJustifiedCheckpoint.root, }; for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) { const attestation = { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: slot, index: committeeIndex, @@ -71,10 +77,10 @@ function getAggregatedAttestationPool(state: CachedBeaconStateAllForks): Aggrega signature: Buffer.alloc(96), }; - const committee = state.getBeaconCommittee(slot, committeeIndex); + const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex); // all attestation has full participation so getAttestationsForBlock() has to do a lot of filter - // aggregate_and_proof messages are all TreeBacked - pool.add(ssz.phase0.Attestation.createTreeBackedFromStruct(attestation), committee, committee); + // aggregate_and_proof messages + pool.add(attestation, committee, committee); } } return pool; diff --git a/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts b/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts index bd3a82170b6e..e1863fcc4e88 100644 --- a/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts +++ b/packages/lodestar/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts @@ -14,7 +14,7 @@ describe("CheckpointStateCache perf tests", function () { before(() => { checkpointStateCache = new CheckpointStateCache({}); state = generateCachedState(); - checkpoint = ssz.phase0.Checkpoint.defaultValue(); + checkpoint = ssz.phase0.Checkpoint.defaultValue; }); itBench("CheckpointStateCache - add get delete", () => { diff --git a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts index 629d3714d3aa..159bc8689b48 100644 --- a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,5 +1,4 @@ import {itBench} from "@dapplion/benchmark"; -import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAggregateAndProof} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof"; @@ -14,9 +13,8 @@ describe("validate gossip signedAggregateAndProof", () => { }); const aggStruct = signedAggregateAndProof; - const aggTreeBacked = ssz.phase0.SignedAggregateAndProof.createTreeBackedFromStruct(aggStruct); - for (const [id, agg] of Object.entries({struct: aggStruct, treeBacked: aggTreeBacked})) { + for (const [id, agg] of Object.entries({struct: aggStruct})) { itBench({ id: `validate gossip signedAggregateAndProof - ${id}`, beforeEach: () => chain.seenAggregators["validatorIndexesByEpoch"].clear(), diff --git a/packages/lodestar/test/perf/chain/validation/attestation.test.ts b/packages/lodestar/test/perf/chain/validation/attestation.test.ts index 29f3e3dc612b..aece2d0a60a6 100644 --- a/packages/lodestar/test/perf/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/perf/chain/validation/attestation.test.ts @@ -1,5 +1,4 @@ import {itBench} from "@dapplion/benchmark"; -import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAttestation} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAttestationValidData} from "../../../utils/validationData/attestation"; @@ -14,9 +13,8 @@ describe("validate gossip attestation", () => { }); const attStruct = attestation; - const attTreeBacked = ssz.phase0.Attestation.createTreeBackedFromStruct(attStruct); - for (const [id, att] of Object.entries({struct: attStruct, treeBacked: attTreeBacked})) { + for (const [id, att] of Object.entries({struct: attStruct})) { itBench({ id: `validate gossip attestation - ${id}`, beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), diff --git a/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts b/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts index 3d604b01a274..690278b9e200 100644 --- a/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts +++ b/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts @@ -1,7 +1,8 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {fastSerializeEth1Data, pickEth1Vote} from "../../../src/eth1/utils/eth1Vote"; -import {ContainerType, ListType} from "@chainsafe/ssz"; +import {ContainerType, ListCompositeType} from "@chainsafe/ssz"; +import {newFilledArray, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; describe("eth1 / pickEth1Vote", () => { const ETH1_FOLLOW_DISTANCE_MAINNET = 2048; @@ -9,23 +10,23 @@ describe("eth1 / pickEth1Vote", () => { const SLOTS_PER_EPOCH_MAINNET = 32; const eth1DataVotesLimit = EPOCHS_PER_ETH1_VOTING_PERIOD_MAINNET * SLOTS_PER_EPOCH_MAINNET; - const stateMainnetType = new ContainerType({ - fields: { - eth1DataVotes: new ListType({elementType: ssz.phase0.Eth1Data, limit: eth1DataVotesLimit}), - }, + const stateMainnetType = new ContainerType({ + eth1DataVotes: new ListCompositeType(ssz.phase0.Eth1Data, eth1DataVotesLimit), }); - const stateNoVotes = stateMainnetType.defaultTreeBacked(); - const stateMaxVotes = stateMainnetType.defaultTreeBacked(); + const stateNoVotes = stateMainnetType.defaultViewDU; + const stateMaxVotes = stateMainnetType.defaultViewDU; - stateMaxVotes.eth1DataVotes = Array.from({length: eth1DataVotesLimit}, () => - ssz.phase0.Eth1Data.createTreeBackedFromStruct({ + // Must convert all instances to create a cache + stateMaxVotes.eth1DataVotes = ssz.phase0.Eth1DataVotes.toViewDU( + newFilledArray(eth1DataVotesLimit, { depositRoot: Buffer.alloc(32, 0xdd), // All votes are the same depositCount: 1e6, blockHash: Buffer.alloc(32, 0xdd), }) ); + stateMaxVotes.commit(); // votesToConsider range: // lte: periodStart - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE, @@ -38,11 +39,11 @@ describe("eth1 / pickEth1Vote", () => { })); itBench("pickEth1Vote - no votes", () => { - pickEth1Vote(stateNoVotes, votesToConsider); + pickEth1Vote((stateNoVotes as unknown) as BeaconStateAllForks, votesToConsider); }); itBench("pickEth1Vote - max votes", () => { - pickEth1Vote(stateMaxVotes, votesToConsider); + pickEth1Vote((stateMaxVotes as unknown) as BeaconStateAllForks, votesToConsider); }); }); @@ -63,7 +64,7 @@ describe("eth1 / pickEth1Vote serializers", () => { depositCount: 1e6, blockHash: Buffer.alloc(32, 0xdd), }; - const eth1DataTree = ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataValue); + const eth1DataTree = ssz.phase0.Eth1Data.toViewDU(eth1DataValue); itBench(`pickEth1Vote - Eth1Data hashTreeRoot value x${ETH1_FOLLOW_DISTANCE_MAINNET}`, () => { for (let i = 0; i < ETH1_FOLLOW_DISTANCE_MAINNET; i++) { @@ -75,9 +76,7 @@ describe("eth1 / pickEth1Vote serializers", () => { itBench({ id: `pickEth1Vote - Eth1Data hashTreeRoot tree x${ETH1_FOLLOW_DISTANCE_MAINNET}`, beforeEach: () => - Array.from({length: ETH1_FOLLOW_DISTANCE_MAINNET}, () => - ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataValue) - ), + Array.from({length: ETH1_FOLLOW_DISTANCE_MAINNET}, () => ssz.phase0.Eth1Data.toViewDU(eth1DataValue)), fn: (eth1DataTrees) => { for (let i = 0; i < eth1DataTrees.length; i++) { ssz.phase0.Eth1Data.hashTreeRoot(eth1DataTrees[i]); diff --git a/packages/lodestar/test/sim/threaded/types.ts b/packages/lodestar/test/sim/threaded/types.ts index b4a64f3b1d89..f52b3767b0f7 100644 --- a/packages/lodestar/test/sim/threaded/types.ts +++ b/packages/lodestar/test/sim/threaded/types.ts @@ -2,7 +2,6 @@ import {IChainConfig} from "@chainsafe/lodestar-config"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; import {ChainEvent} from "../../../src/chain"; import {IBeaconNodeOptions} from "../../../src/node/options"; -import {Json} from "@chainsafe/ssz"; export type NodeWorkerOptions = { params: Pick; @@ -26,5 +25,5 @@ export enum MessageEvent { } export type Message = - | {event: ChainEvent.justified; checkpoint: Json} + | {event: ChainEvent.justified; checkpoint: unknown} | {event: MessageEvent.NodeStarted; multiaddr: string}; diff --git a/packages/lodestar/test/spec/allForks/epochProcessing.ts b/packages/lodestar/test/spec/allForks/epochProcessing.ts index c718b117e152..b4caefe7ee90 100644 --- a/packages/lodestar/test/spec/allForks/epochProcessing.ts +++ b/packages/lodestar/test/spec/allForks/epochProcessing.ts @@ -3,17 +3,16 @@ import fs from "node:fs"; import { CachedBeaconStateAllForks, - allForks, EpochProcess, - createCachedBeaconState, + BeaconStateAllForks, beforeProcessEpoch, } from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; import {IBaseSpecTest} from "../type"; @@ -23,8 +22,8 @@ export type EpochProcessFn = (state: CachedBeaconStateAllForks, epochProcess: Ep * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md */ type EpochProcessingStateTestCase = IBaseSpecTest & { - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; }; /** @@ -39,18 +38,21 @@ export function epochProcessing(fork: ForkName, epochProcessFns: Record( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/epoch_processing/${testDir}`, join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre.clone(); + const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); + const epochProcess = beforeProcessEpoch(state); epochProcessFn(state, epochProcess); + state.commit(); + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, diff --git a/packages/lodestar/test/spec/allForks/finality.ts b/packages/lodestar/test/spec/allForks/finality.ts index 029897d28fbc..460148618c4c 100644 --- a/packages/lodestar/test/spec/allForks/finality.ts +++ b/packages/lodestar/test/spec/allForks/finality.ts @@ -1,49 +1,39 @@ import {join} from "node:path"; -import {TreeBacked} from "@chainsafe/ssz"; -import { - CachedBeaconStateAllForks, - allForks, - altair, - createCachedBeaconState, -} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, altair, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest, shouldVerify} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; import {generateBlocksSZZTypeMapping} from "./sanity"; /* eslint-disable @typescript-eslint/naming-convention */ export function finality(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/finality/finality`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/finality/finality/pyspec_tests`), (testcase) => { - let wrappedState = createCachedBeaconState( - getConfig(fork), - testcase.pre as TreeBacked - ) as CachedBeaconStateAllForks; + let state = createCachedBeaconStateTest(testcase.pre, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock; - wrappedState = allForks.stateTransition( - wrappedState, - ssz[fork].SignedBeaconBlock.createTreeBackedFromStruct(signedBlock), - { - verifyStateRoot: false, - verifyProposer: verify, - verifySignatures: verify, - } - ); + state = allForks.stateTransition(state, signedBlock, { + verifyStateRoot: false, + verifyProposer: verify, + verifySignatures: verify, + }); } - return wrappedState; + + state.commit(); + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -72,6 +62,6 @@ interface IFinalityTestCase extends IBaseSpecTest { blocks_count: number; bls_setting: bigint; }; - pre: altair.BeaconState; - post?: altair.BeaconState; + pre: BeaconStateAllForks; + post?: BeaconStateAllForks; } diff --git a/packages/lodestar/test/spec/allForks/fork.ts b/packages/lodestar/test/spec/allForks/fork.ts index 41a822e8dd58..d3641c940e7c 100644 --- a/packages/lodestar/test/spec/allForks/fork.ts +++ b/packages/lodestar/test/spec/allForks/fork.ts @@ -1,25 +1,25 @@ import {join} from "node:path"; -import {TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, phase0, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; +import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; -import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; export function fork(forkConfig: Partial, pre: ForkName, fork: Exclude): void { const testConfig = createIChainForkConfig(forkConfig); - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/fork/fork`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork/fork/pyspec_tests`), (testcase) => { - const preState = createCachedBeaconState(testConfig, testcase.pre as TreeBacked); + const preState = createCachedBeaconStateTest(testcase.pre, testConfig); return allForks.upgradeStateByFork[fork](preState); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[pre].BeaconState, post: ssz[fork].BeaconState, @@ -35,9 +35,9 @@ export function fork(forkConfig: Partial, pre: ForkName, fork: Exc ); } -type PostBeaconState = Exclude; +type PostBeaconState = Exclude; interface IUpgradeStateCase extends IBaseSpecTest { - pre: allForks.BeaconState; + pre: BeaconStateAllForks; post: PostBeaconState; } diff --git a/packages/lodestar/test/spec/allForks/forkChoice.ts b/packages/lodestar/test/spec/allForks/forkChoice.ts index b8a4d5fe5da0..56adbbeedc6e 100644 --- a/packages/lodestar/test/spec/allForks/forkChoice.ts +++ b/packages/lodestar/test/spec/allForks/forkChoice.ts @@ -1,7 +1,6 @@ import {join} from "node:path"; import {expect} from "chai"; import { - createCachedBeaconState, phase0, allForks, computeEpochAtSlot, @@ -10,6 +9,7 @@ import { getEffectiveBalanceIncrementsZeroInactive, computeStartSlotAtEpoch, EffectiveBalanceIncrements, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; import {initializeForkChoice} from "@chainsafe/lodestar/src/chain/forkChoice"; @@ -24,12 +24,15 @@ import {CheckpointWithHex, ForkChoiceError, ForkChoiceErrorCode, IForkChoice} fr import {ssz, RootHex} from "@chainsafe/lodestar-types"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {ACTIVE_PRESET, SLOTS_PER_EPOCH, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {testLogger} from "../../utils/logger"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/naming-convention */ + const ANCHOR_STATE_FILE_NAME = "anchor_state"; const ANCHOR_BLOCK_FILE_NAME = "anchor_block"; const BLOCK_FILE_NAME = "^(block)_([0-9a-zA-Z]+)$"; @@ -40,14 +43,13 @@ const logger = testLogger("spec-test"); export function forkChoiceTest(fork: ForkName): void { for (const testFolder of ["get_head", "on_block"]) { describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/fork_choice/get_head`, + `${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}/pyspec_tests`), (testcase) => { const {steps, anchorState} = testcase; const currentSlot = anchorState.slot; const config = getConfig(fork); - const tbState = config.getForkTypes(currentSlot).BeaconState.createTreeBackedFromStruct(anchorState); - let state = createCachedBeaconState(config, tbState); + let state = createCachedBeaconStateTest(anchorState, config); const emitter = new ChainEventEmitter(); const forkchoice = initializeForkChoice(config, emitter, currentSlot, state, true); @@ -62,11 +64,14 @@ export function forkChoiceTest(fork: ForkName): void { for (const [i, step] of steps.entries()) { if (isTick(step)) { tickTime = bnToNum(step.tick); - forkchoice.updateTime(Math.floor(tickTime / config.SECONDS_PER_SLOT)); + const currentSlot = Math.floor(tickTime / config.SECONDS_PER_SLOT); + logger.debug("Step tick", {currentSlot, valid: Boolean(step.valid), time: tickTime}); + forkchoice.updateTime(currentSlot); } // attestation step else if (isAttestation(step)) { + logger.debug("Step attestation", {root: step.attestation, valid: Boolean(step.valid)}); const attestation = testcase.attestations.get(step.attestation); if (!attestation) throw Error(`No attestation ${step.attestation}`); forkchoice.onAttestation(state.epochCtx.getIndexedAttestation(attestation)); @@ -76,6 +81,13 @@ export function forkChoiceTest(fork: ForkName): void { else if (isBlock(step)) { const signedBlock = testcase.blocks.get(step.block); if (!signedBlock) throw Error(`No block ${step.block}`); + + // Log the BeaconBlock root instead of the SignedBeaconBlock root, forkchoice references BeaconBlock roots + const blockRoot = config + .getForkTypes(signedBlock.message.slot) + .BeaconBlock.hashTreeRoot(signedBlock.message); + logger.debug("Step block", {slot: signedBlock.message.slot, root: toHexString(blockRoot)}); + const preState = stateCache.get(toHexString(signedBlock.message.parentRoot)); if (!preState) { continue; @@ -83,6 +95,8 @@ export function forkChoiceTest(fork: ForkName): void { } const blockDelaySec = (tickTime - preState.genesisTime) % config.SECONDS_PER_SLOT; state = runStateTranstion(preState, signedBlock, forkchoice, checkpointStateCache, blockDelaySec); + // TODO: May be part of runStateTranstion, necessary to commit again? + state.commit(); cacheState(state, stateCache); } @@ -192,13 +206,13 @@ function runStateTranstion( postState = allForks.processSlots(postState, nextEpochSlot, null); cacheCheckpointState(postState, checkpointCache); } - preEpoch = postState.currentShuffling.epoch; + preEpoch = postState.epochCtx.epoch; postState = allForks.stateTransition(postState, signedBlock, { verifyStateRoot: true, verifyProposer: false, verifySignatures: false, }); - const postEpoch = postState.currentShuffling.epoch; + const postEpoch = postState.epochCtx.epoch; if (postEpoch > preEpoch) { cacheCheckpointState(postState, checkpointCache); } @@ -246,14 +260,13 @@ function runStateTranstion( } function cacheCheckpointState(checkpointState: CachedBeaconStateAllForks, checkpointCache: CheckpointStateCache): void { - const config = checkpointState.config; const slot = checkpointState.slot; if (slot % SLOTS_PER_EPOCH !== 0) { throw new Error(`Invalid checkpoint state slot ${checkpointState.slot}`); } const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = config.getForkTypes(slot).BeaconState.hashTreeRoot(checkpointState); + blockHeader.stateRoot = checkpointState.hashTreeRoot(); } const cp: phase0.Checkpoint = { root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader), @@ -324,7 +337,7 @@ interface IForkChoiceTestCase { description?: string; bls_setting: BigInt; }; - anchorState: allForks.BeaconState; + anchorState: BeaconStateAllForks; anchorBlock: allForks.BeaconBlock; steps: Step[]; blocks: Map; diff --git a/packages/lodestar/test/spec/allForks/genesis.ts b/packages/lodestar/test/spec/allForks/genesis.ts index 7f4258ee19f4..a80ae6f2b72c 100644 --- a/packages/lodestar/test/spec/allForks/genesis.ts +++ b/packages/lodestar/test/spec/allForks/genesis.ts @@ -1,9 +1,13 @@ import {join} from "node:path"; import {expect} from "chai"; -import {phase0, Uint64, Root, ssz, allForks, bellatrix} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {phase0, Root, ssz, bellatrix, TimeSeconds} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {initializeBeaconStateFromEth1, isValidGenesisState} from "@chainsafe/lodestar-beacon-state-transition"; +import { + BeaconStateAllForks, + createEmptyEpochContextImmutableData, + initializeBeaconStateFromEth1, + isValidGenesisState, +} from "@chainsafe/lodestar-beacon-state-transition"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; @@ -11,10 +15,13 @@ import {expectEqualBeaconState} from "../util"; import {IBaseSpecTest} from "../type"; import {getConfig} from "./util"; +// The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the +// proposed genesis-validity conditions are working. + /* eslint-disable @typescript-eslint/naming-convention */ export function genesis(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/genesis/initialization`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/genesis/initialization/pyspec_tests`), (testcase) => { @@ -22,21 +29,33 @@ export function genesis(fork: ForkName): void { for (let i = 0; i < testcase.meta.deposits_count; i++) { deposits.push(testcase[`deposits_${i}`] as phase0.Deposit); } - let executionPayloadHeader: TreeBacked | undefined = undefined; - if (testcase["execution_payload_header"]) { - executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.createTreeBackedFromStruct( - testcase["execution_payload_header"] - ); - } + + const config = getConfig(fork); + const immutableData = createEmptyEpochContextImmutableData(config, { + // TODO: Should the genesisValidatorsRoot be random here? + genesisValidatorsRoot: Buffer.alloc(32, 0), + }); + return initializeBeaconStateFromEth1( getConfig(fork), + immutableData, ssz.Root.fromJson((testcase.eth1 as IGenesisInitCase).eth1_block_hash), bnToNum((testcase.eth1 as IGenesisInitCase).eth1_timestamp), deposits, undefined, - executionPayloadHeader - ) as phase0.BeaconState; + testcase["execution_payload_header"] && + ssz.bellatrix.ExecutionPayloadHeader.toViewDU(testcase["execution_payload_header"]) + ); }, + // eth1.yaml + // ``` + // {eth1_block_hash: '0x1212121212121212121212121212121212121212121212121212121212121212', + // eth1_timestamp: 1578009600} + // ``` + // meta.yaml + // ``` + // {deposits_count: 64} + // ``` { inputTypes: {meta: InputType.YAML, eth1: InputType.YAML}, sszTypes: { @@ -54,6 +73,11 @@ export function genesis(fork: ForkName): void { } ); + interface IGenesisValidityTestCase extends IBaseSpecTest { + is_valid: boolean; + genesis: BeaconStateAllForks; + } + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/genesis/validity`, join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/genesis/validity/pyspec_tests`), @@ -65,8 +89,6 @@ export function genesis(fork: ForkName): void { is_valid: InputType.YAML, genesis: InputType.SSZ_SNAPPY, }, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore sszTypes: { genesis: ssz[fork].BeaconState, }, @@ -89,20 +111,15 @@ function generateDepositSSZTypeMapping(n: number): Record { const {proof: specTestProof, state} = testcase; - const stateTB = state as TreeBacked; - const stateRoot = stateTB.hashTreeRoot(); + const stateRoot = state.hashTreeRoot(); const leaf = fromHexString(specTestProof.leaf); const branch = specTestProof.branch.map((item) => fromHexString(item)); const leafIndex = Number(specTestProof.leaf_index); const depth = Math.floor(Math.log2(leafIndex)); const verified = verifyMerkleBranch(leaf, branch, depth, leafIndex % 2 ** depth, stateRoot); expect(verified, "cannot verify merkle branch").to.be.true; - const lodestarProof = stateTB.tree.getProof({ + + const lodestarProof = new Tree(state.node).getProof({ gindex: specTestProof.leaf_index, type: ProofType.single, }) as SingleProof; + return { leaf: toHexString(lodestarProof.leaf), leaf_index: lodestarProof.gindex, @@ -40,11 +42,9 @@ export function merkle(fork: ForkName): void { state: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, proof: InputType.YAML as const, }, - getSszTypes: () => { - return { - state: ssz[fork].BeaconState, - }; - }, + getSszTypes: () => ({ + state: ssz[fork].BeaconState, + }), timeout: 10000, getExpected: (testCase) => testCase.proof, expectFunc: (testCase, expected, actual) => { @@ -54,7 +54,7 @@ export function merkle(fork: ForkName): void { ); interface IMerkleTestCase extends IBaseSpecTest { - state: allForks.BeaconState; + state: BeaconStateAllForks; proof: IProof; } diff --git a/packages/lodestar/test/spec/allForks/operations.ts b/packages/lodestar/test/spec/allForks/operations.ts index d5f5fd2a568e..766f24ed1a00 100644 --- a/packages/lodestar/test/spec/allForks/operations.ts +++ b/packages/lodestar/test/spec/allForks/operations.ts @@ -1,31 +1,29 @@ import fs from "node:fs"; import {join} from "node:path"; -import {CachedBeaconState, allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; import {ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked, Type} from "@chainsafe/ssz"; +import {Type} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {IBaseSpecTest} from "../type"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ -export type BlockProcessFn = ( - state: CachedBeaconState, - testCase: any -) => void; +export type BlockProcessFn = (state: T, testCase: any) => void; -export type OperationsTestCase = IBaseSpecTest & { - pre: BeaconState; - post: BeaconState; +export type OperationsTestCase = IBaseSpecTest & { + pre: BeaconStateAllForks; + post: BeaconStateAllForks; execution: {execution_valid: boolean}; }; -export function operations( +export function operations( fork: ForkName, - operationFns: Record>, + operationFns: Record>, sszTypes?: Record> ): void { const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/operations`); @@ -40,17 +38,18 @@ export function operations( throw Error(`No operationFn for ${testDir}`); } - describeDirectorySpecTest, BeaconState>( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/operations/${testDir}`, join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); - operationFn(state, testcase); + const state = testcase.pre.clone(); + const cachedState = createCachedBeaconStateTest(state, getConfig(fork)); + operationFn(cachedState as T, testcase); + state.commit(); return state; }, { - inputTypes: {...inputTypeSszTreeBacked, execution: InputType.YAML}, + inputTypes: {...inputTypeSszTreeViewDU, execution: InputType.YAML}, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, diff --git a/packages/lodestar/test/spec/allForks/rewards.ts b/packages/lodestar/test/spec/allForks/rewards.ts index 270bd6d2fd7e..1cc4a3617d37 100644 --- a/packages/lodestar/test/spec/allForks/rewards.ts +++ b/packages/lodestar/test/spec/allForks/rewards.ts @@ -5,17 +5,18 @@ import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import { altair, phase0, - allForks, CachedBeaconStatePhase0, - createCachedBeaconState, + BeaconStateAllForks, + BeaconStateAltair, beforeProcessEpoch, } from "@chainsafe/lodestar-beacon-state-transition"; -import {TreeBacked, VectorType} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {VectorCompositeType} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {inputTypeSszTreeBacked} from "../util"; +import {inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -29,10 +30,7 @@ export function rewards(fork: ForkName): void { } } -const Deltas = new VectorType({ - elementType: ssz.altair.BeaconState.fields.balances, - length: 2, -}); +const deltasType = new VectorCompositeType(ssz.phase0.Balances, 2); export function rewardsPhase0(fork: ForkName): void { const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/rewards`); @@ -42,19 +40,19 @@ export function rewardsPhase0(fork: ForkName): void { join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { const config = getConfig(fork); - const wrappedState = createCachedBeaconState(config, testcase.pre as TreeBacked); + const wrappedState = createCachedBeaconStateTest(testcase.pre, config); const epochProcess = beforeProcessEpoch(wrappedState); return phase0.getAttestationDeltas(wrappedState as CachedBeaconStatePhase0, epochProcess); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, - source_deltas: Deltas, - target_deltas: Deltas, - head_deltas: Deltas, - inclusion_delay_deltas: Deltas, - inactivity_penalty_deltas: Deltas, + source_deltas: deltasType, + target_deltas: deltasType, + head_deltas: deltasType, + inclusion_delay_deltas: deltasType, + inactivity_penalty_deltas: deltasType, }, timeout: 100000000, getExpected: (testCase) => @@ -81,7 +79,7 @@ export function rewardsAltair(fork: ForkName): void { join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { const config = getConfig(fork); - const state = createCachedBeaconState(config, testcase.pre as TreeBacked); + const state = createCachedBeaconStateTest(testcase.pre as BeaconStateAltair, config); const epochProcess = beforeProcessEpoch(state); // To debug this test and get granular results you can tweak inputs to get more granular results // @@ -98,13 +96,13 @@ export function rewardsAltair(fork: ForkName): void { return altair.getRewardsAndPenalties(state, epochProcess); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, - head_deltas: Deltas, - source_deltas: Deltas, - target_deltas: Deltas, - inactivity_penalty_deltas: Deltas, + head_deltas: deltasType, + source_deltas: deltasType, + target_deltas: deltasType, + inactivity_penalty_deltas: deltasType, }, getExpected: (testCase) => sumDeltas([ @@ -124,7 +122,7 @@ export function rewardsAltair(fork: ForkName): void { type Deltas = [number[], number[]]; interface RewardTestCasePhase0 extends IBaseSpecTest { - pre: altair.BeaconState; + pre: BeaconStateAllForks; source_deltas: Deltas; target_deltas: Deltas; head_deltas: Deltas; @@ -133,7 +131,7 @@ interface RewardTestCasePhase0 extends IBaseSpecTest { } interface RewardTestCaseAltair extends IBaseSpecTest { - pre: altair.BeaconState; + pre: BeaconStateAllForks; head_deltas: Deltas; source_deltas: Deltas; target_deltas: Deltas; diff --git a/packages/lodestar/test/spec/allForks/sanity.ts b/packages/lodestar/test/spec/allForks/sanity.ts index a28d00fff31a..9e8f2da8c685 100644 --- a/packages/lodestar/test/spec/allForks/sanity.ts +++ b/packages/lodestar/test/spec/allForks/sanity.ts @@ -1,11 +1,11 @@ import {join} from "node:path"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; -import {TreeBacked} from "@chainsafe/ssz"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {bnToNum} from "@chainsafe/lodestar-utils"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest, shouldVerify} from "../type"; import {getConfig} from "./util"; @@ -18,17 +18,19 @@ export function sanity(fork: ForkName): void { } export function sanitySlot(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/sanity/slots`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/sanity/slots/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre.clone(); + const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); const postState = allForks.processSlots(state, state.slot + bnToNum(testcase.slots)); - return postState.type.createTreeBacked(postState.tree); + // TODO: May be part of runStateTranstion, necessary to commit again? + postState.commit(); + return postState; }, { - inputTypes: {...inputTypeSszTreeBacked, slots: InputType.YAML}, + inputTypes: {...inputTypeSszTreeViewDU, slots: InputType.YAML}, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -44,29 +46,25 @@ export function sanitySlot(fork: ForkName): void { } export function sanityBlock(fork: ForkName, testPath: string): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/sanity/blocks`, join(SPEC_TEST_LOCATION, testPath), (testcase) => { - const stateTB = testcase.pre as TreeBacked; - let wrappedState = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre as BeaconStateAllForks; + let wrappedState = createCachedBeaconStateTest(stateTB, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock; - wrappedState = allForks.stateTransition( - wrappedState, - ssz[fork].SignedBeaconBlock.createTreeBackedFromStruct(signedBlock), - { - verifyStateRoot: verify, - verifyProposer: verify, - verifySignatures: verify, - } - ); + wrappedState = allForks.stateTransition(wrappedState, signedBlock, { + verifyStateRoot: verify, + verifyProposer: verify, + verifySignatures: verify, + }); } return wrappedState; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -98,12 +96,12 @@ interface IBlockSanityTestCase extends IBaseSpecTest { blocks_count: number; bls_setting: bigint; }; - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; } interface IProcessSlotsTestCase extends IBaseSpecTest { - pre: allForks.BeaconState; - post?: allForks.BeaconState; + pre: BeaconStateAllForks; + post?: BeaconStateAllForks; slots: bigint; } diff --git a/packages/lodestar/test/spec/allForks/ssz_static.ts b/packages/lodestar/test/spec/allForks/ssz_static.ts index 8ac5e89eebf8..a8df955874bd 100644 --- a/packages/lodestar/test/spec/allForks/ssz_static.ts +++ b/packages/lodestar/test/spec/allForks/ssz_static.ts @@ -1,113 +1,65 @@ import fs from "node:fs"; import path from "node:path"; -import {describeDirectorySpecTest, InputType, safeType} from "@chainsafe/lodestar-spec-test-util"; -import {Bytes32, ssz} from "@chainsafe/lodestar-types"; -import {expect} from "chai"; -import {CompositeType, ContainerType, Type} from "@chainsafe/ssz"; -import {IBaseSSZStaticTestCase} from "../ssz/type"; +import {ssz} from "@chainsafe/lodestar-types"; +import {Type} from "@chainsafe/ssz"; +import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {ACTIVE_PRESET, ForkName, PresetName} from "@chainsafe/lodestar-params"; +import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType"; +import {parseSszStaticTestcase} from "../utils/sszTestCaseParser"; +import {runValidSszTest} from "../utils/runValidSszTest"; -/* eslint-disable @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, no-console */ +// ssz_static +// | Attestation +// | case_0 +// | roots.yaml +// | serialized.ssz_snappy +// | value.yaml +// +// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + +/* eslint-disable + @typescript-eslint/naming-convention, + @typescript-eslint/no-unsafe-assignment, + @typescript-eslint/no-unsafe-call, + @typescript-eslint/no-unsafe-member-access, + no-console +*/ // eslint-disable-next-line type Types = Record>; -const extraTypes = { - Eth1Block: new ContainerType({ - fields: { - timestamp: ssz.Number64, - depositRoot: ssz.Root, - depositCount: ssz.Number64, - }, - }), -}; - export function sszStatic(fork: ForkName): void { const rootDir = path.join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/ssz_static`); for (const typeName of fs.readdirSync(rootDir)) { - const type = - ((ssz[fork] as Types)[typeName] as Type | undefined) || - ((ssz.altair as Types)[typeName] as Type | undefined) || - ((ssz.phase0 as Types)[typeName] as Type | undefined) || - ((extraTypes as Types)[typeName] as Type | undefined); + /* eslint-disable @typescript-eslint/strict-boolean-expressions */ + const type = (ssz[fork] as Types)[typeName] || (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; if (!type) { throw Error(`No type for ${typeName}`); } - const sszType = safeType(type) as CompositeType; - testStatic(typeName, sszType, fork, ACTIVE_PRESET); + testStatic(typeName, type, fork, ACTIVE_PRESET); + /* eslint-enable @typescript-eslint/strict-boolean-expressions */ } } -interface IResult { - root: Bytes32; - serialized: Uint8Array; -} - -function testStatic(typeName: string, sszType: CompositeType, forkName: ForkName, preset: PresetName): void { +function testStatic(typeName: string, sszType: Type, forkName: ForkName, preset: string): void { const typeDir = path.join(SPEC_TEST_LOCATION, `tests/${preset}/${forkName}/ssz_static/${typeName}`); for (const caseName of fs.readdirSync(typeDir)) { - describeDirectorySpecTest, IResult>( - `${preset}/${forkName}/ssz_static/${typeName}/${caseName}`, - path.join(typeDir, caseName), - (testcase) => { - //debugger; - const serialized = sszType.serialize(testcase.serialized); - const root = sszType.hashTreeRoot(testcase.serialized); - return { - serialized, - root, - }; - }, - { - inputTypes: { - roots: InputType.YAML, - serialized: InputType.SSZ_SNAPPY, - }, - sszTypes: { - serialized: sszType, - }, - getExpected: (testCase) => { - return { - root: Buffer.from(testCase.roots.root.replace("0x", ""), "hex"), - serialized: testCase.serialized_raw, - }; - }, - expectFunc: (testCase, expected, actual) => { - if (true && (!expected.serialized.equals(actual.serialized) || !expected.root.equals(actual.root))) { - console.log("testCase", testCase); - console.log("serialize expected", expected.serialized.toString("hex")); - console.log("serialize actual ", Buffer.from(actual.serialized).toString("hex")); - console.log("hashTreeRoot expected", expected.root.toString("hex")); - console.log("hashTreeRoot actual ", Buffer.from(actual.root).toString("hex")); - /* - const bbroot = Type.fields[2][1] - console.log("s bbroot hash", bbroot.hashTreeRoot(structural.beaconBlockRoot)) - console.log("t bbroot hash", bbroot.hashTreeRoot(tree.beaconBlockRoot)) - */ - // debugger; + describe(`${preset}/${forkName}/ssz_static/${typeName}/${caseName}`, () => { + const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); + const caseDir = path.join(typeDir, caseName); + for (const testId of fs.readdirSync(caseDir)) { + it(testId, function () { + // Mainnet must deal with big full states and hash each one multiple times + if (preset === "mainnet") { + this.timeout(30 * 1000); } - const structural = sszType.deserialize(testCase.serialized_raw); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const tree = sszType.createTreeBackedFromBytes(testCase.serialized_raw); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const treeFromStructural = sszType.createTreeBackedFromStruct(structural); - expect(tree.serialize(), "tree serialization != structural serialization").to.deep.equal( - sszType.serialize(structural) - ); - expect(sszType.equals(tree, treeFromStructural), "tree != treeFromStructural"); - expect(expected.serialized.equals(sszType.serialize(structural))); - expect(expected.root.equals(sszType.hashTreeRoot(structural))); - expect(expected.serialized.equals(actual.serialized), "incorrect serialize").to.be.true; - expect(expected.root.equals(actual.root), "incorrect hashTreeRoot").to.be.true; - }, - //shouldSkip: (a, b, i) => i !== 0, - //shouldSkip: (a, b, i) => b !== 'case_10', + + const testData = parseSszStaticTestcase(path.join(caseDir, testId)); + runValidSszTest(sszTypeNoUint, testData); + }); } - ); + }); } } diff --git a/packages/lodestar/test/spec/allForks/transition.ts b/packages/lodestar/test/spec/allForks/transition.ts index 383664fd760d..8b45a45d135a 100644 --- a/packages/lodestar/test/spec/allForks/transition.ts +++ b/packages/lodestar/test/spec/allForks/transition.ts @@ -1,61 +1,40 @@ import {join} from "node:path"; -import {allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {ssz} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ForkName, ACTIVE_PRESET} from "@chainsafe/lodestar-params"; -import {TreeBacked} from "@chainsafe/ssz"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {bnToNum} from "@chainsafe/lodestar-utils"; - -type CreateTreeBackedSignedBlock = (block: allForks.SignedBeaconBlock) => TreeBacked; -const createTreeBackedSignedBlockByFork: Record = { - // Dummy placeholder Fn for phase0 - [ForkName.phase0]: ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.phase0.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, - [ForkName.altair]: ssz.altair.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.altair.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, - [ForkName.bellatrix]: ssz.bellatrix.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.bellatrix.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, -}; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; export function transition( forkConfig: (forkEpoch: number) => Partial, pre: ForkName, fork: Exclude ): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/transition`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/transition/core/pyspec_tests`), (testcase) => { const meta = testcase.meta; // testConfig is used here to load forkEpoch from meta.yaml const testConfig = createIChainForkConfig(forkConfig(bnToNum(meta.fork_epoch))); - let wrappedState = createCachedBeaconState(testConfig, testcase.pre as TreeBacked); + let state = createCachedBeaconStateTest(testcase.pre, testConfig); for (let i = 0; i < meta.blocks_count; i++) { - let tbSignedBlock: TreeBacked; - if (i <= meta.fork_block) { - const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; - tbSignedBlock = createTreeBackedSignedBlockByFork[pre](signedBlock); - } else { - const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; - tbSignedBlock = createTreeBackedSignedBlockByFork[fork](signedBlock); - } - wrappedState = allForks.stateTransition(wrappedState, tbSignedBlock, { + const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; + state = allForks.stateTransition(state, signedBlock, { verifyStateRoot: true, verifyProposer: false, verifySignatures: false, }); } - return wrappedState; + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, getSszTypes: (meta: ITransitionTestCase["meta"]) => { return { pre: ssz[pre].BeaconState, @@ -100,6 +79,6 @@ interface ITransitionTestCase extends IBaseSpecTest { blocks_count: bigint; bls_setting?: bigint; }; - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; } diff --git a/packages/lodestar/test/spec/altair/epoch_processing.test.ts b/packages/lodestar/test/spec/altair/epoch_processing.test.ts index 03924e049f85..0c1d765b9e21 100644 --- a/packages/lodestar/test/spec/altair/epoch_processing.test.ts +++ b/packages/lodestar/test/spec/altair/epoch_processing.test.ts @@ -1,5 +1,5 @@ import {allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; -import {processParticipationRecordUpdates} from "@chainsafe/lodestar-beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates"; +import {phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkName} from "@chainsafe/lodestar-params"; import {EpochProcessFn, epochProcessing} from "../allForks/epochProcessing"; @@ -12,7 +12,7 @@ epochProcessing(ForkName.altair, { inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, justification_and_finalization: allForks.processJustificationAndFinalization, participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, - participation_record_updates: (processParticipationRecordUpdates as unknown) as EpochProcessFn, + participation_record_updates: (phase0.processParticipationRecordUpdates as unknown) as EpochProcessFn, randao_mixes_reset: allForks.processRandaoMixesReset, registry_updates: allForks.processRegistryUpdates, rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, diff --git a/packages/lodestar/test/spec/altair/operations.test.ts b/packages/lodestar/test/spec/altair/operations.test.ts index cacb20ad546d..5c9086aa3a48 100644 --- a/packages/lodestar/test/spec/altair/operations.test.ts +++ b/packages/lodestar/test/spec/altair/operations.test.ts @@ -1,4 +1,9 @@ -import {CachedBeaconStateAllForks, allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + allForks, + altair, +} from "@chainsafe/lodestar-beacon-state-transition"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; @@ -7,20 +12,20 @@ import {operations, BlockProcessFn} from "../allForks/operations"; /* eslint-disable @typescript-eslint/naming-convention */ // Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( +const sync_aggregate: BlockProcessFn = ( state, testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} ) => { - const block = ssz.altair.BeaconBlock.defaultTreeBacked(); + const block = ssz.altair.BeaconBlock.defaultValue; // processSyncAggregate() needs the full block to get the slot block.slot = state.slot; - block.body.syncAggregate = testCase["sync_aggregate"]; + block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); altair.processSyncAggregate(state, block); }; -operations(ForkName.altair, { +operations(ForkName.altair, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { altair.processAttestations(state, [testCase.attestation]); }, diff --git a/packages/lodestar/test/spec/bellatrix/operations.test.ts b/packages/lodestar/test/spec/bellatrix/operations.test.ts index 44a715a803a5..18ef54410d61 100644 --- a/packages/lodestar/test/spec/bellatrix/operations.test.ts +++ b/packages/lodestar/test/spec/bellatrix/operations.test.ts @@ -16,20 +16,20 @@ import {processExecutionPayload} from "@chainsafe/lodestar-beacon-state-transiti /* eslint-disable @typescript-eslint/naming-convention */ // Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( +const sync_aggregate: BlockProcessFn = ( state, testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} ) => { - const block = ssz.altair.BeaconBlock.defaultTreeBacked(); + const block = ssz.altair.BeaconBlock.defaultValue; // processSyncAggregate() needs the full block to get the slot block.slot = state.slot; - block.body.syncAggregate = testCase["sync_aggregate"]; + block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); altair.processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block); }; -operations(ForkName.bellatrix, { +operations(ForkName.bellatrix, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { altair.processAttestations((state as unknown) as CachedBeaconStateAltair, [testCase.attestation]); }, diff --git a/packages/lodestar/test/spec/phase0/operations.test.ts b/packages/lodestar/test/spec/phase0/operations.test.ts index ac0865d6223c..3e540df86b24 100644 --- a/packages/lodestar/test/spec/phase0/operations.test.ts +++ b/packages/lodestar/test/spec/phase0/operations.test.ts @@ -1,4 +1,9 @@ -import {CachedBeaconStateAllForks, allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; +import { + CachedBeaconStateAllForks, + CachedBeaconStatePhase0, + allForks, + phase0, +} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; import {operations} from "../allForks/operations"; @@ -6,7 +11,7 @@ import {operations} from "../allForks/operations"; /* eslint-disable @typescript-eslint/naming-convention */ /** Describe with which function to run each directory of tests */ -operations(ForkName.phase0, { +operations(ForkName.phase0, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { phase0.processAttestation(state, testCase.attestation); }, diff --git a/packages/lodestar/test/spec/phase0/shuffling.test.ts b/packages/lodestar/test/spec/phase0/shuffling.test.ts index 62e8a456bed8..e54a177ef216 100644 --- a/packages/lodestar/test/spec/phase0/shuffling.test.ts +++ b/packages/lodestar/test/spec/phase0/shuffling.test.ts @@ -1,31 +1,31 @@ import {join} from "node:path"; import {unshuffleList} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {Uint64} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; +import {bnToNum, fromHex} from "@chainsafe/lodestar-utils"; describeDirectorySpecTest( `${ACTIVE_PRESET}/phase0/shuffling/`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/phase0/shuffling/core/shuffle`), (testcase) => { - const seed = Buffer.from(testcase.mapping.seed.replace("0x", ""), "hex"); - const output = Array.from({length: Number(testcase.mapping.count)}, (_, i) => i); + const seed = fromHex(testcase.mapping.seed); + const output = Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i); unshuffleList(output, seed); return output; }, { inputTypes: {mapping: InputType.YAML}, timeout: 10000, - getExpected: (testCase) => testCase.mapping.mapping.map((value) => Number(value)), + getExpected: (testCase) => testCase.mapping.mapping.map((value) => bnToNum(value)), } ); interface IShufflingTestCase extends IBaseSpecTest { mapping: { seed: string; - count: Uint64; - mapping: Uint64[]; + count: bigint; + mapping: bigint[]; }; } diff --git a/packages/lodestar/test/spec/ssz/generic/index.test.ts b/packages/lodestar/test/spec/ssz/generic/index.test.ts index 559f59d0a4f4..7ce4bd79a753 100644 --- a/packages/lodestar/test/spec/ssz/generic/index.test.ts +++ b/packages/lodestar/test/spec/ssz/generic/index.test.ts @@ -1,38 +1,38 @@ import {expect} from "chai"; import path from "node:path"; import fs from "node:fs"; -// eslint-disable-next-line no-restricted-imports -import {parseInvalidTestcase, parseValidTestcase} from "@chainsafe/lodestar-spec-test-util/lib/sszGeneric"; -import {CompositeType, isCompositeType, toHexString, Type} from "@chainsafe/ssz"; import {SPEC_TEST_LOCATION} from "../../specTestVersioning"; - -// Test types defined here +import {parseSszGenericValidTestcase, parseSszGenericInvalidTestcase} from "../../utils/sszTestCaseParser"; +import {runValidSszTest} from "../../utils/runValidSszTest"; import {getTestType} from "./types"; const rootGenericSszPath = path.join(SPEC_TEST_LOCATION, "tests", "general", "phase0", "ssz_generic"); -// ssz_generic -// | basic_vector -// | invalid -// | vec_bool_0 -// | serialized.ssz_snappy -// | valid -// | vec_bool_1_max -// | meta.yaml -// | serialized.ssz_snappy -// | value.yaml -// -// Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md - for (const testType of fs.readdirSync(rootGenericSszPath)) { const testTypePath = path.join(rootGenericSszPath, testType); describe(`${testType} invalid`, () => { const invalidCasesPath = path.join(testTypePath, "invalid"); for (const invalidCase of fs.readdirSync(invalidCasesPath)) { + const onlyId = process.env.ONLY_ID; + if (onlyId && !invalidCase.includes(onlyId)) { + continue; + } + it(invalidCase, () => { + // TODO: Strong type errors and assert that the entire it() throws known errors + if (invalidCase.endsWith("_0")) { + expect(() => getTestType(testType, invalidCase), "Must throw constructing type").to.throw(); + return; + } + const type = getTestType(testType, invalidCase); - const testData = parseInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + const testData = parseSszGenericInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + + /* eslint-disable no-console */ + if (process.env.DEBUG) { + console.log({serialized: Buffer.from(testData.serialized).toString("hex")}); + } // Unlike the valid suite, invalid encodings do not have any value or hash tree root. The serialized data // should simply not be decoded without raising an error. @@ -53,50 +53,20 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { continue; } + const onlyId = process.env.ONLY_ID; + if (onlyId && !validCase.includes(onlyId)) { + continue; + } + it(validCase, () => { const type = getTestType(testType, validCase); - - const testData = parseValidTestcase(path.join(validCasesPath, validCase), type); - const testDataSerialized = toHexString(testData.serialized); - const testDataRoot = testData.root; - - const serialized = wrapErr(() => type.serialize(testData.value), "type.serialize()"); - const value = wrapErr(() => type.deserialize(testData.serialized), "type.deserialize()"); - const root = wrapErr(() => type.hashTreeRoot(testData.value), "type.hashTreeRoot()"); - const valueSerdes = wrapErr(() => type.deserialize(serialized), "type.deserialize(serialized)"); - - expect(valueSerdes).to.deep.equal(testData.value, "round trip serdes"); - expect(toHexString(serialized)).to.equal(testDataSerialized, "struct serialize"); - expect(value).to.deep.equal(testData.value, "struct deserialize"); - expect(toHexString(root)).to.equal(testDataRoot, "struct hashTreeRoot"); - - // If the type is composite, test tree-backed ops - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if (!isCompositeType(type as Type)) return; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const compositeType = type as CompositeType; - - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const treebackedValue = compositeType.createTreeBackedFromStruct(testData.value); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const treeToStruct = compositeType.tree_convertToStruct(treebackedValue.tree); - - expect(treeToStruct).to.deep.equal(testData.value, "tree-backed to struct"); - expect(type.equals(testData.value, treebackedValue), "struct - tree-backed type.equals()").to.be.true; - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - expect(toHexString(treebackedValue.serialize())).to.equal(testDataSerialized, "tree-backed serialize"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - expect(toHexString(treebackedValue.hashTreeRoot())).to.equal(testDataRoot, "tree-backed hashTreeRoot"); + const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); + runValidSszTest(type, { + root: testData.root, + serialized: testData.serialized, + jsonValue: testData.jsonValue, + }); }); } }); } - -function wrapErr(fn: () => T, prefix: string): T { - try { - return fn(); - } catch (e) { - (e as Error).message = `${prefix}: ${(e as Error).message}`; - throw e; - } -} diff --git a/packages/lodestar/test/spec/ssz/generic/types.ts b/packages/lodestar/test/spec/ssz/generic/types.ts index 153f5f84d9e6..31bbeb633cb1 100644 --- a/packages/lodestar/test/spec/ssz/generic/types.ts +++ b/packages/lodestar/test/spec/ssz/generic/types.ts @@ -1,33 +1,39 @@ import { - BigIntUintType, - BitListType, + Type, + BooleanType, + UintBigintType, + UintNumberType, BitVectorType, - booleanType, - byteType, + BitListType, ContainerType, - ListType, - Type, - VectorType, + ListBasicType, + VectorBasicType, + VectorCompositeType, } from "@chainsafe/ssz"; +const bool = new BooleanType(); +const byte = new UintNumberType(1); +const uint8 = new UintNumberType(1); +const uint16 = new UintNumberType(2); +const uint32 = new UintNumberType(4); +const uint64 = new UintBigintType(8); +const uint128 = new UintBigintType(16); +const uint256 = new UintBigintType(32); + /* eslint-disable @typescript-eslint/naming-convention */ // class SingleFieldTestStruct(Container): // A: byte const SingleFieldTestStruct = new ContainerType({ - fields: { - a: byteType, - }, + A: byte, }); // class SmallTestStruct(Container): // A: uint16 // B: uint16 const SmallTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new BigIntUintType({byteLength: 16 / 8}), - }, + A: uint16, + B: uint16, }); // class FixedTestStruct(Container): @@ -35,11 +41,9 @@ const SmallTestStruct = new ContainerType({ // B: uint64 // C: uint32 const FixedTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 8 / 8}), - b: new BigIntUintType({byteLength: 64 / 8}), - c: new BigIntUintType({byteLength: 32 / 8}), - }, + A: uint8, + B: uint64, + C: uint32, }); // class VarTestStruct(Container): @@ -47,11 +51,9 @@ const FixedTestStruct = new ContainerType({ // B: List[uint16, 1024] // C: uint8 const VarTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new ListType({elementType: new BigIntUintType({byteLength: 16 / 8}), limit: 1024}), - c: new BigIntUintType({byteLength: 8 / 8}), - }, + A: uint16, + B: new ListBasicType(uint16, 1024), + C: uint8, }); // class ComplexTestStruct(Container): @@ -63,15 +65,13 @@ const VarTestStruct = new ContainerType({ // F: Vector[FixedTestStruct, 4] // G: Vector[VarTestStruct, 2] const ComplexTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new ListType({elementType: new BigIntUintType({byteLength: 16 / 8}), limit: 128}), - c: new BigIntUintType({byteLength: 8 / 8}), - d: new BitListType({limit: 256}), - e: VarTestStruct, - f: new VectorType({elementType: FixedTestStruct, length: 4}), - g: new VectorType({elementType: VarTestStruct, length: 2}), - }, + A: uint16, + B: new ListBasicType(uint16, 128), + C: uint8, + D: new BitListType(256), + E: VarTestStruct, + F: new VectorCompositeType(FixedTestStruct, 4), + G: new VectorCompositeType(VarTestStruct, 2), }); // class BitsStruct(Container): @@ -81,13 +81,11 @@ const ComplexTestStruct = new ContainerType({ // D: Bitlist[6] // E: Bitvector[8] const BitsStruct = new ContainerType({ - fields: { - a: new BitListType({limit: 5}), - b: new BitVectorType({length: 2}), - c: new BitVectorType({length: 1}), - d: new BitListType({limit: 6}), - e: new BitVectorType({length: 8}), - }, + A: new BitListType(5), + B: new BitVectorType(2), + C: new BitVectorType(1), + D: new BitListType(6), + E: new BitVectorType(8), }); const containerTypes = { @@ -100,13 +98,13 @@ const containerTypes = { }; const vecElementTypes = { - bool: booleanType, - uint8: new BigIntUintType({byteLength: 8 / 8}), - uint16: new BigIntUintType({byteLength: 16 / 8}), - uint32: new BigIntUintType({byteLength: 32 / 8}), - uint64: new BigIntUintType({byteLength: 64 / 8}), - uint128: new BigIntUintType({byteLength: 128 / 8}), - uint256: new BigIntUintType({byteLength: 256 / 8}), + bool, + uint8, + uint16, + uint32, + uint64, + uint128, + uint256, }; export function getTestType(testType: string, testCase: string): Type { @@ -121,7 +119,7 @@ export function getTestType(testType: string, testCase: string): Type { if (elementType === undefined) throw Error(`No vecElementType for ${elementTypeStr}: '${testCase}'`); const length = parseInt(lengthStr); if (isNaN(length)) throw Error(`Bad length ${length}: '${testCase}'`); - return new VectorType({elementType, length}); + return new VectorBasicType(elementType, length); } // `bitlist_{limit}` @@ -130,19 +128,19 @@ export function getTestType(testType: string, testCase: string): Type { // Consider case `bitlist_no_delimiter_empty` const limit = testCase.includes("no_delimiter") ? 0 : parseSecondNum(testCase, "limit"); // TODO: memoize - return new BitListType({limit}); + return new BitListType(limit); } // `bitvec_{length}` // {length}: the length, in bits, of the bitvector. case "bitvector": { // TODO: memoize - return new BitVectorType({length: parseSecondNum(testCase, "length")}); + return new BitVectorType(parseSecondNum(testCase, "length")); } // A boolean has no type variations. Instead, file names just plainly describe the contents for debugging. case "boolean": - return booleanType; + return bool; // {container name} // {container name}: Any of the container names listed below (excluding the `(Container)` python super type) @@ -158,7 +156,7 @@ export function getTestType(testType: string, testCase: string): Type { // {size}: the uint size: 8, 16, 32, 64, 128 or 256. case "uints": { // TODO: memoize - return new BigIntUintType({byteLength: parseSecondNum(testCase, "size") / 8}); + return new UintBigintType((parseSecondNum(testCase, "size") / 8) as 8); } default: diff --git a/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts b/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts deleted file mode 100644 index 3937c6ded7c4..000000000000 --- a/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IUintCase extends IBaseCase { - value: string; - ssz: string; - type: string; - valid: boolean; -} diff --git a/packages/lodestar/test/spec/util.ts b/packages/lodestar/test/spec/util.ts index 27ee0f188f8e..bfa78c10dff3 100644 --- a/packages/lodestar/test/spec/util.ts +++ b/packages/lodestar/test/spec/util.ts @@ -2,15 +2,19 @@ import {expect} from "chai"; import {allForks, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; import {InputType} from "@chainsafe/lodestar-spec-test-util"; -import {ContainerType} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; /** Compare each field in BeaconState to help debug failed test easier. */ export function expectEqualBeaconState( fork: ForkName, - expected: allForks.BeaconState, - actual: allForks.BeaconState + expectedView: BeaconStateAllForks, + actualView: BeaconStateAllForks ): void { - const stateType = ssz[fork].BeaconState as ContainerType; + // TODO: Is it cheaper to compare roots? Or maybe the serialized bytes? + const expected = expectedView.toValue(); + const actual = actualView.toValue(); + + const stateType = ssz[fork].BeaconState as allForks.AllForksSSZTypes["BeaconState"]; if (!stateType.equals(actual, expected)) { expect(stateType.toJson(actual)).to.deep.equal(stateType.toJson(expected)); throw Error("Wrong state"); @@ -18,8 +22,8 @@ export function expectEqualBeaconState( } /** Shortcut for commonly used inputType */ -export const inputTypeSszTreeBacked = { - pre: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, - post: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, +export const inputTypeSszTreeViewDU = { + pre: InputType.SSZ_SNAPPY, + post: InputType.SSZ_SNAPPY, meta: InputType.YAML as const, }; diff --git a/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts new file mode 100644 index 000000000000..01a8aaf683cd --- /dev/null +++ b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts @@ -0,0 +1,52 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment +, @typescript-eslint/no-explicit-any */ + +// ####################################### +// # MUST NOT IMPORT FROM @chainsafe/ssz # +// ####################################### +import { + Type, + UintNumberType, + UintBigintType, + ContainerType, + ListBasicType, + ListCompositeType, + VectorBasicType, + VectorCompositeType, +} from "@chainsafe/ssz"; + +/** + * Transform the type to something that is safe to deserialize + * + * This mainly entails making sure all numbers are bignumbers + */ +export function replaceUintTypeWithUintBigintType>(type: T): T { + if (type instanceof UintNumberType && type.byteLength === 8) { + return (new UintBigintType(type.byteLength) as unknown) as T; + } + + // For Container iterate and replace all sub properties + if (type instanceof ContainerType) { + const fields = {...type.fields}; + for (const key of Object.keys(fields) as (keyof typeof fields)[]) { + fields[key] = replaceUintTypeWithUintBigintType(fields[key]); + } + return (new ContainerType(fields, type.opts) as unknown) as T; + } + + // For List or vectors replace the subType + if (type instanceof ListBasicType) { + return (new ListBasicType(replaceUintTypeWithUintBigintType(type.elementType), type.limit) as unknown) as T; + } + if (type instanceof VectorBasicType) { + return (new VectorBasicType(replaceUintTypeWithUintBigintType(type.elementType), type.length) as unknown) as T; + } + if (type instanceof ListCompositeType) { + return (new ListCompositeType(replaceUintTypeWithUintBigintType(type.elementType), type.limit) as unknown) as T; + } + if (type instanceof VectorCompositeType) { + return (new VectorCompositeType(replaceUintTypeWithUintBigintType(type.elementType), type.length) as unknown) as T; + } + + return type; +} diff --git a/packages/lodestar/test/spec/utils/runValidSszTest.ts b/packages/lodestar/test/spec/utils/runValidSszTest.ts new file mode 100644 index 000000000000..1c7f9cb0eccf --- /dev/null +++ b/packages/lodestar/test/spec/utils/runValidSszTest.ts @@ -0,0 +1,188 @@ +import {expect} from "chai"; +import {Node} from "@chainsafe/persistent-merkle-tree"; +import {Type, CompositeType, fromHexString, toHexString} from "@chainsafe/ssz"; + +type ValidTestCaseData = { + root: string; + serialized: string | Uint8Array; + jsonValue: unknown; +}; + +/* eslint-disable no-console */ + +export function runValidSszTest(type: Type, testData: ValidTestCaseData): void { + const testDataRootHex = testData.root; + const testDataSerialized = + typeof testData.serialized === "string" ? fromHexString(testData.serialized) : testData.serialized; + const testDataSerializedStr = + typeof testData.serialized === "string" ? testData.serialized : toHexString(testData.serialized); + + if (process.env.RENDER_JSON) { + console.log( + JSON.stringify( + testData.jsonValue, + (key, value: unknown) => (typeof value === "bigint" ? value.toString() : value), + 2 + ) + ); + } + + if (process.env.RENDER_SERIALIZED) { + console.log("serialized", testDataSerializedStr); + } + + // JSON -> value - fromJson() + const testDataValue = wrapErr(() => type.fromJson(testData.jsonValue), "type.fromJson()"); + // Use our re-transformed to JSON to ensure the type is the safe + const testDataJson = wrapErr(() => type.toJson(testDataValue), "type.toJson()"); + + function assertBytes(bytes: Uint8Array, msg: string): void { + expect(toHexString(bytes)).to.equal(testDataSerializedStr, `Wrong serialized - ${msg}`); + } + + function assertValue(value: unknown, msg: string): void { + expect(type.toJson(value)).to.deep.equal(testDataJson, `Wrong json - ${msg}`); + } + + function assertRoot(root: Uint8Array, msg: string): void { + expect(toHexString(root)).to.equal(testDataRootHex, `Wrong root - ${msg}`); + } + + function assertNode(node: Node, msg: string): void { + expect(toHexString(node.root)).to.equal(testDataRootHex, `Wrong node - ${msg}`); + } + + { + // value - equals + const isEqual = wrapErr(() => type.equals(testDataValue, testDataValue), "type.equals()"); + expect(isEqual).to.equal(true, "Value is not equal to itself"); + } + + { + // value - clone + const isEqual = wrapErr(() => type.equals(type.clone(testDataValue), testDataValue), "type.clone()"); + expect(isEqual).to.equal(true, "Cloned value is not equal to itself"); + } + + // value -> bytes - serialize() + const serialized = wrapErr(() => type.serialize(testDataValue), "type.serialize()"); + assertBytes(serialized, "type.serialize()"); + + { + // bytes -> value - deserialize() + const value = wrapErr(() => type.deserialize(copy(testDataSerialized)), "type.deserialize()"); + assertValue(value, "type.deserialize()"); + } + + // To print the chunk roots of a type value + // + // $ RENDER_ROOTS=true ONLY_ID="4 arrays" ../../node_modules/.bin/mocha test/unit/byType/vector/valid.test.ts + // + // 0x0000000000000000000000000000000000000000000000000000000000000000 + if (process.env.RENDER_ROOTS) { + if (type.isBasic) { + console.log("ROOTS Basic", toHexString(type.serialize(testDataValue))); + } else { + const roots = (type as CompositeType)["getRoots"](testDataValue); + console.log( + "ROOTS Composite", + roots.map((root) => toHexString(root)) + ); + } + } + + { + // hashTreeRoot() + const root = wrapErr(() => type.hashTreeRoot(testDataValue), "type.hashTreeRoot()"); + assertRoot(root, "type.hashTreeRoot()"); + } + + // value -> tree - value_toTree() + const node = wrapErr(() => type.value_toTree(testDataValue), "type.value_toTree()"); + assertNode(node, "type.value_toTree()"); + + // To print a tree a single test you are debugging do + // + // $ RENDER_TREE=true ONLY_ID="4 arrays" ../../node_modules/.bin/mocha test/unit/byType/vector/valid.test.ts + // + // '1000' => '0x0000000000000000000000000000000000000000000000000000000000000000', + // '1001' => '0x0000000000000000000000000000000000000000000000000000000000000000', + // '1010' => '0x40e2010000000000000000000000000000000000000000000000000000000000', + // '1011' => '0xf1fb090000000000000000000000000000000000000000000000000000000000', + // '1100' => '0x4794030000000000000000000000000000000000000000000000000000000000', + // '1101' => '0xf8ad0b0000000000000000000000000000000000000000000000000000000000', + // '1110' => '0x4e46050000000000000000000000000000000000000000000000000000000000', + // '1111' => '0xff5f0d0000000000000000000000000000000000000000000000000000000000' + if (process.env.RENDER_TREE) { + renderTree(node); + } + + { + // tree -> bytes + const uint8Array = new Uint8Array(type.tree_serializedSize(node)); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + type.tree_serializeToBytes({uint8Array, dataView}, 0, node); + assertBytes(uint8Array, "type.tree_serializeToBytes"); + } + + { + // tree -> value - + const value = wrapErr(() => type.tree_toValue(node), "type.tree_toValue()"); + assertValue(value, "type.tree_toValue()"); + } + + { + // bytes -> tree + const uint8Array = copy(testDataSerialized); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + const node = type.tree_deserializeFromBytes({uint8Array, dataView}, 0, testDataSerialized.length); + assertNode(node, "type.tree_deserializeFromBytes()"); + } + + if (!type.isBasic) { + const compositeType = type as CompositeType; + + const view = compositeType.deserializeToView(copy(testDataSerialized)); + assertNode(compositeType.commitView(view), "deserializeToView"); + + const viewDU = compositeType.deserializeToViewDU(copy(testDataSerialized)); + assertNode(compositeType.commitViewDU(viewDU), "deserializeToViewDU"); + } +} + +function copy(buf: Uint8Array): Uint8Array { + const copy = new Uint8Array(buf.length); + copy.set(buf); + return copy; +} + +function wrapErr(fn: () => T, prefix: string): T { + try { + return fn(); + } catch (e) { + (e as Error).message = `${prefix}: ${(e as Error).message}`; + throw e; + } +} + +export function toJsonOrString(value: unknown): unknown { + if (typeof value === "number" || typeof value === "bigint") { + return value.toString(10); + } else { + return value; + } +} + +function renderTree(node: Node): void { + console.log(gatherLeafNodes(node)); +} + +export function gatherLeafNodes(node: Node, nodes = new Map(), gindex = "1"): Map { + if (node.isLeaf()) { + nodes.set(gindex, toHexString(node.root)); + } else { + gatherLeafNodes(node.left, nodes, gindex + "0"); + gatherLeafNodes(node.right, nodes, gindex + "1"); + } + return nodes; +} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts b/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts deleted file mode 100644 index d4069af35dc6..000000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts +++ /dev/null @@ -1,47 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IBeaconStateComparisonCase extends IBaseCase { - pre: any; - post: any; -} - -export interface IOperationsCase extends IBeaconStateComparisonCase { - // eslint-disable-next-line @typescript-eslint/naming-convention - bls_setting?: bigint; -} - -export interface IAttestationCase extends IOperationsCase { - attestation: any; -} - -export interface IAttesterSlashingCase extends IOperationsCase { - attesterSlashing: any; -} - -export interface IBlockHeaderCase extends IOperationsCase { - block: any; -} - -export interface IDepositCase extends IOperationsCase { - deposit: any; -} - -export interface IProposerSlashingCase extends IOperationsCase { - proposerSlashing: any; -} - -export interface ITransferCase extends IOperationsCase { - transfer: any; -} - -export interface IVoluntaryExitCase extends IOperationsCase { - voluntaryExit: any; -} - -export interface IBlockSanityCase extends IOperationsCase { - blocks: any[]; -} - -export interface ISlotSanityCase extends IOperationsCase { - slots: bigint; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts b/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts deleted file mode 100644 index 234bfcf68f7a..000000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts +++ /dev/null @@ -1,13 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IGenesisInitCase extends IBaseCase { - state: any; - eth1BlockHash: any; - eth1Timestamp: any; - deposits: any; -} - -export interface IGenesisValidityCase extends IBaseCase { - genesis: any; - isValid: true; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts b/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts deleted file mode 100644 index 764b9694ce4e..000000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IShufflingCase extends IBaseCase { - seed: string; - count: bigint; - shuffled: bigint[]; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts b/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts deleted file mode 100644 index 3aa8fa1f4bea..000000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {phase0} from "@chainsafe/lodestar-types"; -import {IBaseSpecTest} from "../../type"; - -export interface IStateTestCase extends IBaseSpecTest { - pre: phase0.BeaconState; - post: phase0.BeaconState; -} diff --git a/packages/lodestar/test/spec/utils/sszTestCaseParser.ts b/packages/lodestar/test/spec/utils/sszTestCaseParser.ts new file mode 100644 index 000000000000..d3c99a6a238d --- /dev/null +++ b/packages/lodestar/test/spec/utils/sszTestCaseParser.ts @@ -0,0 +1,92 @@ +import path from "node:path"; +import fs from "node:fs"; +import {uncompress} from "snappyjs"; +import {loadYaml} from "@chainsafe/lodestar-utils"; +import jsyaml from "js-yaml"; + +/* eslint-disable + @typescript-eslint/explicit-module-boundary-types, + @typescript-eslint/explicit-function-return-type +*/ + +export type ValidTestCaseData = { + root: string; + serialized: Uint8Array; + jsonValue: unknown; +}; + +/** + * ssz_static + * | Attestation + * | case_0 + * | roots.yaml + * | serialized.ssz_snappy + * | value.yaml + * + * Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + */ +export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { + return parseSszValidTestcase(dirpath, "roots.yaml"); +} + +/** + * ssz_generic + * | basic_vector + * | valid + * | vec_bool_1_max + * | meta.yaml + * | serialized.ssz_snappy + * | value.yaml + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericValidTestcase(dirpath: string): ValidTestCaseData { + return parseSszValidTestcase(dirpath, "meta.yaml"); +} + +export function parseSszValidTestcase(dirpath: string, metaFilename: string): ValidTestCaseData { + // The root is stored in meta.yml as: + // root: 0xDEADBEEF + const metaStr = fs.readFileSync(path.join(dirpath, metaFilename), "utf8"); + const meta = jsyaml.load(metaStr) as {root: string}; + if (typeof meta.root !== "string") { + throw Error(`meta.root not a string: ${meta.root}\n${fs}`); + } + + // The serialized value is stored in serialized.ssz_snappy + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); + + // The value is stored in value.yml + const yamlPath = path.join(dirpath, "value.yaml"); + const yamlStr = fs.readFileSync(yamlPath, "utf8"); + const jsonValue = readYamlNumbersAsStrings(yamlStr); + // type.fromJson(loadYamlFile(path.join(dirpath, "value.yaml")) as Json) as T; + + return { + root: meta.root, + serialized, + jsonValue, + }; +} + +/** + * ssz_generic + * | basic_vector + * | invalid + * | vec_bool_0 + * | serialized.ssz_snappy + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericInvalidTestcase(dirpath: string) { + // The serialized value is stored in serialized.ssz_snappy + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); + + return { + serialized, + }; +} + +export function readYamlNumbersAsStrings(yamlStr: string): unknown { + return loadYaml(yamlStr); +} diff --git a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts index a3208eb7026d..5877f617e325 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts @@ -1,6 +1,5 @@ import sinon from "sinon"; import * as blockUtils from "../../../../../../src/api/impl/beacon/blocks/utils"; -import {ssz} from "@chainsafe/lodestar-types"; import {expect, use} from "chai"; import chaiAsPromised from "chai-as-promised"; import {generateEmptySignedBlock} from "../../../../../utils/block"; @@ -31,6 +30,5 @@ describe("api - beacon - getBlock", function () { resolveBlockIdStub.withArgs(sinon.match.any, sinon.match.any, "head").resolves(generateEmptySignedBlock()); const {data: result} = await server.blockApi.getBlock("head"); expect(result).to.not.be.null; - expect(() => ssz.phase0.SignedBeaconBlock.assertValidValue(result)).to.not.throw(); }); }); diff --git a/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts b/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts index fdef6b926ff1..252ebd5df42b 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts @@ -13,8 +13,7 @@ import * as attesterSlashingValidation from "../../../../../../src/chain/validat import * as proposerSlashingValidation from "../../../../../../src/chain/validation/proposerSlashing"; import * as voluntaryExitValidation from "../../../../../../src/chain/validation/voluntaryExit"; -import {phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {phase0} from "@chainsafe/lodestar-types"; import {Eth2Gossipsub} from "../../../../../../src/network/gossip"; import {generateEmptySignedBlockHeader} from "../../../../../utils/block"; import {setupApiImplTestServer} from "../../index.test"; @@ -87,12 +86,12 @@ describe("beacon pool api impl", function () { describe("submitPoolAttesterSlashing", function () { const atterterSlashing: phase0.AttesterSlashing = { attestation1: { - attestingIndices: [0] as List, + attestingIndices: [0], data: generateAttestationData(0, 1, 0, 1), signature: Buffer.alloc(96), }, attestation2: { - attestingIndices: [0] as List, + attestingIndices: [0], data: generateAttestationData(0, 1, 0, 1), signature: Buffer.alloc(96), }, diff --git a/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts b/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts deleted file mode 100644 index 894b64125de9..000000000000 --- a/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -import {config} from "@chainsafe/lodestar-config/default"; -import {CachedBeaconStateAllForks, PubkeyIndexMap} from "@chainsafe/lodestar-beacon-state-transition"; -import {List, toHexString} from "@chainsafe/ssz"; -import {expect, use} from "chai"; -import chaiAsPromised from "chai-as-promised"; -import sinon, {SinonStubbedInstance, SinonStubbedMember} from "sinon"; -import {getBeaconStateApi} from "../../../../../../src/api/impl/beacon/state"; -import * as stateApiUtils from "../../../../../../src/api/impl/beacon/state/utils"; -import {generateState} from "../../../../../utils/state"; -import {generateValidator, generateValidators} from "../../../../../utils/validator"; -import {BeaconChain} from "../../../../../../src/chain"; -import {StubbedBeaconDb} from "../../../../../utils/stub"; -import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; - -use(chaiAsPromised); - -const validatorId1 = toHexString(Buffer.alloc(48, 1)); -const validatorId2 = toHexString(Buffer.alloc(48, 2)); - -describe("beacon api impl - state - validators", function () { - let resolveStateIdStub: SinonStubbedMember; - let toValidatorResponseStub: SinonStubbedMember; - let dbStub: StubbedBeaconDb; - let chainStub: SinonStubbedInstance; - let server: ApiImplTestModules; - - before(function () { - server = setupApiImplTestServer(); - }); - - beforeEach(function () { - resolveStateIdStub = server.sandbox.stub(stateApiUtils, "resolveStateId"); - toValidatorResponseStub = server.sandbox.stub(stateApiUtils, "toValidatorResponse"); - toValidatorResponseStub.returns({ - index: 1, - balance: 3200000, - status: "active_ongoing", - validator: generateValidator(), - }); - dbStub = server.dbStub; - chainStub = server.chainStub; - }); - - afterEach(function () { - server.sandbox.restore(); - }); - - describe("get validators", function () { - it("indices filter", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.onCall(0).returns({ - pubkey2index: ({ - get: () => 0, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState", {indices: [0, 1, 123]}); - expect(validators.length).to.equal(2); - }); - - // TODO: Make a normal test without stubs - it.skip("status filter", async function () { - const numValidators = 10; - resolveStateIdStub.resolves(generateState({validators: generateValidators(numValidators)})); - toValidatorResponseStub.onFirstCall().returns({ - index: 1, - balance: 3200000, - status: "exited_slashed", - validator: generateValidator(), - }); - for (let i = 0; i < 10; i++) { - chainStub.getHeadState.onCall(i).returns({ - pubkey2index: ({ - get: () => i, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - } - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState", { - statuses: ["pending_initialized"], - }); - expect(validators.length).to.equal(9); - }); - - it("statuses and indices filters", async function () { - const numValidators = 10; - const validators = generateValidators(numValidators); - validators[0].exitEpoch = 0; - resolveStateIdStub.resolves(generateState({validators})); - for (let i = 0; i < 10; i++) { - chainStub.getHeadState.onCall(i).returns({ - pubkey2index: ({ - get: () => i, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - } - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: stateValidators} = await api.getStateValidators("someState", { - indices: [0, 1, 2, 123], - statuses: ["pending_initialized"], - }); - expect(stateValidators.length).to.equal(3); - }); - - it("success", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState"); - expect(validators.length).to.equal(10); - }); - }); - - describe("get validator", function () { - it("validator by index not found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - await expect(api.getStateValidator("someState", 15)).to.be.rejectedWith("Validator not found"); - }); - it("validator by index found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - expect(await api.getStateValidator("someState", 1)).to.not.be.null; - }); - it.skip("validator by root not found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.returns({ - pubkey2index: ({ - get: () => undefined, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - await expect(api.getStateValidator("someState", validatorId1)).to.be.rejectedWith("Validator not found"); - }); - it("validator by root found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.returns({ - pubkey2index: ({ - get: () => 2, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - expect(await api.getStateValidator("someState", validatorId1)).to.not.be.null; - }); - }); - - describe("get validators balances", function () { - it.skip("indices filters", async function () { - resolveStateIdStub.resolves( - generateState({ - validators: generateValidators(10), - balances: Array.from({length: 10}, () => 10) as List, - }) - ); - const pubkey2IndexStub = sinon.createStubInstance(PubkeyIndexMap); - pubkey2IndexStub.get.withArgs(validatorId1).returns(3); - pubkey2IndexStub.get.withArgs(validatorId2).returns(25); - chainStub.getHeadState.returns({ - pubkey2index: (pubkey2IndexStub as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: balances} = await api.getStateValidatorBalances("somestate", [1, 24, validatorId1, validatorId2]); - expect(balances.length).to.equal(2); - expect(balances[0].index).to.equal(1); - expect(balances[1].index).to.equal(3); - }); - - it("no filters", async function () { - resolveStateIdStub.resolves( - generateState({ - validators: generateValidators(10), - balances: Array.from({length: 10}, () => 10) as List, - }) - ); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: balances} = await api.getStateValidatorBalances("somestate"); - expect(balances.length).to.equal(10); - expect(balances[0].index).to.equal(0); - expect(balances[0].balance.toString()).to.equal("10"); - }); - }); -}); diff --git a/packages/lodestar/test/unit/api/impl/node/node.test.ts b/packages/lodestar/test/unit/api/impl/node/node.test.ts index b16a45776ccf..12e579517c00 100644 --- a/packages/lodestar/test/unit/api/impl/node/node.test.ts +++ b/packages/lodestar/test/unit/api/impl/node/node.test.ts @@ -3,6 +3,7 @@ import {Connection} from "libp2p"; import {getNodeApi} from "../../../../../src/api/impl/node"; import sinon, {SinonStubbedInstance} from "sinon"; +import {BitArray} from "@chainsafe/ssz"; import {createPeerId, INetwork, Network} from "../../../../../src/network"; import {BeaconSync, IBeaconSync} from "../../../../../src/sync"; import {createKeypairFromPeerId, ENR} from "@chainsafe/discv5/lib"; @@ -58,8 +59,8 @@ describe("node api implementation", function () { networkStub.metadata = { get json(): altair.Metadata { return { - attnets: [true], - syncnets: [], + attnets: BitArray.fromBoolArray([true]), + syncnets: BitArray.fromBitLen(0), seqNumber: BigInt(1), }; }, diff --git a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts index 1b610967bb33..b4a270b60086 100644 --- a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts @@ -2,7 +2,6 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {use, expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {config} from "@chainsafe/lodestar-config/default"; -import {createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkChoice} from "@chainsafe/lodestar-fork-choice"; import {IBeaconChain} from "../../../../../../src/chain"; @@ -10,13 +9,13 @@ import {LocalClock} from "../../../../../../src/chain/clock"; import {FAR_FUTURE_EPOCH} from "../../../../../../src/constants"; import {getValidatorApi} from "../../../../../../src/api/impl/validator"; import {ApiModules} from "../../../../../../src/api/impl/types"; -import {generateInitialMaxBalances} from "../../../../../utils/balances"; import {generateState} from "../../../../../utils/state"; import {IBeaconSync} from "../../../../../../src/sync"; import {generateValidators} from "../../../../../utils/validator"; import {StubbedBeaconDb} from "../../../../../utils/stub"; import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; import {testLogger} from "../../../../../utils/logger"; +import {createCachedBeaconStateTest} from "../../../../../utils/cachedBeaconState"; import {ssz} from "@chainsafe/lodestar-types"; import {MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; @@ -39,7 +38,7 @@ describe("get proposers api impl", function () { syncStub = server.syncStub; chainStub.clock = server.sandbox.createStubInstance(LocalClock); chainStub.forkChoice = server.sandbox.createStubInstance(ForkChoice); - chainStub.getCanonicalBlockAtSlot.resolves(ssz.phase0.SignedBeaconBlock.defaultValue()); + chainStub.getCanonicalBlockAtSlot.resolves(ssz.phase0.SignedBeaconBlock.defaultValue); dbStub = server.dbStub; modules = { chain: server.chainStub, @@ -66,11 +65,11 @@ describe("get proposers api impl", function () { activationEpoch: 0, exitEpoch: FAR_FUTURE_EPOCH, }), - balances: generateInitialMaxBalances(config, 25), + balances: Array.from({length: 25}, () => MAX_EFFECTIVE_BALANCE), }, config ); - const cachedState = createCachedBeaconState(config, state); + const cachedState = createCachedBeaconStateTest(state, config); chainStub.getHeadStateAtCurrentEpoch.resolves(cachedState); sinon.stub(cachedState.epochCtx, "getBeaconProposer").returns(1); const {data: result} = await api.getProposerDuties(0); diff --git a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts index 907328f5ecb1..9579f32c410e 100644 --- a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts @@ -1,23 +1,24 @@ import {expect} from "chai"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; -import {allForks, BLSPubkey, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {getPubkeysForIndex, getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils"; +import {toHexString} from "@chainsafe/ssz"; +import {BLSPubkey, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; describe("api / impl / validator / utils", () => { const vc = 32; const pubkeys: BLSPubkey[] = []; const indexes: ValidatorIndex[] = []; - let state: TreeBacked; + let state: BeaconStateAllForks; before("Prepare state", () => { - state = ssz.phase0.BeaconState.defaultTreeBacked() as TreeBacked; - const validator = ssz.phase0.Validator.defaultValue(); + state = ssz.phase0.BeaconState.defaultViewDU as BeaconStateAllForks; + const validator = ssz.phase0.Validator.defaultValue; const validators = state.validators; for (let i = 0; i < vc; i++) { indexes.push(i); const pubkey = Buffer.alloc(48, i); pubkeys.push(pubkey); - validators.push({...validator, pubkey}); + validators.push(ssz.phase0.Validator.toViewDU({...validator, pubkey})); } }); @@ -25,11 +26,4 @@ describe("api / impl / validator / utils", () => { const pubkeysRes = getPubkeysForIndices(state.validators, indexes); expect(pubkeysRes.map(toHexString)).to.deep.equal(pubkeys.map(toHexString)); }); - - it("getPubkeysForIndex", () => { - for (const index of indexes) { - const pubkeyRes = getPubkeysForIndex(state.validators, index); - expect(toHexString(pubkeyRes)).to.deep.equal(toHexString(pubkeys[index]), `Wrong pubkey for index ${index}`); - } - }); }); diff --git a/packages/lodestar/test/unit/chain/blocks/verifyBlock.test.ts b/packages/lodestar/test/unit/chain/blocks/verifyBlock.test.ts index 949060e226a2..e2d5b7c433a8 100644 --- a/packages/lodestar/test/unit/chain/blocks/verifyBlock.test.ts +++ b/packages/lodestar/test/unit/chain/blocks/verifyBlock.test.ts @@ -17,7 +17,7 @@ describe("chain / blocks / verifyBlock", function () { const currentSlot = 1; beforeEach(function () { - block = ssz.phase0.SignedBeaconBlock.defaultValue(); + block = ssz.phase0.SignedBeaconBlock.defaultValue; block.message.slot = currentSlot; forkChoice = sinon.createStubInstance(ForkChoice); diff --git a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts index 15dc25094fd2..d87a74226ac6 100644 --- a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts +++ b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts @@ -50,7 +50,7 @@ describe("block assembly", function () { const state = generateCachedState({slot: 1}); sinon.stub(state.epochCtx, "getBeaconProposer").returns(2); regenStub.getBlockSlotState.resolves(state); - beaconDB.depositDataRoot.getTreeBacked.resolves(ssz.phase0.DepositDataRootList.defaultTreeBacked()); + beaconDB.depositDataRoot.getDepositRootTreeAtIndex.resolves(ssz.phase0.DepositDataRootList.defaultViewDU); assembleBodyStub.resolves(generateEmptyBlock().body); const eth1 = sandbox.createStubInstance(Eth1ForBlockProduction); diff --git a/packages/lodestar/test/unit/chain/factory/block/body.test.ts b/packages/lodestar/test/unit/chain/factory/block/body.test.ts index 4e9cbbdb9885..d66b0de59e04 100644 --- a/packages/lodestar/test/unit/chain/factory/block/body.test.ts +++ b/packages/lodestar/test/unit/chain/factory/block/body.test.ts @@ -46,12 +46,12 @@ describe("blockAssembly - body", function () { it("should generate block body", async function () { const {chain, opPool, dbStub, aggregatedAttestationPool} = getStubs(); opPool.getSlashingsAndExits.returns([ - [ssz.phase0.AttesterSlashing.defaultValue()], - [ssz.phase0.ProposerSlashing.defaultValue()], + [ssz.phase0.AttesterSlashing.defaultValue], + [ssz.phase0.ProposerSlashing.defaultValue], [generateEmptySignedVoluntaryExit()], ]); aggregatedAttestationPool.getAttestationsForBlock.returns([generateEmptyAttestation()]); - dbStub.depositDataRoot.getTreeBacked.resolves(ssz.phase0.DepositDataRootList.defaultTreeBacked()); + dbStub.depositDataRoot.getDepositRootTreeAtIndex.resolves(ssz.phase0.DepositDataRootList.defaultViewDU); const result = await assembleBody(chain, generateCachedState(), { randaoReveal: Buffer.alloc(96, 0), diff --git a/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts index dd9ab11c2e63..da7574c5cb30 100644 --- a/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts @@ -11,11 +11,12 @@ import { getTemporaryBlockHeader, phase0, CachedBeaconStateAllForks, - createCachedBeaconState, getEffectiveBalanceIncrementsZeroed, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {expect} from "chai"; -import {List, toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState"; import {generateValidators} from "../../../utils/validator"; describe("LodestarForkChoice", function () { @@ -29,7 +30,7 @@ describe("LodestarForkChoice", function () { exitEpoch: FAR_FUTURE_EPOCH, withdrawableEpoch: FAR_FUTURE_EPOCH, }), - balances: Array.from({length: 3}, () => 0) as List, + balances: Array.from({length: 3}, () => 0), // Jan 01 2020 genesisTime: 1577836800, }, @@ -47,7 +48,7 @@ describe("LodestarForkChoice", function () { let state: CachedBeaconStateAllForks; before(() => { - state = createCachedBeaconState(config, anchorState); + state = createCachedBeaconStateTest(anchorState, config); }); beforeEach(() => { @@ -69,7 +70,7 @@ describe("LodestarForkChoice", function () { targetBlock.message.parentRoot = finalizedRoot; // const targetState = runStateTransition(anchorState, targetBlock); - targetBlock.message.stateRoot = config.getForkTypes(targetState.slot).BeaconState.hashTreeRoot(targetState); + targetBlock.message.stateRoot = targetState.hashTreeRoot(); const {block: orphanedBlock, state: orphanedState} = makeChild({block: targetBlock, state: targetState}, 36); const {block: parentBlock, state: parentState} = makeChild({block: targetBlock, state: targetState}, 37); const {block: childBlock, state: childState} = makeChild({block: parentBlock, state: parentState}, 38); @@ -115,28 +116,34 @@ describe("LodestarForkChoice", function () { const block08 = generateSignedBlock({message: {slot: 8}}); block08.message.parentRoot = finalizedRoot; const state08 = runStateTransition(anchorState, block08); - block08.message.stateRoot = config.getForkTypes(state08.slot).BeaconState.hashTreeRoot(state08); + block08.message.stateRoot = state08.hashTreeRoot(); const {block: block12, state: state12} = makeChild({block: block08, state: state08}, 12); const {block: block16, state: state16} = makeChild({block: block12, state: state12}, 16); - state16.currentJustifiedCheckpoint = { + state16.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), epoch: 1, - }; + }); const {block: block20, state: state20} = makeChild({block: block16, state: state16}, 20); const {block: block24, state: state24} = makeChild({block: block20, state: state20}, 24); - state24.finalizedCheckpoint = {root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), epoch: 1}; - state24.currentJustifiedCheckpoint = { + state24.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ + root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), + epoch: 1, + }); + state24.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), epoch: 2, - }; + }); const {block: block28, state: state28} = makeChild({block: block24, state: state24}, 28); const {block: block32, state: state32} = makeChild({block: block28, state: state28}, 32); - state32.finalizedCheckpoint = {root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), epoch: 2}; - state32.currentJustifiedCheckpoint = { + state32.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ + root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), + epoch: 2, + }); + state32.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block24.message), epoch: 3, - }; + }); forkChoice.updateTime(128); forkChoice.onBlock(block08.message, state08, {justifiedBalances, blockDelaySec: 0}); @@ -184,7 +191,7 @@ describe("LodestarForkChoice", function () { const targetBlock = generateSignedBlock({message: {slot: 32}}); targetBlock.message.parentRoot = finalizedRoot; const targetState = runStateTransition(anchorState, targetBlock); - targetBlock.message.stateRoot = config.getForkTypes(targetState.slot).BeaconState.hashTreeRoot(targetState); + targetBlock.message.stateRoot = targetState.hashTreeRoot(); const {block: orphanedBlock, state: orphanedState} = makeChild({block: targetBlock, state: targetState}, 33); const {block: parentBlock, state: parentState} = makeChild({block: targetBlock, state: targetState}, 34); const {block: childBlock, state: childState} = makeChild({block: parentBlock, state: parentState}, 35); @@ -209,24 +216,23 @@ describe("LodestarForkChoice", function () { }); // lightweight state transtion function for this test -function runStateTransition( - preState: TreeBacked, - signedBlock: phase0.SignedBeaconBlock -): TreeBacked { +function runStateTransition(preState: BeaconStateAllForks, signedBlock: phase0.SignedBeaconBlock): BeaconStateAllForks { // Clone state because process slots and block are not pure const postState = preState.clone(); // Process slots (including those with no blocks) since block - allForks.processSlots(createCachedBeaconState(config, postState), signedBlock.message.slot); + allForks.processSlots(createCachedBeaconStateTest(postState, config), signedBlock.message.slot); // processBlock - postState.latestBlockHeader = getTemporaryBlockHeader(config, signedBlock.message); + postState.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU( + getTemporaryBlockHeader(config, signedBlock.message) + ); return postState; } // create a child block/state from a parent block/state and a provided slot function makeChild( - parent: {block: phase0.SignedBeaconBlock; state: TreeBacked}, + parent: {block: phase0.SignedBeaconBlock; state: BeaconStateAllForks}, slot: Slot -): {block: phase0.SignedBeaconBlock; state: TreeBacked} { +): {block: phase0.SignedBeaconBlock; state: BeaconStateAllForks} { const childBlock = generateSignedBlock({message: {slot}}); const parentRoot = ssz.phase0.BeaconBlock.hashTreeRoot(parent.block.message); childBlock.message.parentRoot = parentRoot; @@ -241,7 +247,7 @@ export function createIndexedAttestation( validatorIndex: ValidatorIndex ): phase0.IndexedAttestation { return { - attestingIndices: [validatorIndex] as List, + attestingIndices: [validatorIndex], data: { slot: block.message.slot, index: 0, diff --git a/packages/lodestar/test/unit/chain/genesis/genesis.test.ts b/packages/lodestar/test/unit/chain/genesis/genesis.test.ts index bfaa42ad5d68..7d47d043552b 100644 --- a/packages/lodestar/test/unit/chain/genesis/genesis.test.ts +++ b/packages/lodestar/test/unit/chain/genesis/genesis.test.ts @@ -2,7 +2,7 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {SecretKey, PublicKey} from "@chainsafe/bls"; -import {DOMAIN_DEPOSIT} from "@chainsafe/lodestar-params"; +import {DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; import {config} from "@chainsafe/lodestar-config/default"; import { computeDomain, @@ -121,7 +121,7 @@ function generateDeposit(index: ValidatorIndex, secretKey: SecretKey, publicKey: const depositMessage = { pubkey: publicKey.toBytes(), withdrawalCredentials: Buffer.alloc(32, index), - amount: 32e18, + amount: MAX_EFFECTIVE_BALANCE, }; const signingRoot = computeSigningRoot(ssz.phase0.DepositMessage, depositMessage, domain); const signature = secretKey.sign(signingRoot); diff --git a/packages/lodestar/test/unit/chain/lightclient/proof.test.ts b/packages/lodestar/test/unit/chain/lightclient/proof.test.ts index 5be711e846a2..9481f01f8a40 100644 --- a/packages/lodestar/test/unit/chain/lightclient/proof.test.ts +++ b/packages/lodestar/test/unit/chain/lightclient/proof.test.ts @@ -1,8 +1,7 @@ +import {BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; import {SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; import {altair, ssz} from "@chainsafe/lodestar-types"; -import {verifyMerkleBranch} from "@chainsafe/lodestar-utils"; -import {hash} from "@chainsafe/persistent-merkle-tree"; -import {TreeBacked} from "@chainsafe/ssz"; +import {verifyMerkleBranch, hash} from "@chainsafe/lodestar-utils"; import {expect} from "chai"; import {getNextSyncCommitteeBranch, getSyncCommitteesWitness} from "../../../../src/chain/lightClient/proofs"; @@ -11,17 +10,18 @@ const nextSyncCommitteeGindex = 55; const syncCommitteesGindex = 27; describe("chain / lightclient / proof", () => { - let state: TreeBacked; + let state: BeaconStateAltair; let stateRoot: Uint8Array; const currentSyncCommittee = fillSyncCommittee(Buffer.alloc(48, 0xbb)); const nextSyncCommittee = fillSyncCommittee(Buffer.alloc(48, 0xcc)); before("random state", () => { - state = ssz.altair.BeaconState.defaultTreeBacked(); - state.currentSyncCommittee = currentSyncCommittee; - state.nextSyncCommittee = nextSyncCommittee; - stateRoot = ssz.altair.BeaconState.hashTreeRoot(state); + state = ssz.altair.BeaconState.defaultViewDU; + state.currentSyncCommittee = ssz.altair.SyncCommittee.toViewDU(currentSyncCommittee); + state.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(nextSyncCommittee); + // Note: .hashTreeRoot() automatically commits() + stateRoot = state.hashTreeRoot(); }); it("SyncCommittees proof", () => { diff --git a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index ba28dde8f68f..0f04b15a7bdb 100644 --- a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -3,7 +3,6 @@ import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {createIChainForkConfig, defaultChainConfig} from "@chainsafe/lodestar-config"; import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List} from "@chainsafe/ssz"; import {expect} from "chai"; import {ssz, phase0} from "@chainsafe/lodestar-types"; import { @@ -14,7 +13,9 @@ import { import {InsertOutcome} from "../../../../src/chain/opPools/types"; import {generateAttestation, generateEmptyAttestation} from "../../../utils/attestation"; import {generateCachedState} from "../../../utils/state"; +import {renderBitArray} from "../../../utils/render"; import sinon from "sinon"; +import {BitArray} from "@chainsafe/ssz"; describe("AggregatedAttestationPool", function () { let pool: AggregatedAttestationPool; @@ -71,7 +72,14 @@ describe("MatchingDataAttestationGroup", function () { const attestationSeed = generateEmptyAttestation(); const attestationDataRoot = ssz.phase0.AttestationData.serialize(attestationSeed.data); let sk1: SecretKey; - const attestation1 = {...attestationSeed, ...{aggregationBits: [true, true, false] as List}}; + const attestation1: phase0.Attestation = { + ...attestationSeed, + ...{aggregationBits: BitArray.fromBoolArray([true, true, false])}, + }; + + function attestationFromBits(bitsBoolArr: boolean[]): phase0.Attestation { + return {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray(bitsBoolArr)}}; + } before(async () => { await initBLS(); @@ -88,7 +96,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - new data, getAttestations() return 2", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -96,7 +104,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - new data, remove existing attestation, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, true, true] as List}}; + const attestation2 = attestationFromBits([true, true, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set(committee)}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -104,7 +112,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - already known, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, false] as List}}; + const attestation2 = attestationFromBits([true, false, false]); // attestingIndices is subset of an existing one const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100])}); expect(result).to.be.equal(InsertOutcome.AlreadyKnown, "incorrect InsertOutcome"); @@ -113,19 +121,18 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - aggregate into existing attestation, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [false, false, true] as List}}; + const attestation2 = attestationFromBits([false, false, true]); const sk2 = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); attestation2.signature = sk2.sign(attestationDataRoot).toBytes(); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([300])}); expect(result).to.be.equal(InsertOutcome.Aggregated, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); expect(attestations.length).to.be.equal(1, "expect exactly 1 aggregated attestation"); - expect(attestations[0].aggregationBits).to.be.deep.equal([true, true, true], "incorrect aggregationBits"); - const aggregatedSignature = bls.Signature.fromBytes( - attestations[0].signature.valueOf() as Uint8Array, - undefined, - true + expect(renderBitArray(attestations[0].aggregationBits)).to.be.deep.equal( + renderBitArray(BitArray.fromBoolArray([true, true, true])), + "incorrect aggregationBits" ); + const aggregatedSignature = bls.Signature.fromBytes(attestations[0].signature, undefined, true); expect( aggregatedSignature.verifyAggregate([sk1.toPublicKey(), sk2.toPublicKey()], attestationDataRoot) ).to.be.equal(true, "invalid aggregated signature"); @@ -152,7 +159,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("getAttestationsForBlock - return 2", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestationsForBlock(new Set([200])); @@ -174,7 +181,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("getAttestations", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -184,8 +191,9 @@ describe("MatchingDataAttestationGroup", function () { describe("aggregateInto", function () { const attestationSeed = generateEmptyAttestation(); - const attestation1 = {...attestationSeed, ...{aggregationBits: [false, true] as List}}; - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false] as List}}; + const attestation1 = {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray([false, true])}}; + const attestation2 = {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray([true, false])}}; + const mergedBitArray = BitArray.fromBoolArray([true, true]); // = [false, true] + [true, false] const attestationDataRoot = ssz.phase0.AttestationData.serialize(attestationSeed.data); let sk1: SecretKey; let sk2: SecretKey; @@ -201,15 +209,21 @@ describe("aggregateInto", function () { const attWithIndex1 = {attestation: attestation1, attestingIndices: new Set([100])}; const attWithIndex2 = {attestation: attestation2, attestingIndices: new Set([200])}; aggregateInto(attWithIndex1, attWithIndex2); - expect(attWithIndex1.attestingIndices).to.be.deep.equal(new Set([100, 200]), "invalid aggregated attestingIndices"); - expect(attWithIndex1.attestation.aggregationBits).to.be.deep.equal([true, true], "invalid aggregationBits"); - const aggregatedSignature = bls.Signature.fromBytes( - attWithIndex1.attestation.signature.valueOf() as Uint8Array, - undefined, - true + expect(setToArr(attWithIndex1.attestingIndices)).to.be.deep.equal( + [100, 200], + "invalid aggregated attestingIndices" ); + expect(renderBitArray(attWithIndex1.attestation.aggregationBits)).to.be.deep.equal( + renderBitArray(mergedBitArray), + "invalid aggregationBits" + ); + const aggregatedSignature = bls.Signature.fromBytes(attWithIndex1.attestation.signature, undefined, true); expect( aggregatedSignature.verifyAggregate([sk1.toPublicKey(), sk2.toPublicKey()], attestationDataRoot) ).to.be.equal(true, "invalid aggregated signature"); }); }); + +function setToArr(set: Set): T[] { + return Array.from(set.values()); +} diff --git a/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts b/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts index af311f2fadee..07a5cc1263a1 100644 --- a/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts @@ -49,12 +49,8 @@ describe("chain / opPools / SyncCommitteeMessagePool", function () { expect(contribution.subcommitteeIndex).to.be.equal(subcommitteeIndex); const newIndices = [...newIndicesInSubSyncCommittee, indexInSubcommittee]; const aggregationBits = contribution.aggregationBits; - for (let index = 0; index < aggregationBits.length; index++) { - if (newIndices.includes(index)) { - expect(aggregationBits[index]).to.be.true; - } else { - expect(aggregationBits[index]).to.be.false; - } + for (let index = 0; index < aggregationBits.bitLen; index++) { + expect(aggregationBits.get(index)).to.equal(newIndices.includes(index), `Wrong bit value index ${index}`); } } }); diff --git a/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts b/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts index 17fe4f10bbd1..11d60097b841 100644 --- a/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts @@ -1,19 +1,20 @@ import {altair, ssz} from "@chainsafe/lodestar-types"; import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; - +import bls, {SecretKey} from "@chainsafe/bls"; +import {BitArray} from "@chainsafe/ssz"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {expect} from "chai"; import { aggregate, - contributionToFast, replaceIfBetter, SyncContributionAndProofPool, SyncContributionFast, } from "../../../../src/chain/opPools/syncContributionAndProofPool"; import {generateContributionAndProof, generateEmptyContribution} from "../../../utils/contributionAndProof"; import {InsertOutcome} from "../../../../src/chain/opPools/types"; -import bls, {SecretKey} from "@chainsafe/bls"; +import {EMPTY_SIGNATURE} from "../../../../src/constants"; +import {renderBitArray} from "../../../utils/render"; describe("chain / opPools / SyncContributionAndProofPool", function () { let cache: SyncContributionAndProofPool; @@ -36,26 +37,28 @@ describe("chain / opPools / SyncContributionAndProofPool", function () { }); cache.add(newContributionAndProof, syncCommitteeParticipants); const aggregate = cache.getAggregate(slot, beaconBlockRoot); - expect(ssz.altair.SyncAggregate.equals(aggregate, ssz.altair.SyncAggregate.defaultValue())).to.be.false; + expect(ssz.altair.SyncAggregate.equals(aggregate, ssz.altair.SyncAggregate.defaultValue)).to.be.false; // TODO Test it's correct. Modify the contributions above so they have 1 bit set to true - expect(aggregate.syncCommitteeBits.length).to.be.equal(32); + expect(aggregate.syncCommitteeBits.bitLen).to.be.equal(32); }); }); describe("replaceIfBetter", function () { + const numParticipants = 2; let bestContribution: SyncContributionFast; // const subnetSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); beforeEach(() => { bestContribution = { - syncSubcommitteeBits: [true, true, false, false, false, false, false, false], - numParticipants: 2, - syncSubcommitteeSignature: new Uint8Array(0), + syncSubcommitteeBits: BitArray.fromBoolArray([true, true, false, false, false, false, false, false]), + numParticipants, + syncSubcommitteeSignature: EMPTY_SIGNATURE, }; }); + it("less participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[0] = true; - expect(replaceIfBetter(bestContribution, contribution, 0)).to.be.equal( + contribution.aggregationBits.set(0, true); + expect(replaceIfBetter(bestContribution, contribution, numParticipants - 1)).to.be.equal( InsertOutcome.NotBetterThan, "less participant item should not replace the best contribution" ); @@ -63,9 +66,7 @@ describe("replaceIfBetter", function () { it("same participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[0] = true; - contribution.aggregationBits[7] = true; - expect(replaceIfBetter(bestContribution, contribution, 2)).to.be.equal( + expect(replaceIfBetter(bestContribution, contribution, numParticipants)).to.be.equal( InsertOutcome.NotBetterThan, "same participant item should not replace the best contribution" ); @@ -73,41 +74,17 @@ describe("replaceIfBetter", function () { it("more participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[3] = true; - contribution.aggregationBits[4] = true; - contribution.aggregationBits[5] = true; - expect(replaceIfBetter(bestContribution, contribution, 3)).to.be.equal( + const numParticipantsNew = numParticipants + 1; + + expect(replaceIfBetter(bestContribution, contribution, numParticipantsNew)).to.be.equal( InsertOutcome.NewData, "more participant item should replace the best contribution" ); - expect(bestContribution.syncSubcommitteeBits).to.be.deep.equal( - [false, false, false, true, true, true, false, false], - "incorect subcommittees" - ); - expect(bestContribution.numParticipants).to.be.equal(3, "incorrect numParticipants"); - }); -}); - -describe("contributionToFast", function () { - let sk1: SecretKey; - before(async () => { - await initBLS(); - sk1 = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); - }); - - it("convert a contribution to SyncContributionFast", () => { - const contribution = generateEmptyContribution(); - contribution.aggregationBits[3] = true; - contribution.aggregationBits[4] = true; - contribution.aggregationBits[5] = true; - contribution.signature = sk1.sign(Buffer.alloc(32)).toBytes(); - const fast = contributionToFast(contribution, 3); - expect(fast.syncSubcommitteeBits).to.be.deep.equal( - [false, false, false, true, true, true, false, false], + expect(renderBitArray(bestContribution.syncSubcommitteeBits)).to.be.deep.equal( + renderBitArray(contribution.aggregationBits), "incorect subcommittees" ); - expect(fast.numParticipants).to.be.equal(3, "incorrect numParticipants"); - // no need to check sygnature + expect(bestContribution.numParticipants).to.be.equal(numParticipantsNew, "incorrect numParticipants"); }); }); @@ -130,7 +107,7 @@ describe("aggregate", function () { for (let subnet = 0; subnet < numSubnet; subnet++) { bestContributionBySubnet.set(subnet, { // first participation of each subnet is true - syncSubcommitteeBits: [true, false, false, false, false, false, false, false], + syncSubcommitteeBits: BitArray.fromBoolArray([true, false, false, false, false, false, false, false]), numParticipants: 1, syncSubcommitteeSignature: sks[subnet].sign(blockRoot).toBytes(), }); @@ -142,12 +119,15 @@ describe("aggregate", function () { // first participation of each subnet is true expectSyncCommittees[subnet * 8] = true; } - expect(syncAggregate.syncCommitteeBits).to.be.deep.equal(expectSyncCommittees, "incorrect sync committees"); + expect(renderBitArray(syncAggregate.syncCommitteeBits)).to.be.deep.equal( + renderBitArray(BitArray.fromBoolArray(expectSyncCommittees)), + "incorrect sync committees" + ); expect( bls.verifyAggregate( testSks.map((sk) => sk.toPublicKey().toBytes()), blockRoot, - syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array + syncAggregate.syncCommitteeSignature ) ).to.be.equal(true, "invalid aggregated signature"); }); diff --git a/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts b/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts index 81fa49c7115f..29e02aa5032c 100644 --- a/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts +++ b/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts @@ -3,12 +3,13 @@ import {IEpochShuffling} from "@chainsafe/lodestar-beacon-state-transition"; import {StateContextCache} from "../../../../src/chain/stateCache"; import {generateCachedState} from "../../../utils/state"; import {ZERO_HASH} from "../../../../src/constants"; -import {ByteVector, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; +import {Root} from "@chainsafe/lodestar-types"; describe("StateContextCache", function () { let cache: StateContextCache; - let key1: ByteVector, key2: ByteVector; + let key1: Root, key2: Root; const shuffling: IEpochShuffling = { epoch: 0, activeIndices: [], diff --git a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts index 186f91a0edd7..b776cbef82fe 100644 --- a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts @@ -95,8 +95,8 @@ describe("chain / validation / aggregateAndProof", () => { const {chain, signedAggregateAndProof} = getValidData(); // Unset all aggregationBits const {aggregationBits} = signedAggregateAndProof.message.aggregate; - for (let i = 0, len = aggregationBits.length; i < len; i++) { - aggregationBits[i] = false; + for (let i = 0, len = aggregationBits.bitLen; i < len; i++) { + aggregationBits.set(i, false); } await expectError(chain, signedAggregateAndProof, AttestationErrorCode.EMPTY_AGGREGATION_BITFIELD); @@ -141,8 +141,8 @@ describe("chain / validation / aggregateAndProof", () => { const bitIndex = 1; const {chain, signedAggregateAndProof} = getValidData({bitIndex}); // Change the bit index so the signature is validated against a different pubkey - signedAggregateAndProof.message.aggregate.aggregationBits[bitIndex] = false; - signedAggregateAndProof.message.aggregate.aggregationBits[bitIndex + 1] = true; + signedAggregateAndProof.message.aggregate.aggregationBits.set(bitIndex, false); + signedAggregateAndProof.message.aggregate.aggregationBits.set(bitIndex + 1, true); await expectError(chain, signedAggregateAndProof, AttestationErrorCode.INVALID_SIGNATURE); }); diff --git a/packages/lodestar/test/unit/chain/validation/attestation.test.ts b/packages/lodestar/test/unit/chain/validation/attestation.test.ts index eeca2567101e..5e5c2e5cd1a2 100644 --- a/packages/lodestar/test/unit/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/unit/chain/validation/attestation.test.ts @@ -1,5 +1,6 @@ import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; import {IBeaconChain} from "../../../../src/chain"; import {AttestationErrorCode} from "../../../../src/chain/errors"; import {validateGossipAttestation} from "../../../../src/chain/validation"; @@ -65,7 +66,7 @@ describe("chain / validation / attestation", () => { // Unset the single aggregationBits const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); - attestation.aggregationBits[bitIndex] = false; + attestation.aggregationBits.set(bitIndex, false); await expectError(chain, attestation, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET); }); @@ -74,7 +75,7 @@ describe("chain / validation / attestation", () => { // Set an extra bit in the attestation const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); - attestation.aggregationBits[bitIndex + 1] = true; + attestation.aggregationBits.set(bitIndex + 1, true); await expectError(chain, attestation, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET); }); @@ -98,7 +99,10 @@ describe("chain / validation / attestation", () => { it("WRONG_NUMBER_OF_AGGREGATION_BITS", async () => { const {chain, attestation, subnet} = getValidData(); // Increase the length of aggregationBits beyond the committee size - attestation.aggregationBits[attestation.aggregationBits.length] = false; + attestation.aggregationBits = new BitArray( + attestation.aggregationBits.uint8Array, + attestation.aggregationBits.bitLen + 1 + ); await expectError(chain, attestation, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS); }); @@ -123,8 +127,8 @@ describe("chain / validation / attestation", () => { const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); // Change the bit index so the signature is validated against a different pubkey - attestation.aggregationBits[bitIndex] = false; - attestation.aggregationBits[bitIndex + 1] = true; + attestation.aggregationBits.set(bitIndex, false); + attestation.aggregationBits.set(bitIndex + 1, true); await expectError(chain, attestation, subnet, AttestationErrorCode.INVALID_SIGNATURE); }); diff --git a/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts b/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts index 321cc301864e..beec08510bc3 100644 --- a/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts +++ b/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts @@ -11,7 +11,6 @@ import {validateGossipAttesterSlashing} from "../../../../src/chain/validation/a import {AttesterSlashingErrorCode} from "../../../../src/chain/errors/attesterSlashingError"; import {OpPool} from "../../../../src/chain/opPools"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; -import {List} from "@chainsafe/ssz"; describe("GossipMessageValidator", () => { const sandbox = sinon.createSandbox(); @@ -35,7 +34,7 @@ describe("GossipMessageValidator", () => { describe("validate attester slashing", () => { it("should return invalid attester slashing - already exisits", async () => { - const attesterSlashing = ssz.phase0.AttesterSlashing.defaultValue(); + const attesterSlashing = ssz.phase0.AttesterSlashing.defaultValue; opPool.hasSeenAttesterSlashing.returns(true); await expectRejectedWithLodestarError( @@ -45,7 +44,7 @@ describe("GossipMessageValidator", () => { }); it("should return invalid attester slashing - invalid", async () => { - const attesterSlashing = ssz.phase0.AttesterSlashing.defaultValue(); + const attesterSlashing = ssz.phase0.AttesterSlashing.defaultValue; await expectRejectedWithLodestarError( validateGossipAttesterSlashing(chainStub, attesterSlashing), @@ -54,17 +53,17 @@ describe("GossipMessageValidator", () => { }); it("should return valid attester slashing", async () => { - const attestationData = ssz.phase0.AttestationData.defaultValue(); + const attestationData = ssz.phase0.AttestationData.defaultValue; const attesterSlashing: phase0.AttesterSlashing = { attestation1: { data: attestationData, signature: Buffer.alloc(96, 0), - attestingIndices: [0] as List, + attestingIndices: [0], }, attestation2: { data: {...attestationData, slot: 1}, // Make it different so it's slashable signature: Buffer.alloc(96, 0), - attestingIndices: [0] as List, + attestingIndices: [0], }, }; diff --git a/packages/lodestar/test/unit/chain/validation/block.test.ts b/packages/lodestar/test/unit/chain/validation/block.test.ts index 5a252ea7ae93..a631448a2f11 100644 --- a/packages/lodestar/test/unit/chain/validation/block.test.ts +++ b/packages/lodestar/test/unit/chain/validation/block.test.ts @@ -22,7 +22,7 @@ describe("gossip block validation", function () { let job: allForks.SignedBeaconBlock; const proposerIndex = 0; const clockSlot = 32; - const block = ssz.phase0.BeaconBlock.defaultValue(); + const block = ssz.phase0.BeaconBlock.defaultValue; block.slot = clockSlot; const signature = EMPTY_SIGNATURE; diff --git a/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts b/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts index 00c46129ada8..f171b41adc9a 100644 --- a/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts +++ b/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts @@ -2,6 +2,7 @@ import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {defaultChainConfig} from "@chainsafe/lodestar-config"; import sinon from "sinon"; import {SinonStubbedInstance} from "sinon"; +import {BitArray} from "@chainsafe/ssz"; import {BeaconChain, IBeaconChain} from "../../../../src/chain"; import {LocalClock} from "../../../../src/chain/clock"; import {SyncCommitteeErrorCode} from "../../../../src/chain/errors/syncCommitteeError"; @@ -12,7 +13,7 @@ import {validateSyncCommitteeGossipContributionAndProof} from "../../../../src/c import * as syncCommitteeUtils from "@chainsafe/lodestar-beacon-state-transition/lib/util/aggregator"; import {SinonStubFn} from "../../../utils/types"; import {generateCachedStateWithPubkeys} from "../../../utils/state"; -import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; +import {SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {createIChainForkConfig} from "@chainsafe/lodestar-config"; import {SeenContributionAndProof} from "../../../../src/chain/seenCache"; @@ -27,6 +28,8 @@ describe("Sync Committee Contribution And Proof validation", function () { const currentSlot = SLOTS_PER_EPOCH * (altairForkEpoch + 1); // eslint-disable-next-line @typescript-eslint/naming-convention const config = createIChainForkConfig(Object.assign({}, defaultChainConfig, {ALTAIR_FORK_EPOCH: altairForkEpoch})); + // all validators have same pubkey + const aggregatorIndex = 15; before(async function () { await initBLS(); @@ -37,7 +40,6 @@ describe("Sync Committee Contribution And Proof validation", function () { (chain as { seenContributionAndProof: SeenContributionAndProof; }).seenContributionAndProof = new SeenContributionAndProof(); - chain.getGenesisTime.returns(Math.floor(Date.now() / 1000)); clockStub = sandbox.createStubInstance(LocalClock); chain.clock = clockStub; clockStub.isCurrentSlotGivenGossipDisparity.returns(true); @@ -52,7 +54,7 @@ describe("Sync Committee Contribution And Proof validation", function () { clockStub.isCurrentSlotGivenGossipDisparity.returns(false); sandbox.stub(clockStub, "currentSlot").get(() => 100); - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: 1}}); + const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: 1}, aggregatorIndex}); await expectRejectedWithLodestarError( validateSyncCommitteeGossipContributionAndProof(chain, signedContributionAndProof), SyncCommitteeErrorCode.NOT_CURRENT_SLOT @@ -62,6 +64,7 @@ describe("Sync Committee Contribution And Proof validation", function () { it("should throw error - subcommitteeIndex is not in allowed range", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ contribution: {slot: currentSlot, subcommitteeIndex: 10000}, + aggregatorIndex, }); await expectRejectedWithLodestarError( @@ -71,7 +74,10 @@ describe("Sync Committee Contribution And Proof validation", function () { }); it("should throw error - there is same contribution with same aggregator and index and slot", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex, + }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); chain.seenContributionAndProof.isKnown = () => true; @@ -82,7 +88,10 @@ describe("Sync Committee Contribution And Proof validation", function () { }); it("should throw error - no participant", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex, + }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); isSyncCommitteeAggregatorStub.returns(false); @@ -94,7 +103,8 @@ describe("Sync Committee Contribution And Proof validation", function () { it("should throw error - invalid aggregator", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ - contribution: {slot: currentSlot, aggregationBits: [true]}, + contribution: {slot: currentSlot, aggregationBits: BitArray.fromSingleBit(SYNC_COMMITTEE_SUBNET_SIZE, 0)}, + aggregatorIndex, }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); @@ -109,20 +119,24 @@ describe("Sync Committee Contribution And Proof validation", function () { * Skip this spec: [REJECT] The aggregator's validator index is within the current sync committee -- i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in state.current_sync_committee.pubkeys. * because we check the aggregator index already and we always sync sync pubkeys with indices */ - it.skip("should throw error - aggregator index is not in sync committee", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + it("should throw error - aggregator index is not in sync committee", async function () { + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex: Infinity, + }); isSyncCommitteeAggregatorStub.returns(true); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); await expectRejectedWithLodestarError( validateSyncCommitteeGossipContributionAndProof(chain, signedContributionAndProof), - SyncCommitteeErrorCode.AGGREGATOR_PUBKEY_UNKNOWN + SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE ); }); it("should throw error - invalid selection_proof signature", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ - contribution: {slot: currentSlot, aggregationBits: [true]}, + contribution: {slot: currentSlot, aggregationBits: BitArray.fromSingleBit(SYNC_COMMITTEE_SUBNET_SIZE, 0)}, + aggregatorIndex, }); isSyncCommitteeAggregatorStub.returns(true); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); diff --git a/packages/lodestar/test/unit/chain/validation/proposerSlashing.test.ts b/packages/lodestar/test/unit/chain/validation/proposerSlashing.test.ts index 50521a55d2f1..b59115a2810e 100644 --- a/packages/lodestar/test/unit/chain/validation/proposerSlashing.test.ts +++ b/packages/lodestar/test/unit/chain/validation/proposerSlashing.test.ts @@ -33,7 +33,7 @@ describe("validate proposer slashing", () => { }); it("should return invalid proposer slashing - existing", async () => { - const proposerSlashing = ssz.phase0.ProposerSlashing.defaultValue(); + const proposerSlashing = ssz.phase0.ProposerSlashing.defaultValue; opPool.hasSeenProposerSlashing.returns(true); await expectRejectedWithLodestarError( @@ -43,7 +43,7 @@ describe("validate proposer slashing", () => { }); it("should return invalid proposer slashing - invalid", async () => { - const proposerSlashing = ssz.phase0.ProposerSlashing.defaultValue(); + const proposerSlashing = ssz.phase0.ProposerSlashing.defaultValue; // Make it invalid proposerSlashing.signedHeader1.message.slot = 1; proposerSlashing.signedHeader2.message.slot = 0; @@ -55,8 +55,8 @@ describe("validate proposer slashing", () => { }); it("should return valid proposer slashing", async () => { - const signedHeader1 = ssz.phase0.SignedBeaconBlockHeader.defaultValue(); - const signedHeader2 = ssz.phase0.SignedBeaconBlockHeader.defaultValue(); + const signedHeader1 = ssz.phase0.SignedBeaconBlockHeader.defaultValue; + const signedHeader2 = ssz.phase0.SignedBeaconBlockHeader.defaultValue; // Make it different, so slashable signedHeader2.message.stateRoot = Buffer.alloc(32, 1); diff --git a/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts b/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts index d6e5551bc44d..3f8c002f5bb8 100644 --- a/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts +++ b/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts @@ -42,7 +42,6 @@ describe("Sync Committee Signature validation", function () { (chain as { seenSyncCommitteeMessages: SeenSyncCommitteeMessages; }).seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); - chain.getGenesisTime.returns(Math.floor(Date.now() / 1000)); clockStub = sandbox.createStubInstance(LocalClock); chain.clock = clockStub; clockStub.isCurrentSlotGivenGossipDisparity.returns(true); diff --git a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts index 65b8203a29e4..7e85d21d458b 100644 --- a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts +++ b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts @@ -3,7 +3,6 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {config} from "@chainsafe/lodestar-config/default"; import { phase0, - createCachedBeaconState, CachedBeaconStateAllForks, computeEpochAtSlot, computeDomain, @@ -22,6 +21,7 @@ import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {DOMAIN_VOLUNTARY_EXIT, FAR_FUTURE_EPOCH, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {PointFormat, SecretKey} from "@chainsafe/bls"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState"; describe("validate voluntary exit", () => { const sandbox = sinon.createSandbox(); @@ -33,13 +33,13 @@ describe("validate voluntary exit", () => { before(() => { const sk = SecretKey.fromKeygen(); - const stateEmpty = ssz.phase0.BeaconState.defaultTreeBacked(); + const stateEmpty = ssz.phase0.BeaconState.defaultValue; // Validator has to be active for long enough stateEmpty.slot = config.SHARD_COMMITTEE_PERIOD * SLOTS_PER_EPOCH; // Add a validator that's active since genesis and ready to exit - stateEmpty.validators[0] = { + const validator = ssz.phase0.Validator.toViewDU({ pubkey: sk.toPublicKey().toBytes(PointFormat.compressed), withdrawalCredentials: Buffer.alloc(32, 0), effectiveBalance: 32e9, @@ -48,7 +48,8 @@ describe("validate voluntary exit", () => { activationEpoch: 0, exitEpoch: FAR_FUTURE_EPOCH, withdrawableEpoch: FAR_FUTURE_EPOCH, - }; + }); + stateEmpty.validators[0] = validator; const voluntaryExit = { epoch: 0, @@ -63,7 +64,7 @@ describe("validate voluntary exit", () => { signedVoluntaryExit = {message: voluntaryExit, signature: sk.sign(signingRoot).toBytes()}; const _state = generateState(stateEmpty, config); - state = createCachedBeaconState(createIBeaconConfig(config, _state.genesisValidatorsRoot), _state); + state = createCachedBeaconStateTest(_state, createIBeaconConfig(config, _state.genesisValidatorsRoot)); }); beforeEach(() => { diff --git a/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts b/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts index 58ff6dce107e..cf96e6e356fd 100644 --- a/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts +++ b/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts @@ -121,7 +121,7 @@ describe("block archive repository", function () { ).to.be.true; expect( spy.withArgs( - encodeKey(Bucket.index_blockArchiveParentRootIndex, block.message.parentRoot.valueOf() as Uint8Array), + encodeKey(Bucket.index_blockArchiveParentRootIndex, block.message.parentRoot), intToBytes(block.message.slot, 8, "be") ).calledOnce ).to.be.true; @@ -139,7 +139,7 @@ describe("block archive repository", function () { ).to.be.true; expect( spy.withArgs( - encodeKey(Bucket.index_blockArchiveParentRootIndex, blocks[0].message.parentRoot.valueOf() as Uint8Array), + encodeKey(Bucket.index_blockArchiveParentRootIndex, blocks[0].message.parentRoot), intToBytes(blocks[0].message.slot, 8, "be") ).calledTwice ).to.be.true; diff --git a/packages/lodestar/test/unit/db/api/repository.test.ts b/packages/lodestar/test/unit/db/api/repository.test.ts index c4239f251b22..504943b304e6 100644 --- a/packages/lodestar/test/unit/db/api/repository.test.ts +++ b/packages/lodestar/test/unit/db/api/repository.test.ts @@ -17,11 +17,9 @@ interface TestType { } // eslint-disable-next-line @typescript-eslint/naming-convention -const TestSSZType = new ContainerType({ - fields: { - bool: ssz.Boolean, - bytes: ssz.Bytes32, - }, +const TestSSZType = new ContainerType({ + bool: ssz.Boolean, + bytes: ssz.Bytes32, }); class TestRepository extends Repository { @@ -104,19 +102,15 @@ describe("database repository", function () { it("should delete given items", async function () { await repository.batchDelete(["1", "2", "3"]); - expect( - controller.batchDelete.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 3)) - .calledOnce - ).to.be.true; + expect(controller.batchDelete.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 3)).calledOnce).to + .be.true; }); it("should delete given items by value", async function () { const item = {bool: true, bytes: Buffer.alloc(32)}; await repository.batchRemove([item, item]); - expect( - controller.batchDelete.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 2)) - .calledOnce - ).to.be.true; + expect(controller.batchDelete.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 2)).calledOnce).to + .be.true; }); it("should add multiple values", async function () { @@ -124,10 +118,8 @@ describe("database repository", function () { {bool: true, bytes: Buffer.alloc(32)}, {bool: false, bytes: Buffer.alloc(32)}, ]); - expect( - controller.batchPut.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 2)) - .calledOnce - ).to.be.true; + expect(controller.batchPut.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 2)).calledOnce).to.be + .true; }); it("should fetch values stream", async function () { diff --git a/packages/lodestar/test/unit/eth1/utils/deposits.test.ts b/packages/lodestar/test/unit/eth1/utils/deposits.test.ts index a65ae2d6f4f4..3d50c16301e2 100644 --- a/packages/lodestar/test/unit/eth1/utils/deposits.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/deposits.test.ts @@ -1,16 +1,15 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; -import {Root, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; import {verifyMerkleBranch} from "@chainsafe/lodestar-utils"; import {filterBy} from "../../../utils/db"; -import {getTreeAtIndex} from "../../../../src/util/tree"; import {Eth1ErrorCode} from "../../../../src/eth1/errors"; import {generateDepositData, generateDepositEvent} from "../../../utils/deposit"; import {generateState} from "../../../utils/state"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {getDeposits, getDepositsWithProofs, DepositGetter} from "../../../../src/eth1/utils/deposits"; +import {DepositTree} from "../../../../src/db/repositories/depositDataRoot"; chai.use(chaiAsPromised); @@ -102,8 +101,8 @@ describe("eth1 / util / deposits", function () { describe("getDepositsWithProofs", () => { it("return empty array if no pending deposits", function () { - const initialValues = [Buffer.alloc(32)] as List; - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct(initialValues); + const initialValues = [Buffer.alloc(32)]; + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU(initialValues); const depositCount = 0; const eth1Data = generateEth1Data(depositCount, depositRootTree); @@ -121,7 +120,7 @@ describe("eth1 / util / deposits", function () { }) ); - const depositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const depositRootTree = ssz.phase0.DepositDataRootList.defaultViewDU; for (const depositEvent of depositEvents) { depositRootTree.push(ssz.phase0.DepositData.hashTreeRoot(depositEvent.depositData)); } @@ -138,10 +137,10 @@ describe("eth1 / util / deposits", function () { expect( verifyMerkleBranch( ssz.phase0.DepositData.hashTreeRoot(deposit.data), - Array.from(deposit.proof).map((p) => p.valueOf() as Uint8Array), + Array.from(deposit.proof).map((p) => p), 33, index, - eth1Data.depositRoot.valueOf() as Uint8Array + eth1Data.depositRoot ), `Wrong merkle proof on deposit ${index}` ).to.be.true; @@ -150,10 +149,10 @@ describe("eth1 / util / deposits", function () { }); }); -function generateEth1Data(depositCount: number, depositRootTree?: TreeBacked>): phase0.Eth1Data { +function generateEth1Data(depositCount: number, depositRootTree?: DepositTree): phase0.Eth1Data { return { blockHash: Buffer.alloc(32), - depositRoot: depositRootTree ? getTreeAtIndex(depositRootTree, depositCount - 1).hashTreeRoot() : Buffer.alloc(32), + depositRoot: depositRootTree ? depositRootTree.sliceTo(depositCount - 1).hashTreeRoot() : Buffer.alloc(32), depositCount, }; } diff --git a/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts b/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts index 97523d212bec..b1f674a7f870 100644 --- a/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts @@ -2,17 +2,17 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {pick} from "lodash"; import {Root, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; +import {toHex} from "@chainsafe/lodestar-utils"; import {iteratorFromArray} from "../../../utils/interator"; -import {mapToObj} from "../../../utils/map"; import { getEth1DataForBlocks, getDepositsByBlockNumber, getDepositRootByDepositCount, } from "../../../../src/eth1/utils/eth1Data"; +import {Eth1Block} from "../../../../src/eth1/interface"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {Eth1ErrorCode} from "../../../../src/eth1/errors"; -import {Eth1Block} from "../../../../src/eth1/interface"; +import {DepositTree} from "../../../../src/db/repositories/depositDataRoot"; chai.use(chaiAsPromised); @@ -21,7 +21,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: string; blocks: Eth1Block[]; deposits: phase0.DepositEvent[]; - depositRootTree: TreeBacked>; + depositRootTree: DepositTree; lastProcessedDepositBlockNumber: number; expectedEth1Data?: Partial[]; error?: Eth1ErrorCode; @@ -48,8 +48,8 @@ describe("eth1 / util / getEth1DataForBlocks", function () { const lastProcessedDepositBlockNumber = expectedEth1Data[expectedEth1Data.length - 1].blockNumber; // Pre-fill the depositTree with roots for all deposits - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct( - Array.from({length: deposits[deposits.length - 1].index + 1}, (_, i) => Buffer.alloc(32, i)) as List + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU( + Array.from({length: deposits[deposits.length - 1].index + 1}, (_, i) => Buffer.alloc(32, i)) ); return { @@ -67,7 +67,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "No deposits yet, should throw with NoDepositsForBlockRange", blocks: [getMockBlock({blockNumber: 0})], deposits: [], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU, lastProcessedDepositBlockNumber: 0, error: Eth1ErrorCode.NO_DEPOSITS_FOR_BLOCK_RANGE, }; @@ -78,7 +78,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "With deposits and no deposit roots, should throw with NotEnoughDepositRoots", blocks: [getMockBlock({blockNumber: 0})], deposits: [getMockDeposit({blockNumber: 0, index: 0})], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU, lastProcessedDepositBlockNumber: 0, error: Eth1ErrorCode.NOT_ENOUGH_DEPOSIT_ROOTS, }; @@ -89,7 +89,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "Empty case", blocks: [], deposits: [], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU, lastProcessedDepositBlockNumber: 0, expectedEth1Data: [], }; @@ -207,12 +207,12 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { interface ITestCase { id: string; depositCounts: number[]; - depositRootTree: TreeBacked>; + depositRootTree: DepositTree; expectedMap: Map; } const fullRootMap = new Map(); - const fullDepositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const fullDepositRootTree = ssz.phase0.DepositDataRootList.defaultViewDU; for (let i = 0; i < 10; i++) { fullDepositRootTree.push(Buffer.alloc(32, i)); fullRootMap.set(fullDepositRootTree.length, fullDepositRootTree.hashTreeRoot()); @@ -242,7 +242,7 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { }; }, () => { - const emptyTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const emptyTree = ssz.phase0.DepositDataRootList.defaultViewDU; return { id: "Empty case", depositCounts: [], @@ -256,11 +256,19 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { const {id, depositCounts, depositRootTree, expectedMap} = testCase(); it(id, function () { const map = getDepositRootByDepositCount(depositCounts, depositRootTree); - expect(mapToObj(map)).to.deep.equal(mapToObj(expectedMap)); + expect(renderDepositRootByDepositCount(map)).to.deep.equal(renderDepositRootByDepositCount(expectedMap)); }); } }); +function renderDepositRootByDepositCount(map: Map): Record { + const data: Record = {}; + for (const [key, root] of Object.entries(map)) { + data[key] = toHex(root); + } + return data; +} + function getMockBlock({blockNumber}: {blockNumber: number}): Eth1Block { return { blockNumber, diff --git a/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts b/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts index 3038be9fa5c8..2cad0289b798 100644 --- a/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; import {config} from "@chainsafe/lodestar-config/default"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {generateState} from "../../../utils/state"; import {filterBy} from "../../../utils/db"; import { @@ -82,10 +82,10 @@ describe("eth1 / util / eth1Vote", function () { for (const testCase of testCases) { const {id, eth1DataVotesInState, votesToConsider, expectedEth1Vote} = testCase(); - it(`get eth1 vote: ${id}`, async function () { - const state = generateState({slot: 5, eth1DataVotes: eth1DataVotesInState as List}); + it(id, async function () { + const state = generateState({slot: 5, eth1DataVotes: eth1DataVotesInState}); const eth1Vote = pickEth1Vote(state, votesToConsider); - expect(ssz.phase0.Eth1Data.equals(eth1Vote, expectedEth1Vote)).to.be.true; + expect(ssz.phase0.Eth1Data.toJson(eth1Vote)).to.deep.equal(ssz.phase0.Eth1Data.toJson(expectedEth1Vote)); }); } }); @@ -94,7 +94,7 @@ describe("eth1 / util / eth1Vote", function () { // Function array to scope votes in each test case defintion const testCases: (() => { id: string; - state: TreeBacked; + state: BeaconStateAllForks; eth1Datas: IEth1DataWithTimestamp[]; expectedVotesToConsider: phase0.Eth1Data[]; })[] = [ @@ -132,7 +132,10 @@ describe("eth1 / util / eth1Vote", function () { filterBy(eth1Datas, timestampRange, (eth1Data) => eth1Data.timestamp); const votesToConsider = await getEth1VotesToConsider(config, state, eth1DataGetter); - expect(votesToConsider).to.deep.equal(expectedVotesToConsider); + + expect(votesToConsider.map((eth1Data) => ssz.phase0.Eth1Data.toJson(eth1Data))).to.deep.equal( + expectedVotesToConsider.map((eth1Data) => ssz.phase0.Eth1Data.toJson(eth1Data)) + ); }); } }); @@ -161,7 +164,7 @@ function getEth1DataBlock(eth1DataBlock: Partial): IEth1 * @param config * @param state */ -function getTimestampInRange(config: IChainForkConfig, state: TreeBacked): number { +function getTimestampInRange(config: IChainForkConfig, state: BeaconStateAllForks): number { const {SECONDS_PER_ETH1_BLOCK, ETH1_FOLLOW_DISTANCE} = config; const periodStart = votingPeriodStartTime(config, state); return periodStart - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE; diff --git a/packages/lodestar/test/unit/metrics/utils.ts b/packages/lodestar/test/unit/metrics/utils.ts index d5d0473a254d..aebab37fd166 100644 --- a/packages/lodestar/test/unit/metrics/utils.ts +++ b/packages/lodestar/test/unit/metrics/utils.ts @@ -3,6 +3,6 @@ import {ssz} from "@chainsafe/lodestar-types"; import {createMetrics, IMetrics} from "../../../src/metrics"; export function createMetricsTest(): IMetrics { - const state = ssz.phase0.BeaconState.defaultValue(); + const state = ssz.phase0.BeaconState.defaultViewDU; return createMetrics({enabled: true, timeout: 12000}, config, state); } diff --git a/packages/lodestar/test/unit/network/attestationService.test.ts b/packages/lodestar/test/unit/network/attestationService.test.ts index 77f7928a8886..d24b3c350b0f 100644 --- a/packages/lodestar/test/unit/network/attestationService.test.ts +++ b/packages/lodestar/test/unit/network/attestationService.test.ts @@ -1,4 +1,3 @@ -import {allForks} from "@chainsafe/lodestar-types"; import { ATTESTATION_SUBNET_COUNT, EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION, @@ -6,14 +5,13 @@ import { SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; -import {getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; // eslint-disable-next-line no-restricted-imports import * as mathUtils from "@chainsafe/lodestar-utils/lib/math"; import * as shuffleUtils from "../../../src/util/shuffle"; import sinon, {SinonStubbedInstance} from "sinon"; import {MockBeaconChain} from "../../utils/mocks/chain/chain"; import {generateState} from "../../utils/state"; -import {TreeBacked} from "@chainsafe/ssz"; import {testLogger} from "../../utils/logger"; import {expect} from "chai"; import {SinonStubFn} from "../../utils/types"; @@ -39,7 +37,7 @@ describe("AttnetsService", function () { let metadata: MetadataController; let chain: IBeaconChain; - let state: allForks.BeaconState; + let state: BeaconStateAllForks; const logger = testLogger(); const subscription: CommitteeSubscription = { validatorIndex: 2021, @@ -66,7 +64,7 @@ describe("AttnetsService", function () { genesisTime: Math.floor(Date.now() / 1000), chainId: 0, networkId: BigInt(0), - state: state as TreeBacked, + state: state as BeaconStateAllForks, config, }); // load getCurrentSlot first, vscode not able to debug without this diff --git a/packages/lodestar/test/unit/network/peers/metastore.test.ts b/packages/lodestar/test/unit/network/peers/metastore.test.ts index 1f5e0ffbd2c8..1cd4bc18675a 100644 --- a/packages/lodestar/test/unit/network/peers/metastore.test.ts +++ b/packages/lodestar/test/unit/network/peers/metastore.test.ts @@ -4,6 +4,8 @@ import {ReqRespEncoding} from "../../../../src/network/reqresp"; import {expect} from "chai"; import PeerId from "peer-id"; import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; +import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import MetadataBook from "libp2p/src/peer-store/metadata-book"; import ProtoBook from "libp2p/src/peer-store/proto-book"; @@ -47,10 +49,10 @@ describe("Libp2pPeerMetadataStore", function () { it("can store and retrieve metadata", function () { const store = new Libp2pPeerMetadataStore(metabookStub); const value: altair.Metadata = { - attnets: Array.from({length: 64}, () => true), + attnets: BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT), seqNumber: BigInt(20), // This will serialize fine, to 0x00 - syncnets: [], + syncnets: BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT), }; store.metadata.set(peerId, value); const result = store.metadata.get(peerId); @@ -64,6 +66,6 @@ describe("Libp2pPeerMetadataStore", function () { store.rpcScore.set(peerId, value); const result = store.rpcScore.get(peerId); - expect(ssz.Number64.equals(result as number, value)).to.be.true; + expect(result).to.equal(value); }); }); diff --git a/packages/lodestar/test/unit/network/peers/priorization.test.ts b/packages/lodestar/test/unit/network/peers/priorization.test.ts index a7ffb1046ca0..287cbb2fcbb7 100644 --- a/packages/lodestar/test/unit/network/peers/priorization.test.ts +++ b/packages/lodestar/test/unit/network/peers/priorization.test.ts @@ -1,6 +1,8 @@ import {expect} from "chai"; import PeerId from "peer-id"; import {phase0, altair} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; +import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {prioritizePeers} from "../../../../src/network/peers/utils/prioritizePeers"; import {getAttnets} from "../../../utils/network"; import {RequestedSubnet} from "../../../../src/network/peers/utils"; @@ -14,6 +16,7 @@ describe("network / peers / priorization", () => { peer.toB58String = () => `peer-${i}`; peers.push(peer); } + const none = BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT); const testCases: { id: string; @@ -38,7 +41,7 @@ describe("network / peers / priorization", () => { }, { id: "Don't request a subnet query when enough peers are connected to it", - connectedPeers: [{id: peers[0], syncnets: [], attnets: getAttnets([3]), score: 0}], + connectedPeers: [{id: peers[0], syncnets: none, attnets: getAttnets([3]), score: 0}], activeAttnets: [3], activeSyncnets: [], opts: {targetPeers: 1, maxPeers: 1}, @@ -52,10 +55,10 @@ describe("network / peers / priorization", () => { { id: "Disconnect worst peers without duty", connectedPeers: [ - {id: peers[0], syncnets: [], attnets: getAttnets([3]), score: 0}, - {id: peers[1], syncnets: [], attnets: [], score: 0}, - {id: peers[2], syncnets: [], attnets: [], score: -20}, - {id: peers[3], syncnets: [], attnets: [], score: -40}, + {id: peers[0], syncnets: none, attnets: getAttnets([3]), score: 0}, + {id: peers[1], syncnets: none, attnets: none, score: 0}, + {id: peers[2], syncnets: none, attnets: none, score: -20}, + {id: peers[3], syncnets: none, attnets: none, score: -40}, ], activeAttnets: [3], activeSyncnets: [], @@ -71,14 +74,14 @@ describe("network / peers / priorization", () => { { id: "Complete example: Disconnect peers and request a subnet query", connectedPeers: [ - {id: peers[0], syncnets: [], attnets: getAttnets([0, 1, 2]), score: 0}, - {id: peers[1], syncnets: [], attnets: getAttnets([0, 1, 2]), score: -10}, - {id: peers[2], syncnets: [], attnets: getAttnets([0, 1]), score: 0}, - {id: peers[3], syncnets: [], attnets: getAttnets([0]), score: -10}, - {id: peers[4], syncnets: [], attnets: getAttnets([2]), score: 0}, - {id: peers[5], syncnets: [], attnets: getAttnets([0, 2]), score: -20}, - {id: peers[6], syncnets: [], attnets: getAttnets([1, 2, 3]), score: 0}, - {id: peers[7], syncnets: [], attnets: getAttnets([1, 2]), score: -10}, + {id: peers[0], syncnets: none, attnets: getAttnets([0, 1, 2]), score: 0}, + {id: peers[1], syncnets: none, attnets: getAttnets([0, 1, 2]), score: -10}, + {id: peers[2], syncnets: none, attnets: getAttnets([0, 1]), score: 0}, + {id: peers[3], syncnets: none, attnets: getAttnets([0]), score: -10}, + {id: peers[4], syncnets: none, attnets: getAttnets([2]), score: 0}, + {id: peers[5], syncnets: none, attnets: getAttnets([0, 2]), score: -20}, + {id: peers[6], syncnets: none, attnets: getAttnets([1, 2, 3]), score: 0}, + {id: peers[7], syncnets: none, attnets: getAttnets([1, 2]), score: -10}, ], activeAttnets: [1, 3], activeSyncnets: [], diff --git a/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts b/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts index eee689c7d208..d602f29b75c6 100644 --- a/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts +++ b/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts @@ -1,5 +1,7 @@ import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {ssz} from "@chainsafe/lodestar-types"; +import {toHex} from "@chainsafe/lodestar-utils"; +import {BitArray} from "@chainsafe/ssz"; import {expect} from "chai"; import {deserializeEnrSubnets} from "../../../../../src/network/peers/utils/enrSubnetsDeserialize"; @@ -17,7 +19,9 @@ describe("ENR syncnets", () => { it(`Deserialize syncnet ${bytes}`, () => { const bytesBuf = Buffer.from(bytes, "hex"); - expect(ssz.altair.SyncSubnets.deserialize(bytesBuf)).to.deep.equal(bools); + expect(toHex(ssz.altair.SyncSubnets.deserialize(bytesBuf).uint8Array)).to.deep.equal( + toHex(BitArray.fromBoolArray(bools).uint8Array) + ); expect( deserializeEnrSubnets(bytesBuf, SYNC_COMMITTEE_SUBNET_COUNT).slice(0, SYNC_COMMITTEE_SUBNET_COUNT) diff --git a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts index 7c9c95d6d473..7428fba6fe82 100644 --- a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts +++ b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts @@ -51,13 +51,13 @@ describe("network / reqresp / sszSnappy / decode", () => { id: "if it read more than maxEncodedLen", type: ssz.phase0.Ping, error: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, - chunks: [Buffer.from(varint.encode(ssz.phase0.Ping.getMinSerializedLength())), Buffer.alloc(100)], + chunks: [Buffer.from(varint.encode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)], }, { id: "if failed ssz snappy input malformed", type: ssz.phase0.Status, error: SszSnappyErrorCode.DECOMPRESSOR_ERROR, - chunks: [Buffer.from(varint.encode(ssz.phase0.Status.minSize())), Buffer.from("wrong snappy data")], + chunks: [Buffer.from(varint.encode(ssz.phase0.Status.minSize)), Buffer.from("wrong snappy data")], }, ]; diff --git a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts index 128e63f64cbd..b0aa73f339e7 100644 --- a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts +++ b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {RequestOrIncomingResponseBody, RequestOrResponseType} from "../../../../../../src/network/reqresp/types"; @@ -57,11 +57,11 @@ export const sszSnappySignedBeaconBlockPhase0: ISszSnappyTestData, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [] as phase0.ProposerSlashing[], + attesterSlashings: [] as phase0.AttesterSlashing[], + attestations: [] as phase0.Attestation[], + deposits: [] as phase0.Deposit[], + voluntaryExits: [] as phase0.SignedVoluntaryExit[], }, }, signature: Buffer.alloc(96, 0xda), @@ -83,7 +83,7 @@ export const sszSnappySignedBeaconBlockAltair: ISszSnappyTestData { +export function generateRoots(count: number, offset = 0): Root[] { const roots: Root[] = []; for (let i = 0; i < count; i++) { roots.push(Buffer.alloc(32, i + offset)); } - return roots as List; + return roots; } /** diff --git a/packages/lodestar/test/unit/sync/backfill/verify.test.ts b/packages/lodestar/test/unit/sync/backfill/verify.test.ts index c8edad0fa685..46992ce04d4d 100644 --- a/packages/lodestar/test/unit/sync/backfill/verify.test.ts +++ b/packages/lodestar/test/unit/sync/backfill/verify.test.ts @@ -1,5 +1,4 @@ import {BackfillSyncErrorCode, BackfillSyncError} from "./../../../../src/sync/backfill/errors"; -import {Json} from "@chainsafe/ssz"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {config} from "@chainsafe/lodestar-config/default"; import {phase0, ssz} from "@chainsafe/lodestar-types"; @@ -24,7 +23,7 @@ describe("backfill sync - verify block sequence", function () { it("should fail with sequence not anchored", function () { const blocks = getBlocks(); - const wrongAncorRoot = ssz.Root.defaultValue(); + const wrongAncorRoot = ssz.Root.defaultValue; expect(() => verifyBlockSequence(beaconConfig, blocks, wrongAncorRoot)).to.throw( BackfillSyncErrorCode.NOT_ANCHORED ); @@ -45,9 +44,9 @@ describe("backfill sync - verify block sequence", function () { //first 4 mainnet blocks function getBlocks(): phase0.SignedBeaconBlock[] { - const json = JSON.parse(readFileSync(path.join(__dirname, "./blocks.json"), "utf-8")) as Json[]; + const json = JSON.parse(readFileSync(path.join(__dirname, "./blocks.json"), "utf-8")) as unknown[]; return json.map((b) => { - return ssz.phase0.SignedBeaconBlock.fromJson(b, {case: "snake"}); + return ssz.phase0.SignedBeaconBlock.fromJson(b); }); } }); diff --git a/packages/lodestar/test/unit/sync/unknownBlock.test.ts b/packages/lodestar/test/unit/sync/unknownBlock.test.ts index deac5ae081f7..463c7f7c122f 100644 --- a/packages/lodestar/test/unit/sync/unknownBlock.test.ts +++ b/packages/lodestar/test/unit/sync/unknownBlock.test.ts @@ -16,9 +16,9 @@ describe("sync / UnknownBlockSync", () => { it("fetch and process multiple unknown block parents", async () => { const peer = getValidPeerId(); const peerIdStr = peer.toB58String(); - const blockA = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); - const blockB = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); - const blockC = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); + const blockA = ssz.phase0.SignedBeaconBlock.defaultValue; + const blockB = ssz.phase0.SignedBeaconBlock.defaultValue; + const blockC = ssz.phase0.SignedBeaconBlock.defaultValue; blockA.message.slot = 1; blockB.message.slot = 2; blockC.message.slot = 3; diff --git a/packages/lodestar/test/utils/aggregationBits.ts b/packages/lodestar/test/utils/aggregationBits.ts deleted file mode 100644 index 0793e16ba0da..000000000000 --- a/packages/lodestar/test/utils/aggregationBits.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {List} from "@chainsafe/ssz"; - -/** Create a filled bits array with a single true bit */ -export function toSingleBit(len: number, index: number): List { - const bits = ([] as boolean[]) as List; - for (let i = 0; i < len; i++) bits[i] = false; - bits[index] = true; - return bits; -} diff --git a/packages/lodestar/test/utils/attestation.ts b/packages/lodestar/test/utils/attestation.ts index b0e9ce5c1c26..ba5d4b9a3f63 100644 --- a/packages/lodestar/test/utils/attestation.ts +++ b/packages/lodestar/test/utils/attestation.ts @@ -1,9 +1,9 @@ -import {List} from "@chainsafe/ssz"; import {CommitteeIndex, Epoch, Slot, phase0} from "@chainsafe/lodestar-types"; import crypto from "node:crypto"; import deepmerge from "deepmerge"; import {isPlainObject} from "@chainsafe/lodestar-utils"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; +import {BitArray} from "@chainsafe/ssz"; /** * Generates a fake attestation data for test purposes. @@ -38,7 +38,7 @@ export function generateAttestationData( export function generateAttestation(override: RecursivePartial = {}): phase0.Attestation { return deepmerge>( { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: 0, index: 0, diff --git a/packages/lodestar/test/utils/balances.ts b/packages/lodestar/test/utils/balances.ts deleted file mode 100644 index 32f0b76a5c84..000000000000 --- a/packages/lodestar/test/utils/balances.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {List} from "@chainsafe/ssz"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; - -export function generateInitialMaxBalances(config: IChainForkConfig, count?: number): List { - return Array.from({length: count ?? config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT}, () => - Number(MAX_EFFECTIVE_BALANCE) - ) as List; -} diff --git a/packages/lodestar/test/utils/block.ts b/packages/lodestar/test/utils/block.ts index eb13d4004c66..5cfd6ff3c347 100644 --- a/packages/lodestar/test/utils/block.ts +++ b/packages/lodestar/test/utils/block.ts @@ -2,7 +2,6 @@ import {ssz} from "@chainsafe/lodestar-types"; import {config as defaultConfig} from "@chainsafe/lodestar-config/default"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {allForks, phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; import {IProtoBlock, ExecutionStatus} from "@chainsafe/lodestar-fork-choice"; import {isPlainObject} from "@chainsafe/lodestar-utils"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; @@ -24,11 +23,11 @@ export function generateEmptyBlock(): phase0.BeaconBlock { depositCount: 0, }, graffiti: Buffer.alloc(32), - proposerSlashings: ([] as phase0.ProposerSlashing[]) as List, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [] as phase0.ProposerSlashing[], + attesterSlashings: [] as phase0.AttesterSlashing[], + attestations: [] as phase0.Attestation[], + deposits: [] as phase0.Deposit[], + voluntaryExits: [] as phase0.SignedVoluntaryExit[], }, }; } diff --git a/packages/lodestar/test/utils/cachedBeaconState.ts b/packages/lodestar/test/utils/cachedBeaconState.ts new file mode 100644 index 000000000000..7cac0c6b5c86 --- /dev/null +++ b/packages/lodestar/test/utils/cachedBeaconState.ts @@ -0,0 +1,14 @@ +import { + BeaconStateAllForks, + BeaconStateCache, + createCachedBeaconState, + createEmptyEpochContextImmutableData, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; + +export function createCachedBeaconStateTest( + state: T, + chainConfig: IChainForkConfig +): T & BeaconStateCache { + return createCachedBeaconState(state, createEmptyEpochContextImmutableData(chainConfig, state)); +} diff --git a/packages/lodestar/test/utils/contributionAndProof.ts b/packages/lodestar/test/utils/contributionAndProof.ts index 35efdd03fd71..0e5b8bb0a0c4 100644 --- a/packages/lodestar/test/utils/contributionAndProof.ts +++ b/packages/lodestar/test/utils/contributionAndProof.ts @@ -2,12 +2,12 @@ import {EMPTY_SIGNATURE} from "@chainsafe/lodestar-beacon-state-transition"; import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {altair} from "@chainsafe/lodestar-types"; import {isPlainObject, RecursivePartial} from "@chainsafe/lodestar-utils"; -import {fromHexString, List} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import deepmerge from "deepmerge"; export function generateEmptyContribution(): altair.SyncCommitteeContribution { return { - aggregationBits: Array.from({length: SYNC_COMMITTEE_SUBNET_SIZE}, () => false) as List, + aggregationBits: BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_SIZE), beaconBlockRoot: Buffer.alloc(32), signature: fromHexString( "99cb82bc69b4111d1a828963f0316ec9aa38c4e9e041a8afec86cd20dfe9a590999845bf01d4689f3bbe3df54e48695e081f1216027b577c7fccf6ab0a4fcc75faf8009c6b55e518478139f604f542d138ae3bc34bad01ee6002006d64c4ff82" diff --git a/packages/lodestar/test/utils/errors.ts b/packages/lodestar/test/utils/errors.ts index b949fc62f594..1cff5e683036 100644 --- a/packages/lodestar/test/utils/errors.ts +++ b/packages/lodestar/test/utils/errors.ts @@ -1,6 +1,5 @@ import {expect} from "chai"; import {LodestarError, mapValues} from "@chainsafe/lodestar-utils"; -import {Json} from "@chainsafe/ssz"; export function expectThrowsLodestarError(fn: () => void, expectedErr: LodestarError | string): void { try { @@ -49,7 +48,7 @@ export function expectLodestarError(err1: LodestarErro expect(errMeta1).to.deep.equal(errMeta2, "Wrong LodestarError metadata"); } -export function getErrorMetadata(err: LodestarError | Error | Json): Json { +export function getErrorMetadata(err: LodestarError | Error | unknown): unknown { if (err instanceof LodestarError) { return mapValues(err.getMetadata(), (value) => getErrorMetadata(value as any)); } else if (err instanceof Error) { diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index 229857dba231..653b43d258e0 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -1,10 +1,10 @@ import {AbortController} from "@chainsafe/abort-controller"; import sinon from "sinon"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; -import {allForks, Number64, Root, Slot, ssz, Uint16, Uint64} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/ssz"; +import {allForks, UintNum64, Root, Slot, ssz, Uint16, UintBn64} from "@chainsafe/lodestar-types"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {CheckpointWithHex, IForkChoice, IProtoBlock, ExecutionStatus} from "@chainsafe/lodestar-fork-choice"; @@ -36,19 +36,20 @@ import {ExecutionEngineDisabled} from "../../../../src/executionEngine"; import {ReqRespBlockResponse} from "../../../../src/network/reqresp/types"; import {testLogger} from "../../logger"; import {ReprocessController} from "../../../../src/chain/reprocess"; +import {createCachedBeaconStateTest} from "@chainsafe/lodestar-beacon-state-transition/test/utils/state"; /* eslint-disable @typescript-eslint/no-empty-function */ export interface IMockChainParams { - genesisTime?: Number64; + genesisTime?: UintNum64; chainId: Uint16; - networkId: Uint64; - state: TreeBacked; + networkId: UintBn64; + state: BeaconStateAllForks; config: IBeaconConfig; } export class MockBeaconChain implements IBeaconChain { - readonly genesisTime: Number64; + readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; readonly eth1 = new Eth1ForBlockProductionDisabled(); readonly executionEngine = new ExecutionEngineDisabled(); @@ -60,7 +61,7 @@ export class MockBeaconChain implements IBeaconChain { stateCache: StateContextCache; checkpointStateCache: CheckpointStateCache; chainId: Uint16; - networkId: Uint64; + networkId: UintBn64; clock: IBeaconClock; regen: IStateRegenerator; emitter: ChainEventEmitter; @@ -81,7 +82,7 @@ export class MockBeaconChain implements IBeaconChain { readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof = new SeenContributionAndProof(); - private state: TreeBacked; + private state: BeaconStateAllForks; private abortController: AbortController; constructor({genesisTime, chainId, networkId, state, config}: IMockChainParams) { @@ -123,11 +124,11 @@ export class MockBeaconChain implements IBeaconChain { } getHeadState(): CachedBeaconStateAllForks { - return createCachedBeaconState(this.config, this.state); + return createCachedBeaconStateTest(this.state, this.config); } async getHeadStateAtCurrentEpoch(): Promise { - return createCachedBeaconState(this.config, this.state); + return createCachedBeaconStateTest(this.state, this.config); } async getCanonicalBlockAtSlot(slot: Slot): Promise { @@ -144,10 +145,6 @@ export class MockBeaconChain implements IBeaconChain { })); } - getGenesisTime(): Number64 { - return Math.floor(Date.now() / 1000); - } - async receiveBlock(): Promise {} async processBlock(): Promise {} async processChainSegment(): Promise {} @@ -179,7 +176,7 @@ export class MockBeaconChain implements IBeaconChain { } function mockForkChoice(): IForkChoice { - const root = ssz.Root.defaultValue() as Uint8Array; + const root = ssz.Root.defaultValue as Uint8Array; const rootHex = toHexString(root); const block: IProtoBlock = { slot: 0, diff --git a/packages/lodestar/test/utils/network.ts b/packages/lodestar/test/utils/network.ts index 1eb403d1a7ac..394cc8869969 100644 --- a/packages/lodestar/test/utils/network.ts +++ b/packages/lodestar/test/utils/network.ts @@ -1,6 +1,7 @@ import PeerId from "peer-id"; import {Multiaddr} from "multiaddr"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; import {Network} from "../../src/network"; import {NodejsNode} from "../../src/network/nodejs"; import {createPeerId} from "../../src/network"; @@ -41,10 +42,10 @@ export function onPeerDisconnect(network: Network): Promise { /** * Generate valid filled attnets BitVector */ -export function getAttnets(subnetIds: number[] = []): boolean[] { - const attnets = new Array(ATTESTATION_SUBNET_COUNT).fill(false); +export function getAttnets(subnetIds: number[] = []): BitArray { + const attnets = BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT); for (const subnetId of subnetIds) { - attnets[subnetId] = true; + attnets.set(subnetId, true); } return attnets; } @@ -52,10 +53,10 @@ export function getAttnets(subnetIds: number[] = []): boolean[] { /** * Generate valid filled syncnets BitVector */ -export function getSyncnets(subnetIds: number[] = []): boolean[] { - const attnets = new Array(SYNC_COMMITTEE_SUBNET_COUNT).fill(false); +export function getSyncnets(subnetIds: number[] = []): BitArray { + const syncnets = BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT); for (const subnetId of subnetIds) { - attnets[subnetId] = true; + syncnets.set(subnetId, true); } - return attnets; + return syncnets; } diff --git a/packages/lodestar/test/utils/node/beacon.ts b/packages/lodestar/test/utils/node/beacon.ts index 2702473d0d3a..c212d0e1f7fe 100644 --- a/packages/lodestar/test/utils/node/beacon.ts +++ b/packages/lodestar/test/utils/node/beacon.ts @@ -6,6 +6,8 @@ import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig, createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ILogger, RecursivePartial} from "@chainsafe/lodestar-utils"; import {LevelDbController} from "@chainsafe/lodestar-db"; +import {phase0} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {BeaconNode} from "../../../src/node"; import {createNodeJsLibp2p} from "../../../src/network/nodejs"; import {createPeerId} from "../../../src/network"; @@ -16,8 +18,6 @@ import {defaultOptions} from "../../../src/node/options"; import {BeaconDb} from "../../../src/db"; import {testLogger} from "../logger"; import {InteropStateOpts} from "../../../src/node/utils/interop/state"; -import {TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0} from "@chainsafe/lodestar-types"; export async function getDevBeaconNode( opts: { @@ -27,7 +27,7 @@ export async function getDevBeaconNode( logger?: ILogger; peerId?: PeerId; peerStoreDir?: string; - anchorState?: TreeBacked; + anchorState?: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; } & InteropStateOpts ): Promise { diff --git a/packages/lodestar/test/utils/node/simTest.ts b/packages/lodestar/test/utils/node/simTest.ts index 40316b430300..adb3e9a0f6a4 100644 --- a/packages/lodestar/test/utils/node/simTest.ts +++ b/packages/lodestar/test/utils/node/simTest.ts @@ -11,11 +11,11 @@ import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-pa import {Epoch, Slot} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {ILogger, mapValues} from "@chainsafe/lodestar-utils"; +import {toHexString} from "@chainsafe/ssz"; import {BeaconNode} from "../../../src"; import {ChainEvent} from "../../../src/chain"; import {linspace} from "../../../src/util/numpy"; import {RegenCaller} from "../../../src/chain/regen"; -import {toHexString} from "@chainsafe/ssz"; /* eslint-disable no-console */ @@ -69,7 +69,7 @@ export function simTestInfoTracker(bn: BeaconNode, logger: ILogger): () => void // Recover the pre-epoch transition state, use any random caller for regen const checkpointState = await bn.chain.regen.getCheckpointState(checkpoint, RegenCaller.onForkChoiceFinalized); const lastSlot = computeStartSlotAtEpoch(checkpoint.epoch) - 1; - const lastStateRoot = checkpointState.stateRoots[lastSlot % SLOTS_PER_HISTORICAL_ROOT]; + const lastStateRoot = checkpointState.stateRoots.get(lastSlot % SLOTS_PER_HISTORICAL_ROOT); const lastState = await bn.chain.regen.getState(toHexString(lastStateRoot), RegenCaller.onForkChoiceFinalized); logParticipation(lastState); } @@ -91,7 +91,7 @@ export function simTestInfoTracker(bn: BeaconNode, logger: ILogger): () => void function sumAttestationBits(block: allForks.BeaconBlock): number { return Array.from(block.body.attestations).reduce( - (total, att) => total + Array.from(att.aggregationBits).filter(Boolean).length, + (total, att) => total + att.aggregationBits.getTrueBitIndexes().length, 0 ); } diff --git a/packages/lodestar/test/utils/render.ts b/packages/lodestar/test/utils/render.ts new file mode 100644 index 000000000000..9001bf0e6c05 --- /dev/null +++ b/packages/lodestar/test/utils/render.ts @@ -0,0 +1,5 @@ +import {BitArray} from "@chainsafe/ssz"; + +export function renderBitArray(bitArray: BitArray): string { + return Buffer.from(bitArray.uint8Array).toString("hex"); +} diff --git a/packages/lodestar/test/utils/state.ts b/packages/lodestar/test/utils/state.ts index 7bcb26cf7ef5..1f35427045c7 100644 --- a/packages/lodestar/test/utils/state.ts +++ b/packages/lodestar/test/utils/state.ts @@ -1,8 +1,14 @@ import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; -import {CachedBeaconStateAllForks, createCachedBeaconState, phase0} from "@chainsafe/lodestar-beacon-state-transition"; -import {List, TreeBacked} from "@chainsafe/ssz"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + createCachedBeaconState, + phase0, + PubkeyIndexMap, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {BitArray} from "@chainsafe/ssz"; import {allForks, altair, Root, ssz} from "@chainsafe/lodestar-types"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import { EPOCHS_PER_HISTORICAL_VECTOR, EPOCHS_PER_SLASHINGS_VECTOR, @@ -23,13 +29,10 @@ import {initBLS} from "@chainsafe/lodestar-cli/src/util"; */ type TestBeaconState = Partial; -const phase0States = new Map>(); -const altairStates = new Map>(); - /** * Generate beaconState, by default it will generate a mostly empty state with "just enough" to be valid-ish * NOTE: All fields can be overridden through `opts`. - * should allow 1st test calling generateState more time since TreeBacked.createValue api is expensive. + * should allow 1st test calling generateState more time since creating a new instance is expensive. * * @param {TestBeaconState} opts * @param config @@ -40,7 +43,7 @@ export function generateState( config = minimalConfig, isAltair = false, withPubkey = false -): TreeBacked { +): BeaconStateAllForks { const validatorOpts = { activationEpoch: 0, effectiveBalance: MAX_EFFECTIVE_BALANCE, @@ -75,21 +78,21 @@ export function generateState( }, blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: ([] as Root[]) as List, + historicalRoots: [] as Root[], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: ([] as phase0.Eth1Data[]) as List, + eth1DataVotes: [] as phase0.Eth1Data[], eth1DepositIndex: 0, - validators: validators as List, - balances: Array.from({length: numValidators}, () => MAX_EFFECTIVE_BALANCE) as List, + validators: validators, + balances: Array.from({length: numValidators}, () => MAX_EFFECTIVE_BALANCE), randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: Array.from({length: EPOCHS_PER_SLASHINGS_VECTOR}, () => BigInt(0)), - previousEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - currentEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - justificationBits: Array.from({length: 4}, () => false), + previousEpochAttestations: [] as phase0.PendingAttestation[], + currentEpochAttestations: [] as phase0.PendingAttestation[], + justificationBits: BitArray.fromBitLen(4), previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, root: ZERO_HASH, @@ -102,46 +105,29 @@ export function generateState( epoch: GENESIS_EPOCH, root: ZERO_HASH, }, + ...opts, }; + if (isAltair) { const defaultAltairState: altair.BeaconState = { - ...ssz.altair.BeaconState.struct_defaultValue(), + ...ssz.altair.BeaconState.defaultValue, ...defaultState, - previousEpochParticipation: [ - ...[0xff, 0xff], - ...Array.from({length: numValidators - 2}, () => 0), - ] as List, - currentEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)] as List, + previousEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)], + currentEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)], currentSyncCommittee: { pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), - aggregatePubkey: ssz.BLSPubkey.defaultValue(), + aggregatePubkey: ssz.BLSPubkey.defaultValue, }, nextSyncCommittee: { pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), - aggregatePubkey: ssz.BLSPubkey.defaultValue(), + aggregatePubkey: ssz.BLSPubkey.defaultValue, }, }; - const state = - altairStates.get(config) ?? - (ssz.altair.BeaconState.createTreeBackedFromStruct(defaultAltairState) as TreeBacked); - altairStates.set(config, state); + return ssz.altair.BeaconState.toViewDU(defaultAltairState); } else { - const state = - phase0States.get(config) ?? - (ssz.phase0.BeaconState.createTreeBackedFromStruct(defaultState) as TreeBacked); - phase0States.set(config, state); - } - const resultState = (isAltair - ? altairStates.get(config)?.clone() - : phase0States.get(config)?.clone()) as TreeBacked; - - for (const key in opts) { - const newValue = opts[key as keyof TestBeaconState]; - // eslint-disable-next-line - resultState[key as keyof TestBeaconState] = (newValue as unknown) as any; + return ssz.phase0.BeaconState.toViewDU(defaultState); } - return resultState; } /** @@ -153,7 +139,12 @@ export function generateCachedState( isAltair = false ): CachedBeaconStateAllForks { const state = generateState(opts, config, isAltair); - return createCachedBeaconState(config, state); + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }) as CachedBeaconStateAllForks; } /** @@ -166,6 +157,5 @@ export async function generateCachedStateWithPubkeys( ): Promise { // somehow this is called in the test but BLS isn't init await initBLS(); - const state = generateState(opts, config, isAltair, true); - return createCachedBeaconState(config, state); + return generateCachedState(opts, config, isAltair); } diff --git a/packages/lodestar/test/utils/stub/index.ts b/packages/lodestar/test/utils/stub/index.ts index 2096e154bfb2..28d24d111b4d 100644 --- a/packages/lodestar/test/utils/stub/index.ts +++ b/packages/lodestar/test/utils/stub/index.ts @@ -1,13 +1,6 @@ import {SinonStubbedInstance} from "sinon"; +import {IBeaconChain} from "../../../src/chain"; -import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; -import {IBeaconChain, ChainEventEmitter} from "../../../src/chain"; - -interface IStubbedChain extends IBeaconChain { - forkChoice: SinonStubbedInstance; - emitter: SinonStubbedInstance; -} - -export type StubbedChain = IStubbedChain & SinonStubbedInstance; +export type StubbedChain = IBeaconChain & SinonStubbedInstance; export * from "./beaconDb"; diff --git a/packages/lodestar/test/utils/validationData/attestation.ts b/packages/lodestar/test/utils/validationData/attestation.ts index e08aacb343f6..45722e8ca9c4 100644 --- a/packages/lodestar/test/utils/validationData/attestation.ts +++ b/packages/lodestar/test/utils/validationData/attestation.ts @@ -16,11 +16,9 @@ import { } from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {SeenAttesters} from "../../../src/chain/seenCache"; import {BlsSingleThreadVerifier} from "../../../src/chain/bls"; -import {computeSubnetForSlot} from "../../../src/chain/validation"; import {signCached} from "../cache"; import {ClockStatic} from "../clock"; -import {toSingleBit} from "../aggregationBits"; -import {toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; @@ -77,9 +75,8 @@ export function getAttestationValidData( }, } as Partial) as IForkChoice; - const committeeIndices = state.getBeaconCommittee(attSlot, attIndex); + const committeeIndices = state.epochCtx.getBeaconCommittee(attSlot, attIndex); const validatorIndex = committeeIndices[bitIndex]; - const aggregationBits = toSingleBit(committeeIndices.length, bitIndex); const attestationData: phase0.AttestationData = { slot: attSlot, @@ -101,12 +98,12 @@ export function getAttestationValidData( const sk = getSecretKeyFromIndexCached(validatorIndex); const attestation: phase0.Attestation = { - aggregationBits, + aggregationBits: BitArray.fromSingleBit(committeeIndices.length, bitIndex), data: attestationData, signature: signCached(sk, signingRoot), }; - const subnet = computeSubnetForSlot(state, attSlot, attIndex); + const subnet = state.epochCtx.computeSubnetForSlot(attSlot, attIndex); // Add state to regen const regen = ({ diff --git a/packages/lodestar/test/utils/validator.ts b/packages/lodestar/test/utils/validator.ts index 939951d6cc19..eec5760f3de0 100644 --- a/packages/lodestar/test/utils/validator.ts +++ b/packages/lodestar/test/utils/validator.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {FAR_FUTURE_EPOCH} from "../../src/constants"; @@ -33,6 +33,6 @@ export function generateValidator(opts: Partial = {}): phase0. * @param opts * @returns {Validator[]} */ -export function generateValidators(n: number, opts?: Partial): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: Partial): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 5ea74dd1b827..2217d46ed7b5 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -46,7 +46,6 @@ ], "dependencies": { "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "^0.8.20", "async-retry": "^1.3.3", "axios": "^0.21.0", "chai": "^4.2.0", diff --git a/packages/spec-test-util/src/index.ts b/packages/spec-test-util/src/index.ts index 157f29b3552c..fb039991ef09 100644 --- a/packages/spec-test-util/src/index.ts +++ b/packages/spec-test-util/src/index.ts @@ -1,4 +1,3 @@ export * from "./downloadTests"; -export * from "./multi"; export * from "./single"; -export * from "./transform"; +export * from "./sszGeneric"; diff --git a/packages/spec-test-util/src/multi.ts b/packages/spec-test-util/src/multi.ts deleted file mode 100644 index a3a486f9f255..000000000000 --- a/packages/spec-test-util/src/multi.ts +++ /dev/null @@ -1,88 +0,0 @@ -import fs from "node:fs"; -import {loadYaml} from "@chainsafe/lodestar-utils"; -import {expect} from "chai"; - -/* eslint-disable - @typescript-eslint/no-unsafe-call, - @typescript-eslint/no-unsafe-member-access, - @typescript-eslint/no-unsafe-return, - @typescript-eslint/no-unsafe-assignment, - @typescript-eslint/no-explicit-any, - @typescript-eslint/no-unused-vars, - @typescript-eslint/naming-convention, - func-names */ - -export interface IBaseCase { - description: string; -} - -/** - * TestSpec - represent structure of yaml file containing spec test cases - * TestCase - single test case, usually under test_cases property in yaml file - */ -// eslint-disable-next-line @typescript-eslint/naming-convention -interface TestSpec { - title: string; - summary: string; - forks_timeline: string; - forks: string; - config: string; - runner: string; - handler: string; - test_cases: TestCase[]; -} - -/** - * Run yaml Eth2.0 bulk spec tests (m) for a certain function - * Compares actual vs expected for all test cases - * @param {string} testYamlPath - path to yaml spec test - * @param {Function} testFunc - function to use to generate output - * @param {Function} getInput - function to convert test case into input array - * @param {Function} getExpected - function to convert test case into a - * comparable expected output - * @param {Function} getActual - function to convert function output into - * comparable actual output - * @param {Function} shouldError - function to convert test case into a - * boolean, if the case should result in an error - * @param {Function} shouldSkip - function to convert test case into a boolean, - * if the case should be skipped - * @param {Function} expectFunc - function to run expectations against expected - * and actual output - * @param timeout - how long to wait before marking tests as failed (default 2000ms). Set to 0 to wait infinitely - */ -export function describeMultiSpec( - testYamlPath: string, - testFunc: (...args: any) => any, - getInput: (testCase: TestCase) => any, - getExpected: (testCase: TestCase) => any, - getActual: (result: any) => Result, - shouldError = (testCase: TestCase, index: number) => false, - shouldSkip = (testCase: TestCase, index: number) => false, - expectFunc = (testCase: TestCase, expect: any, expected: any, actual: any) => expect(actual).to.be.equal(expected), - timeout = 10 * 60 * 1000 -): void { - const testSpec = loadYaml>(fs.readFileSync(testYamlPath, "utf8")); - - const testSuiteName = `${testSpec.runner} - ${testSpec.handler} - ${testSpec.title} - ${testSpec.config}`; - - describe(testSuiteName, function () { - this.timeout(timeout); - for (const [index, testCase] of testSpec.test_cases.entries()) { - if (shouldSkip(testCase, index)) { - continue; - } - const description = index + (testCase.description ? " - " + testCase.description : ""); - it(description, function () { - const inputs = getInput(testCase); - if (shouldError(testCase, index)) { - expect(testFunc.bind(null, ...inputs)).to.throw(); - } else { - const result = testFunc(...inputs); - const actual = getActual(result); - const expected = getExpected(testCase); - expectFunc(testCase, expect, expected, actual); - } - }); - } - }); -} diff --git a/packages/spec-test-util/src/single.ts b/packages/spec-test-util/src/single.ts index 600300ce948f..d725caa7583f 100644 --- a/packages/spec-test-util/src/single.ts +++ b/packages/spec-test-util/src/single.ts @@ -1,7 +1,6 @@ import {expect} from "chai"; -import fs, {readdirSync, readFileSync, existsSync} from "node:fs"; +import fs from "node:fs"; import {basename, join, parse} from "node:path"; -import {Type, CompositeType} from "@chainsafe/ssz"; import {uncompress} from "snappyjs"; import {loadYaml} from "@chainsafe/lodestar-utils"; @@ -23,6 +22,12 @@ export type ExpandedInputType = { treeBacked: boolean; }; +type SszTypeGeneric = { + typeName: string; + deserialize: (bytes: Uint8Array) => unknown; + deserializeToViewDU?: (bytes: Uint8Array) => unknown; +}; + export function toExpandedInputType(inputType: InputType | ExpandedInputType): ExpandedInputType { if ((inputType as ExpandedInputType).type) { return inputType as ExpandedInputType; @@ -44,12 +49,12 @@ export interface ISpecTestOptions { */ inputTypes?: {[K in keyof NonNullable]?: InputType | ExpandedInputType}; - sszTypes?: Record>; + sszTypes?: Record; /** * Some tests need to access the test case in order to generate ssz types for each input file. */ - getSszTypes?: (meta: TestCase["meta"]) => Record>; + getSszTypes?: (meta: TestCase["meta"]) => Record; /** * loadInputFiles sometimes not create TestCase due to abnormal input file names. @@ -108,7 +113,8 @@ export function describeDirectorySpecTest this.timeout(options.timeout || "10 min"); } - const testCases = readdirSync(testCaseDirectoryPath) + const testCases = fs + .readdirSync(testCaseDirectoryPath) .map((name) => join(testCaseDirectoryPath, name)) .filter(isDirectory); @@ -129,7 +135,7 @@ function generateTestCase( // some tests require to load meta.yaml first in order to know respective ssz types. const metaFilePath = join(testCaseDirectoryPath, "meta.yaml"); let meta: TestCase["meta"] = undefined; - if (existsSync(metaFilePath)) { + if (fs.existsSync(metaFilePath)) { meta = loadYaml(fs.readFileSync(metaFilePath, "utf8")); } let testCase = loadInputFiles(testCaseDirectoryPath, options, meta); @@ -160,7 +166,7 @@ function loadInputFiles( meta?: TestCase["meta"] ): TestCase { const testCase: any = {}; - readdirSync(directory) + fs.readdirSync(directory) .map((name) => join(directory, name)) .filter((file) => { if (isDirectory(file)) { @@ -180,10 +186,10 @@ function loadInputFiles( testCase[inputName] = deserializeInputFile(file, inputName, inputType, options, meta); switch (inputType) { case InputType.SSZ: - testCase[`${inputName}_raw`] = readFileSync(file); + testCase[`${inputName}_raw`] = fs.readFileSync(file); break; case InputType.SSZ_SNAPPY: - testCase[`${inputName}_raw`] = uncompress(readFileSync(file)); + testCase[`${inputName}_raw`] = uncompress(fs.readFileSync(file)); break; } if (!options.inputProcessing) throw Error("inputProcessing is not defined"); @@ -217,11 +223,12 @@ function deserializeInputFile( } else if (inputType === InputType.SSZ || inputType === InputType.SSZ_SNAPPY) { const sszTypes = options.getSszTypes ? options.getSszTypes(meta) : options.sszTypes; if (!sszTypes) throw Error("sszTypes is not defined"); - let data = readFileSync(file); + let data = fs.readFileSync(file); if (inputType === InputType.SSZ_SNAPPY) { data = uncompress(data); } - let sszType: Type | undefined; + + let sszType: SszTypeGeneric | undefined; for (const key of Object.keys(sszTypes)) { // most tests configure with exact match // fork_choice tests configure with regex @@ -230,15 +237,23 @@ function deserializeInputFile( break; } } - if (sszType) { - if ((options.inputTypes?.[inputName as keyof TestCase] as ExpandedInputType).treeBacked) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - return (sszType as CompositeType).createTreeBackedFromBytes(data); - } else { - return sszType.deserialize(data); + + if (!sszType) { + throw Error("Cannot find ssz type for inputName " + inputName); + } + + // TODO: Refactor this to be typesafe + if (sszType.typeName === "BeaconState") { + if (!sszType.deserializeToViewDU) { + throw Error("BeaconState type has no deserializeToViewDU method"); } + return sszType.deserializeToViewDU(data); } else { - throw Error("Cannot find ssz type for inputName " + inputName); + return sszType.deserialize(data); } } } + +export function loadYamlFile(path: string): Record { + return loadYaml(fs.readFileSync(path, "utf8")); +} diff --git a/packages/spec-test-util/src/sszGeneric.ts b/packages/spec-test-util/src/sszGeneric.ts index 8fb4f5337ab5..a40b5faa8115 100644 --- a/packages/spec-test-util/src/sszGeneric.ts +++ b/packages/spec-test-util/src/sszGeneric.ts @@ -1,65 +1,63 @@ -import path, {join} from "node:path"; -import fs, {readFileSync, readdirSync} from "node:fs"; -import {Json, Type} from "@chainsafe/ssz"; -import {loadYaml, objectToExpectedCase} from "@chainsafe/lodestar-utils"; +import path from "node:path"; +import fs from "node:fs"; +import {loadYaml} from "@chainsafe/lodestar-utils"; import {uncompress} from "snappyjs"; -export interface IValidTestcase { - root: string; - serialized: Uint8Array; - value: T; -} +/* eslint-disable + @typescript-eslint/explicit-module-boundary-types, + @typescript-eslint/explicit-function-return-type +*/ -export interface IInvalidTestcase { - path: string; +export type ValidTestCaseData = { + root: string; serialized: Uint8Array; -} + jsonValue: unknown; +}; -export function parseValidTestcase(dirpath: string, type: Type): IValidTestcase { +export function parseSszValidTestcase(dirpath: string, metaFilename: string): ValidTestCaseData { // The root is stored in meta.yml as: // root: 0xDEADBEEF - const metaStr = fs.readFileSync(path.join(dirpath, "meta.yaml"), "utf8"); - const meta = loadYaml<{root: string}>(metaStr); + const metaStr = fs.readFileSync(path.join(dirpath, metaFilename), "utf8"); + const meta = loadYaml(metaStr) as {root: string}; if (typeof meta.root !== "string") { throw Error(`meta.root not a string: ${meta.root}\n${fs}`); } + // The serialized value is stored in serialized.ssz_snappy - const serialized = uncompress(readFileSync(join(dirpath, "serialized.ssz_snappy"))); + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); // The value is stored in value.yml - const yamlSnake = loadYaml(fs.readFileSync(join(dirpath, "value.yaml"), "utf8")); - const yamlCamel = objectToExpectedCase(yamlSnake, "camel"); - const value = type.fromJson(yamlCamel as Json); + const yamlPath = path.join(dirpath, "value.yaml"); + const yamlStrNumbersAsNumbers = fs.readFileSync(yamlPath, "utf8"); + const jsonValue = readYamlNumbersAsStrings(yamlStrNumbersAsNumbers); + + // type.fromJson(loadYamlFile(path.join(dirpath, "value.yaml")) as Json) as T; return { root: meta.root, serialized, - value, + jsonValue, }; } -export function parseInvalidTestcase(path: string): IInvalidTestcase { +/** + * ssz_generic + * | basic_vector + * | invalid + * | vec_bool_0 + * | serialized.ssz_snappy + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy - const serialized = uncompress(readFileSync(join(path, "serialized.ssz_snappy"))); + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); return { - path, serialized, }; } -export function getValidTestcases(path: string, prefix: string, type: Type): IValidTestcase[] { - const subdirs = readdirSync(path); - return subdirs - .filter((dir) => dir.includes(prefix)) - .map((d) => join(path, d)) - .map((p) => parseValidTestcase(p, type)) as IValidTestcase[]; -} - -export function getInvalidTestcases(path: string, prefix: string): IInvalidTestcase[] { - const subdirs = readdirSync(path); - return subdirs - .filter((dir) => dir.includes(prefix)) - .map((d) => join(path, d)) - .map(parseInvalidTestcase); +export function readYamlNumbersAsStrings(yamlStr: string): unknown { + return loadYaml(yamlStr); } diff --git a/packages/spec-test-util/src/transform.ts b/packages/spec-test-util/src/transform.ts deleted file mode 100644 index 1bba4c0082fc..000000000000 --- a/packages/spec-test-util/src/transform.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment -, @typescript-eslint/no-explicit-any */ -import {Type, UintType, BigIntUintType, byteType, isCompositeType} from "@chainsafe/ssz"; - -/** - * Transform the type to something that is safe to deserialize - * - * This mainly entails making sure all numbers are bignumbers - */ -export function safeType(type: Type): Type { - if (type === byteType) { - return type; - } else if (!isCompositeType(type)) { - if ((type as UintType).byteLength) { - return new BigIntUintType({byteLength: (type as UintType).byteLength}); - } else { - return type; - } - } else { - const props = Object.getOwnPropertyDescriptors(type) as any; - if (props.elementType) { - if (props.elementType.byteLength !== 1) { - props.elementType.value = safeType(props.elementType.value); - } - } - if (props.fields) { - props.fields.value = {...props.fields.value}; - for (const fieldName of Object.keys(props.fields.value)) { - props.fields.value[fieldName] = safeType(props.fields.value[fieldName]); - } - } - const newtype = Object.create(Object.getPrototypeOf(type), props); - return newtype as Type; - } -} diff --git a/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml b/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml deleted file mode 100644 index 42bf48f37fc1..000000000000 --- a/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml +++ /dev/null @@ -1,14 +0,0 @@ -title: Bulk test -summary: simulates multiple test cases in one file -forks_timeline: mainnet -forks: [phase0] -config: mainnet -runner: bulkRunner -handler: test -test_cases: - - input: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f" - description: "test1" - output: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f" - - input: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f62" - description: "test2" - output: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f62" diff --git a/packages/spec-test-util/test/e2e/multi/index.test.ts b/packages/spec-test-util/test/e2e/multi/index.test.ts deleted file mode 100644 index e4a1fe9943e3..000000000000 --- a/packages/spec-test-util/test/e2e/multi/index.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import {IBaseCase, describeMultiSpec} from "../../../src"; -import path from "node:path"; - -interface IBulkTestCase extends IBaseCase { - input: string; - output: string; -} - -describeMultiSpec( - path.join(__dirname, "../_test_files/multi/bulk.yml"), - (input) => input, - (testCase) => [testCase.input], - (testCase) => testCase.output, - (result) => result -); diff --git a/packages/spec-test-util/test/e2e/single/index.test.ts b/packages/spec-test-util/test/e2e/single/index.test.ts index 34934c72ea50..89aaefbf303d 100644 --- a/packages/spec-test-util/test/e2e/single/index.test.ts +++ b/packages/spec-test-util/test/e2e/single/index.test.ts @@ -1,11 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import fs, {unlinkSync, writeFileSync} from "node:fs"; +import {unlinkSync, writeFileSync} from "node:fs"; import {join} from "node:path"; -import {ContainerType, Type, Json} from "@chainsafe/ssz"; +import {ContainerType, Type} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; -import {describeDirectorySpecTest, InputType} from "../../../src/single"; -import {loadYaml} from "@chainsafe/lodestar-utils"; +import {describeDirectorySpecTest, InputType, loadYamlFile} from "../../../src/single"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -22,18 +21,16 @@ export interface ISimpleCase extends Iterable { }; } -const inputSchema = new ContainerType({ - fields: { - test: ssz.Boolean, - number: ssz.Number64, - }, +const sampleContainerType = new ContainerType({ + test: ssz.Boolean, + number: ssz.UintNum64, }); before(() => { - yamlToSSZ(join(__dirname, "../_test_files/single/case0/input.yaml"), inputSchema); - yamlToSSZ(join(__dirname, "../_test_files/single/case0/output.yaml"), ssz.Number64); - yamlToSSZ(join(__dirname, "../_test_files/single/case1/input.yaml"), inputSchema); - yamlToSSZ(join(__dirname, "../_test_files/single/case1/output.yaml"), ssz.Number64); + yamlToSSZ(join(__dirname, "../_test_files/single/case0/input.yaml"), sampleContainerType); + yamlToSSZ(join(__dirname, "../_test_files/single/case0/output.yaml"), ssz.UintNum64); + yamlToSSZ(join(__dirname, "../_test_files/single/case1/input.yaml"), sampleContainerType); + yamlToSSZ(join(__dirname, "../_test_files/single/case1/output.yaml"), ssz.UintNum64); }); after(() => { @@ -55,8 +52,8 @@ describeDirectorySpecTest( output: InputType.YAML, }, sszTypes: { - input: inputSchema, - output: ssz.Number64, + input: sampleContainerType, + output: ssz.UintNum64, }, shouldError: (testCase) => !testCase.input.test, getExpected: (testCase) => testCase.output, @@ -64,10 +61,6 @@ describeDirectorySpecTest( ); function yamlToSSZ(file: string, sszSchema: Type): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const input: any = sszSchema.fromJson(loadYaml(fs.readFileSync(file, "utf8"))); - if (input.number) { - input.number = Number(input.number); - } + const input = sszSchema.fromJson(loadYamlFile(file)) as {test: boolean; number: number}; writeFileSync(file.replace(".yaml", ".ssz"), sszSchema.serialize(input)); } diff --git a/packages/types/README.md b/packages/types/README.md index 4294f0decc9f..e227680bfb79 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -45,7 +45,7 @@ import {ssz, Epoch} from "@chainsafe/lodestar-types"; const EpochType: Type = ssz.Epoch; -const e = EpochType.defaultValue(); +const e = EpochType.defaultValue; ``` ### By fork @@ -55,8 +55,8 @@ Lodestar types support multiple different consensus forks. In order to easily di ```typescript import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; -const phase0State: phase0.BeaconState = ssz.phase0.BeaconState.defaultValue(); -const altairState: altair.BeaconState = ssz.altair.BeaconState.defaultValue(); +const phase0State: phase0.BeaconState = ssz.phase0.BeaconState.defaultValue; +const altairState: altair.BeaconState = ssz.altair.BeaconState.defaultValue; ``` Primitive types are directly available without a namespace. @@ -64,7 +64,7 @@ Primitive types are directly available without a namespace. ```typescript import {Epoch, ssz} from "@chainsafe/lodestar-types"; -const epoch: Epoch = ssz.Epoch.defaultValue(); +const epoch: Epoch = ssz.Epoch.defaultValue; ``` In some cases, we need interfaces that accept types across all forks, eg: when the fork is not known ahead of time. Typescript interfaces for this purpose are exported under the `allForks` namespace. SSZ Types typed to these interfaces are also provided under an `allForks` namespace, but keyed by `ForkName`. @@ -73,7 +73,7 @@ In some cases, we need interfaces that accept types across all forks, eg: when t import {ForkName} from "@chainsafe/lodestar-params"; import {allForks, ssz} from "@chainsafe/lodestar-types"; -const state: allForks.BeaconState = ssz.allForks[ForkName.phase0].BeaconState.defaultValue(); +const state: allForks.BeaconState = ssz.allForks[ForkName.phase0].BeaconState.defaultValue; ``` ## License diff --git a/packages/types/package.json b/packages/types/package.json index 190e3efaf0ff..2e3b342320f5 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -36,7 +36,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/lodestar-params": "^0.35.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" }, "keywords": [ "ethereum", diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index 0cfff5e34e63..616be35633d4 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -1,7 +1,3 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {ForkName} from "@chainsafe/lodestar-params"; - -import {AllForksSSZTypes} from "./types"; import {ssz as phase0} from "../phase0"; import {ssz as altair} from "../altair"; import {ssz as bellatrix} from "../bellatrix"; @@ -10,26 +6,26 @@ import {ssz as bellatrix} from "../bellatrix"; * Index the ssz types that differ by fork * A record of AllForksSSZTypes indexed by fork */ -export const allForks: {[K in ForkName]: AllForksSSZTypes} = { +export const allForks = { phase0: { - BeaconBlockBody: phase0.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: phase0.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: phase0.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: phase0.BeaconState as AllForksSSZTypes["BeaconState"], + BeaconBlockBody: phase0.BeaconBlockBody, + BeaconBlock: phase0.BeaconBlock, + SignedBeaconBlock: phase0.SignedBeaconBlock, + BeaconState: phase0.BeaconState, Metadata: phase0.Metadata, }, altair: { - BeaconBlockBody: altair.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: altair.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: altair.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: altair.BeaconState as AllForksSSZTypes["BeaconState"], - Metadata: altair.Metadata as AllForksSSZTypes["Metadata"], + BeaconBlockBody: altair.BeaconBlockBody, + BeaconBlock: altair.BeaconBlock, + SignedBeaconBlock: altair.SignedBeaconBlock, + BeaconState: altair.BeaconState, + Metadata: altair.Metadata, }, bellatrix: { - BeaconBlockBody: bellatrix.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: bellatrix.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: bellatrix.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: bellatrix.BeaconState as AllForksSSZTypes["BeaconState"], - Metadata: altair.Metadata as AllForksSSZTypes["Metadata"], + BeaconBlockBody: bellatrix.BeaconBlockBody, + BeaconBlock: bellatrix.BeaconBlock, + SignedBeaconBlock: bellatrix.SignedBeaconBlock, + BeaconState: bellatrix.BeaconState, + Metadata: altair.Metadata, }, }; diff --git a/packages/types/src/allForks/types.ts b/packages/types/src/allForks/types.ts index 8c2a2d14785f..8556ed514af6 100644 --- a/packages/types/src/allForks/types.ts +++ b/packages/types/src/allForks/types.ts @@ -1,8 +1,10 @@ -import {ContainerType} from "@chainsafe/ssz"; - +import {CompositeType, ContainerType, ValueOf, CompositeView, CompositeViewDU} from "@chainsafe/ssz"; import {ts as phase0} from "../phase0"; import {ts as altair} from "../altair"; import {ts as bellatrix} from "../bellatrix"; +import {ssz as phase0Ssz} from "../phase0"; +import {ssz as altairSsz} from "../altair"; +import {ssz as bellatrixSsz} from "../bellatrix"; // Re-export union types for types that are _known_ to differ @@ -24,12 +26,46 @@ export type AllForksTypes = { }; /** - * SSZ Types known to change between forks + * An AllForks type must accept as any parameter the UNION of all fork types. + * The generic argument of `AllForksTypeOf` must be the union of the fork types: + * + * + * For example, `allForks.BeaconState.defaultValue` must return + * ``` + * phase0.BeaconState | altair.BeaconState | bellatrix.BeaconState + * ``` + * + * And `allForks.BeaconState.serialize()` must accept as parameter + * ``` + * phase0.BeaconState | altair.BeaconState | bellatrix.BeaconState + * ``` + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AllForksTypeOf> = CompositeType< + ValueOf, + CompositeView, + CompositeViewDU +>; + +/** + * SSZ Types known to change between forks. + * + * Re-wrapping a union of fields in a new ContainerType allows to pass a generic block to .serialize() + * - .serialize() requires a value with ONLY the common fork fields + * - .deserialize() and ValueOf return a value with ONLY the general fork fields */ export type AllForksSSZTypes = { - BeaconBlockBody: ContainerType; - BeaconBlock: ContainerType; - SignedBeaconBlock: ContainerType; - BeaconState: ContainerType; - Metadata: ContainerType; + BeaconBlockBody: AllForksTypeOf< + typeof phase0Ssz.BeaconBlockBody | typeof altairSsz.BeaconBlockBody | typeof bellatrixSsz.BeaconBlockBody + >; + BeaconBlock: AllForksTypeOf< + typeof phase0Ssz.BeaconBlock | typeof altairSsz.BeaconBlock | typeof bellatrixSsz.BeaconBlock + >; + SignedBeaconBlock: AllForksTypeOf< + typeof phase0Ssz.SignedBeaconBlock | typeof altairSsz.SignedBeaconBlock | typeof bellatrixSsz.SignedBeaconBlock + >; + BeaconState: AllForksTypeOf< + typeof phase0Ssz.BeaconState | typeof altairSsz.BeaconState | typeof bellatrixSsz.BeaconState + >; + Metadata: AllForksTypeOf; }; diff --git a/packages/types/src/altair/sszTypes.ts b/packages/types/src/altair/sszTypes.ts index 23e6cf650683..294a55818292 100644 --- a/packages/types/src/altair/sszTypes.ts +++ b/packages/types/src/altair/sszTypes.ts @@ -1,6 +1,5 @@ -import {BitVectorType, ContainerType, VectorType, ListType, RootType, Vector} from "@chainsafe/ssz"; +import {BitVectorType, ContainerType, ListBasicType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; import { - JUSTIFICATION_BITS_LENGTH, FINALIZED_ROOT_DEPTH, NEXT_SYNC_COMMITTEE_DEPTH, SYNC_COMMITTEE_SUBNET_COUNT, @@ -8,22 +7,19 @@ import { SLOTS_PER_HISTORICAL_ROOT, HISTORICAL_ROOTS_LIMIT, VALIDATOR_REGISTRY_LIMIT, - EPOCHS_PER_HISTORICAL_VECTOR, - EPOCHS_PER_SLASHINGS_VECTOR, + EPOCHS_PER_SYNC_COMMITTEE_PERIOD, + SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; -import {Root} from "../primitive/types"; -import {ssz as phase0Ssz, ts as phase0Types} from "../phase0"; -import {ssz as primitiveSsz} from "../primitive"; -import {LazyVariable} from "../utils/lazyVar"; -import * as altair from "./types"; +import * as phase0Ssz from "../phase0/sszTypes"; +import * as primitiveSsz from "../primitive/sszTypes"; const { Bytes32, - Number64, + UintNum64, + UintBn64, Slot, SubcommitteeIndex, ValidatorIndex, - Gwei, Root, Version, BLSPubkey, @@ -31,176 +27,127 @@ const { ParticipationFlags, } = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); +export const SyncSubnets = new BitVectorType(SYNC_COMMITTEE_SUBNET_COUNT); -export const SyncSubnets = new BitVectorType({ - length: SYNC_COMMITTEE_SUBNET_COUNT, -}); - -export const Metadata = new ContainerType({ - fields: { - ...phase0Ssz.Metadata.fields, +export const Metadata = new ContainerType( + { + seqNumber: UintBn64, + attnets: phase0Ssz.AttestationSubnets, syncnets: SyncSubnets, }, - // New keys are strictly appended, phase0 key order is preserved - casingMap: { - ...phase0Ssz.Metadata.casingMap, - syncnets: "syncnets", - }, -}); + {typeName: "Metadata", jsonCase: "eth2"} +); -export const SyncCommittee = new ContainerType({ - fields: { - pubkeys: new VectorType({elementType: BLSPubkey, length: SYNC_COMMITTEE_SIZE}), +export const SyncCommittee = new ContainerType( + { + pubkeys: new VectorCompositeType(BLSPubkey, SYNC_COMMITTEE_SIZE), aggregatePubkey: BLSPubkey, }, - casingMap: { - pubkeys: "pubkeys", - aggregatePubkey: "aggregate_pubkey", - }, -}); + {typeName: "SyncCommittee", jsonCase: "eth2"} +); -export const SyncCommitteeMessage = new ContainerType({ - fields: { +export const SyncCommitteeMessage = new ContainerType( + { slot: Slot, beaconBlockRoot: Root, validatorIndex: ValidatorIndex, signature: BLSSignature, }, - casingMap: { - slot: "slot", - beaconBlockRoot: "beacon_block_root", - validatorIndex: "validator_index", - signature: "signature", - }, -}); + {typeName: "SyncCommitteeMessage", jsonCase: "eth2"} +); -export const SyncCommitteeContribution = new ContainerType({ - fields: { +export const SyncCommitteeContribution = new ContainerType( + { slot: Slot, beaconBlockRoot: Root, subcommitteeIndex: SubcommitteeIndex, - aggregationBits: new BitVectorType({length: SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT}), + aggregationBits: new BitVectorType(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT), signature: BLSSignature, }, - casingMap: { - slot: "slot", - beaconBlockRoot: "beacon_block_root", - subcommitteeIndex: "subcommittee_index", - aggregationBits: "aggregation_bits", - signature: "signature", - }, -}); + {typeName: "SyncCommitteeContribution", jsonCase: "eth2"} +); -export const ContributionAndProof = new ContainerType({ - fields: { +export const ContributionAndProof = new ContainerType( + { aggregatorIndex: ValidatorIndex, contribution: SyncCommitteeContribution, selectionProof: BLSSignature, }, - casingMap: { - aggregatorIndex: "aggregator_index", - contribution: "contribution", - selectionProof: "selection_proof", - }, -}); + {typeName: "ContributionAndProof", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedContributionAndProof = new ContainerType({ - fields: { +export const SignedContributionAndProof = new ContainerType( + { message: ContributionAndProof, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedContributionAndProof", jsonCase: "eth2"} +); -export const SyncAggregatorSelectionData = new ContainerType({ - fields: { +export const SyncAggregatorSelectionData = new ContainerType( + { slot: Slot, subcommitteeIndex: SubcommitteeIndex, }, - casingMap: { - slot: "slot", - subcommitteeIndex: "subcommittee_index", - }, -}); + {typeName: "SyncAggregatorSelectionData", jsonCase: "eth2"} +); -export const SyncCommitteeBits = new BitVectorType({ - length: SYNC_COMMITTEE_SIZE, -}); +export const SyncCommitteeBits = new BitVectorType(SYNC_COMMITTEE_SIZE); -export const SyncAggregate = new ContainerType({ - fields: { +export const SyncAggregate = new ContainerType( + { syncCommitteeBits: SyncCommitteeBits, syncCommitteeSignature: BLSSignature, }, - casingMap: { - syncCommitteeBits: "sync_committee_bits", - syncCommitteeSignature: "sync_committee_signature", - }, -}); + {typeName: "SyncCommitteeBits", jsonCase: "eth2"} +); -// Re-declare with the new expanded type -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: phase0Ssz.HistoricalBatch.casingMap, -}); + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { ...phase0Ssz.BeaconBlockBody.fields, syncAggregate: SyncAggregate, }, - casingMap: { - ...phase0Ssz.BeaconBlockBody.casingMap, - syncAggregate: "sync_aggregate", - }, -}); + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const BeaconBlock = new ContainerType({ - fields: { +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, - // Reclare expandedType() with altair block and altair state - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: phase0Ssz.BeaconBlock.casingMap, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); -export const EpochParticipation = new ListType({elementType: ParticipationFlags, limit: VALIDATOR_REGISTRY_LIMIT}); -export const InactivityScores = new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}); +export const EpochParticipation = new ListBasicType(ParticipationFlags, VALIDATOR_REGISTRY_LIMIT); +export const InactivityScores = new ListBasicType(UintNum64, VALIDATOR_REGISTRY_LIMIT); // we don't reuse phase0.BeaconState fields since we need to replace some keys // and we cannot keep order doing that -export const BeaconState = new ContainerType({ - fields: { - genesisTime: Number64, +export const BeaconState = new ContainerType( + { + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: Slot, fork: phase0Ssz.Fork, @@ -208,25 +155,22 @@ export const BeaconState = new ContainerType({ latestBlockHeader: phase0Ssz.BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: phase0Ssz.Eth1Data, eth1DataVotes: phase0Ssz.Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry - validators: new ListType({elementType: phase0Ssz.Validator, limit: VALIDATOR_REGISTRY_LIMIT}), - balances: new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}), - randaoMixes: new VectorType({elementType: Bytes32, length: EPOCHS_PER_HISTORICAL_VECTOR}), + validators: phase0Ssz.Validators, + balances: phase0Ssz.Balances, + randaoMixes: phase0Ssz.RandaoMixes, // Slashings - slashings: new VectorType({elementType: Gwei, length: EPOCHS_PER_SLASHINGS_VECTOR}), + slashings: phase0Ssz.Slashings, // Participation previousEpochParticipation: EpochParticipation, currentEpochParticipation: EpochParticipation, // Finality - justificationBits: new BitVectorType({length: JUSTIFICATION_BITS_LENGTH}), + justificationBits: phase0Ssz.JustificationBits, previousJustifiedCheckpoint: phase0Ssz.Checkpoint, currentJustifiedCheckpoint: phase0Ssz.Checkpoint, finalizedCheckpoint: phase0Ssz.Checkpoint, @@ -236,37 +180,35 @@ export const BeaconState = new ContainerType({ currentSyncCommittee: SyncCommittee, nextSyncCommittee: SyncCommittee, }, - casingMap: { - ...phase0Ssz.BeaconState.casingMap, - inactivityScores: "inactivity_scores", - currentSyncCommittee: "current_sync_committee", - nextSyncCommittee: "next_sync_committee", + {typeName: "BeaconState", jsonCase: "eth2"} +); + +export const LightClientSnapshot = new ContainerType( + { + header: phase0Ssz.BeaconBlockHeader, + currentSyncCommittee: SyncCommittee, + nextSyncCommittee: SyncCommittee, }, -}); + {typeName: "LightClientSnapshot", jsonCase: "eth2"} +); -export const LightClientUpdate = new ContainerType({ - fields: { +export const LightClientUpdate = new ContainerType( + { attestedHeader: phase0Ssz.BeaconBlockHeader, nextSyncCommittee: SyncCommittee, - nextSyncCommitteeBranch: new VectorType({ - elementType: Bytes32, - length: NEXT_SYNC_COMMITTEE_DEPTH, - }), + nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH), finalizedHeader: phase0Ssz.BeaconBlockHeader, - finalityBranch: new VectorType({elementType: Bytes32, length: FINALIZED_ROOT_DEPTH}), + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH), syncCommitteeAggregate: SyncAggregate, forkVersion: Version, }, - casingMap: { - attestedHeader: "attested_header", - nextSyncCommittee: "next_sync_committee", - nextSyncCommitteeBranch: "next_sync_committee_branch", - finalizedHeader: "finalized_header", - finalityBranch: "finality_branch", - syncCommitteeAggregate: "sync_committee_aggregate", - forkVersion: "fork_version", - }, -}); + {typeName: "LightClientUpdate", jsonCase: "eth2"} +); -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); +export const LightClientStore = new ContainerType( + { + snapshot: LightClientSnapshot, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), + }, + {typeName: "LightClientStore", jsonCase: "eth2"} +); diff --git a/packages/types/src/altair/types.ts b/packages/types/src/altair/types.ts index 62a9ab20f064..6507c6b146f7 100644 --- a/packages/types/src/altair/types.ts +++ b/packages/types/src/altair/types.ts @@ -1,126 +1,19 @@ -import {BitVector, List, BitList, Vector} from "@chainsafe/ssz"; -import { - Number64, - Uint64, - ParticipationFlags, - Bytes32, - BLSSignature, - Version, - BLSPubkey, - Slot, - Root, - ValidatorIndex, - SubcommitteeIndex, -} from "../primitive/types"; -import * as phase0 from "../phase0/types"; - -export type SyncSubnets = BitVector; - -export interface Metadata { - seqNumber: Uint64; - attnets: phase0.AttestationSubnets; - syncnets: SyncSubnets; -} - -export interface SyncCommittee { - pubkeys: Vector; - aggregatePubkey: BLSPubkey; -} - -export interface SyncCommitteeMessage { - slot: Slot; - beaconBlockRoot: Root; - validatorIndex: ValidatorIndex; - signature: BLSSignature; -} - -export interface SyncCommitteeContribution { - slot: Slot; - beaconBlockRoot: Root; - subcommitteeIndex: SubcommitteeIndex; - aggregationBits: BitList; - signature: BLSSignature; -} - -export interface ContributionAndProof { - aggregatorIndex: ValidatorIndex; - contribution: SyncCommitteeContribution; - selectionProof: BLSSignature; -} - -export interface SignedContributionAndProof { - message: ContributionAndProof; - signature: BLSSignature; -} - -export interface SyncAggregatorSelectionData { - slot: Slot; - subcommitteeIndex: SubcommitteeIndex; -} - -export interface SyncAggregate { - syncCommitteeBits: BitVector; - syncCommitteeSignature: BLSSignature; -} - -export interface BeaconBlockBody extends phase0.BeaconBlockBody { - syncAggregate: SyncAggregate; -} - -export interface BeaconBlock extends phase0.BeaconBlock { - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock extends phase0.SignedBeaconBlock { - message: BeaconBlock; -} - -export interface BeaconState - extends Omit { - // Participation - previousEpochParticipation: List; - currentEpochParticipation: List; - // Inactivity - inactivityScores: List; - // Sync - currentSyncCommittee: SyncCommittee; - nextSyncCommittee: SyncCommittee; -} - -/** - * Spec v1.0.1 - */ -export interface LightClientUpdate { - /** The beacon block header that is attested to by the sync committee */ - attestedHeader: phase0.BeaconBlockHeader; - /** Next sync committee corresponding to the header */ - nextSyncCommittee: SyncCommittee; - nextSyncCommitteeBranch: Vector; - /** The finalized beacon block header attested to by Merkle branch */ - finalizedHeader: phase0.BeaconBlockHeader; - finalityBranch: Vector; - /** Sync committee aggregate signature */ - syncCommitteeAggregate: SyncAggregate; - /** Fork version for the aggregate signature */ - forkVersion: Version; -} - -/** - * Spec v1.0.1 - */ -export interface LightClientStore { - /** Beacon block header that is finalized */ - finalizedHeader: phase0.BeaconBlockHeader; - /** Sync committees corresponding to the header */ - currentSyncCommittee: SyncCommittee; - nextSyncCommittee: SyncCommittee; - /** Best available header to switch finalized head to if we see nothing else */ - bestValidUpdate?: LightClientUpdate; - /** Most recent available reasonably-safe header */ - optimisticHeader: phase0.BeaconBlockHeader; - /** Max number of active participants in a sync committee (used to calculate - * safety threshold) - */ - previousMaxActiveParticipants: Uint64; - currentMaxActiveParticipants: Uint64; -} +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type SyncSubnets = ValueOf; +export type Metadata = ValueOf; +export type SyncCommittee = ValueOf; +export type SyncCommitteeMessage = ValueOf; +export type SyncCommitteeContribution = ValueOf; +export type ContributionAndProof = ValueOf; +export type SignedContributionAndProof = ValueOf; +export type SyncAggregatorSelectionData = ValueOf; +export type SyncAggregate = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type LightClientSnapshot = ValueOf; +export type LightClientUpdate = ValueOf; +export type LightClientStore = ValueOf; diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 4007ba8dae74..661acbf1e346 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -1,4 +1,7 @@ -import {byteType, ByteVectorType, ContainerType, List, ListType, RootType, Vector, VectorType} from "@chainsafe/ssz"; +import {ByteListType, ByteVectorType, ContainerType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; +import {ssz as primitiveSsz} from "../primitive"; +import {ssz as phase0Ssz} from "../phase0"; +import {ssz as altairSsz} from "../altair"; import { BYTES_PER_LOGS_BLOOM, HISTORICAL_ROOTS_LIMIT, @@ -7,188 +10,111 @@ import { MAX_EXTRA_DATA_BYTES, SLOTS_PER_HISTORICAL_ROOT, } from "@chainsafe/lodestar-params"; -import {ssz as primitiveSsz, ts as primitive} from "../primitive"; -import {ssz as phase0Ssz, ts as phase0} from "../phase0"; -import {ssz as altairSsz} from "../altair"; -import * as bellatrix from "./types"; -import {LazyVariable} from "../utils/lazyVar"; -import {Uint256} from "../primitive/sszTypes"; - -const {Bytes20, Bytes32, Number64, Slot, ValidatorIndex, Root, BLSSignature, Uint8} = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); +const {Bytes20, Bytes32, UintNum64, Slot, ValidatorIndex, Root, BLSSignature, UintBn256: Uint256} = primitiveSsz; /** * ByteList[MAX_BYTES_PER_TRANSACTION] * * Spec v1.0.1 */ -export const Transaction = new ListType({elementType: byteType, limit: MAX_BYTES_PER_TRANSACTION}); +export const Transaction = new ByteListType(MAX_BYTES_PER_TRANSACTION); -export const Transactions = new ListType>({ - elementType: Transaction, - limit: MAX_TRANSACTIONS_PER_PAYLOAD, -}); +/** + * Union[OpaqueTransaction] + * + * Spec v1.0.1 + */ +export const Transactions = new ListCompositeType(Transaction, MAX_TRANSACTIONS_PER_PAYLOAD); const executionPayloadFields = { parentHash: Root, feeRecipient: Bytes20, stateRoot: Bytes32, receiptsRoot: Bytes32, - logsBloom: new ByteVectorType({length: BYTES_PER_LOGS_BLOOM}), + logsBloom: new ByteVectorType(BYTES_PER_LOGS_BLOOM), random: Bytes32, - blockNumber: Number64, - gasLimit: Number64, - gasUsed: Number64, - timestamp: Number64, + blockNumber: UintNum64, + gasLimit: UintNum64, + gasUsed: UintNum64, + timestamp: UintNum64, // TODO: if there is perf issue, consider making ByteListType - extraData: new ListType({limit: MAX_EXTRA_DATA_BYTES, elementType: Uint8}), + extraData: new ByteListType(MAX_EXTRA_DATA_BYTES), baseFeePerGas: Uint256, // Extra payload fields blockHash: Root, }; -const executionPayloadCasingMap = { - parentHash: "parent_hash", - feeRecipient: "fee_recipient", - stateRoot: "state_root", - receiptsRoot: "receipts_root", - logsBloom: "logs_bloom", - random: "random", - blockNumber: "block_number", - gasLimit: "gas_limit", - gasUsed: "gas_used", - timestamp: "timestamp", - extraData: "extra_data", - baseFeePerGas: "base_fee_per_gas", - blockHash: "block_hash", -}; -/** - * ```python - * class ExecutionPayload(Container): - * # Execution block header fields - * parent_hash: Hash32 - * coinbase: Bytes20 # 'beneficiary' in the yellow paper - * state_root: Bytes32 - * receipt_root: Bytes32 # 'receipts root' in the yellow paper - * logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] - * random: Bytes32 # 'difficulty' in the yellow paper - * block_number: uint64 # 'number' in the yellow paper - * gas_limit: uint64 - * gas_used: uint64 - * timestamp: uint64 - * extra_data: ByteList[MAX_EXTRA_DATA_BYTES] - * base_fee_per_gas: Bytes32 # base fee introduced in EIP-1559, little-endian serialized - * # Extra payload fields - * block_hash: Hash32 # Hash of execution block - * transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] - * ``` - * - * Spec v1.0.1 - */ -export const ExecutionPayload = new ContainerType({ - fields: { +export const ExecutionPayload = new ContainerType( + { ...executionPayloadFields, transactions: Transactions, }, - casingMap: { - ...executionPayloadCasingMap, - transactions: "transactions", - }, -}); + {typeName: "ExecutionPayload", jsonCase: "eth2"} +); -/** - * ```python - * class ExecutionPayload(Container): - * ... - * transactions_root: Root - * ``` - * - * Spec v1.0.1 - */ -export const ExecutionPayloadHeader = new ContainerType({ - fields: { +export const ExecutionPayloadHeader = new ContainerType( + { ...executionPayloadFields, transactionsRoot: Root, }, - casingMap: { - ...executionPayloadCasingMap, - transactionsRoot: "transactions_root", - }, -}); + {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} +); -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { ...altairSsz.BeaconBlockBody.fields, executionPayload: ExecutionPayload, }, - casingMap: { - ...altairSsz.BeaconBlockBody.casingMap, - executionPayload: "execution_payload", - }, -}); + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const BeaconBlock = new ContainerType({ - fields: { +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, // Reclare expandedType() with altair block and altair state - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: altairSsz.BeaconBlock.casingMap, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); -export const PowBlock = new ContainerType({ - fields: { +export const PowBlock = new ContainerType( + { blockHash: Root, parentHash: Root, totalDifficulty: Uint256, }, - casingMap: { - blockHash: "block_hash", - parentHash: "parent_hash", - totalDifficulty: "total_difficulty", - }, -}); + {typeName: "PowBlock", jsonCase: "eth2"} +); // Re-declare with the new expanded type -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: phase0Ssz.HistoricalBatch.casingMap, -}); + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); // we don't reuse phase0.BeaconState fields since we need to replace some keys // and we cannot keep order doing that -export const BeaconState = new ContainerType({ - fields: { - genesisTime: Number64, +export const BeaconState = new ContainerType( + { + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: primitiveSsz.Slot, fork: phase0Ssz.Fork, @@ -196,14 +122,11 @@ export const BeaconState = new ContainerType({ latestBlockHeader: phase0Ssz.BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: phase0Ssz.Eth1Data, eth1DataVotes: phase0Ssz.Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry validators: phase0Ssz.Validators, balances: phase0Ssz.Balances, @@ -224,13 +147,7 @@ export const BeaconState = new ContainerType({ currentSyncCommittee: altairSsz.SyncCommittee, nextSyncCommittee: altairSsz.SyncCommittee, // Execution - latestExecutionPayloadHeader: ExecutionPayloadHeader, // [New in Bellatrix] + latestExecutionPayloadHeader: ExecutionPayloadHeader, // [New in Merge] }, - casingMap: { - ...altairSsz.BeaconState.casingMap, - latestExecutionPayloadHeader: "latest_execution_payload_header", - }, -}); - -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); + {typeName: "BeaconState", jsonCase: "eth2"} +); diff --git a/packages/types/src/bellatrix/types.ts b/packages/types/src/bellatrix/types.ts index c7034c891b49..c7bbc384070b 100644 --- a/packages/types/src/bellatrix/types.ts +++ b/packages/types/src/bellatrix/types.ts @@ -1,52 +1,11 @@ -import * as altair from "../altair/types"; -import {Root, Bytes32, Number64, ExecutionAddress, Uint256} from "../primitive/types"; - -export type Transaction = Uint8Array; - -type ExecutionPayloadFields = { - // Execution block header fields - parentHash: Root; - feeRecipient: ExecutionAddress; - stateRoot: Bytes32; - receiptsRoot: Bytes32; - logsBloom: Uint8Array; - random: Bytes32; - blockNumber: number; - gasLimit: Number64; - gasUsed: Number64; - timestamp: Number64; - extraData: Uint8Array; - baseFeePerGas: Uint256; - // Extra payload fields - blockHash: Bytes32; -}; - -export type ExecutionPayload = ExecutionPayloadFields & { - transactions: Transaction[]; -}; - -export type ExecutionPayloadHeader = ExecutionPayloadFields & { - transactionsRoot: Root; -}; - -export interface BeaconBlockBody extends altair.BeaconBlockBody { - executionPayload: ExecutionPayload; -} - -export interface BeaconBlock extends altair.BeaconBlock { - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock extends altair.SignedBeaconBlock { - message: BeaconBlock; -} - -export interface BeaconState extends altair.BeaconState { - latestExecutionPayloadHeader: ExecutionPayloadHeader; -} - -export type PowBlock = { - blockHash: Root; - parentHash: Root; - totalDifficulty: bigint; -}; +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type Transaction = ValueOf; +export type ExecutionPayload = ValueOf; +export type ExecutionPayloadHeader = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type PowBlock = ValueOf; diff --git a/packages/types/src/phase0/sszTypes.ts b/packages/types/src/phase0/sszTypes.ts index e33052296512..467c07156a31 100644 --- a/packages/types/src/phase0/sszTypes.ts +++ b/packages/types/src/phase0/sszTypes.ts @@ -1,13 +1,12 @@ import { BitListType, BitVectorType, - ContainerLeafNodeStructType, ContainerType, - List, - ListType, - RootType, - Vector, - VectorType, + ContainerNodeStructType, + ListBasicType, + ListCompositeType, + VectorBasicType, + VectorCompositeType, } from "@chainsafe/ssz"; import { ATTESTATION_SUBNET_COUNT, @@ -28,15 +27,13 @@ import { SLOTS_PER_HISTORICAL_ROOT, VALIDATOR_REGISTRY_LIMIT, } from "@chainsafe/lodestar-params"; -import {ssz as primitiveSsz, ts as primitiveTs} from "../primitive"; -import {LazyVariable} from "../utils/lazyVar"; -import * as phase0 from "./types"; +import * as primitiveSsz from "../primitive/sszTypes"; const { Boolean, Bytes32, - Number64, - Uint64, + UintNum64, + UintBn64, Slot, Epoch, CommitteeIndex, @@ -50,412 +47,314 @@ const { Domain, } = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); - // Misc types // ========== -export const AttestationSubnets = new BitVectorType({ - length: ATTESTATION_SUBNET_COUNT, -}); +export const AttestationSubnets = new BitVectorType(ATTESTATION_SUBNET_COUNT); -export const BeaconBlockHeader = new ContainerType({ - fields: { +export const BeaconBlockHeader = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, parentRoot: Root, stateRoot: Root, bodyRoot: Root, }, - casingMap: { - slot: "slot", - proposerIndex: "proposer_index", - parentRoot: "parent_root", - stateRoot: "state_root", - bodyRoot: "body_root", - }, -}); + {typeName: "BeaconBlockHeader", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlockHeader = new ContainerType({ - fields: { +export const SignedBeaconBlockHeader = new ContainerType( + { message: BeaconBlockHeader, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlockHeader", jsonCase: "eth2"} +); -export const Checkpoint = new ContainerType({ - fields: { +export const Checkpoint = new ContainerType( + { epoch: Epoch, root: Root, }, - expectedCase: "notransform", -}); + {typeName: "Checkpoint", jsonCase: "eth2"} +); -export const CommitteeBits = new BitListType({ - limit: MAX_VALIDATORS_PER_COMMITTEE, -}); +export const CommitteeBits = new BitListType(MAX_VALIDATORS_PER_COMMITTEE); -export const CommitteeIndices = new ListType>({ - elementType: ValidatorIndex, - limit: MAX_VALIDATORS_PER_COMMITTEE, -}); +export const CommitteeIndices = new ListBasicType(ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE); -export const DepositMessage = new ContainerType({ - fields: { +export const DepositMessage = new ContainerType( + { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, - amount: Number64, - }, - casingMap: { - pubkey: "pubkey", - withdrawalCredentials: "withdrawal_credentials", - amount: "amount", + amount: UintNum64, }, -}); + {typeName: "DepositMessage", jsonCase: "eth2"} +); -export const DepositData = new ContainerType({ - fields: { - // Fields order is strickly preserved - ...DepositMessage.fields, +export const DepositData = new ContainerType( + { + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64, signature: BLSSignature, }, - casingMap: { - ...DepositMessage.casingMap, - signature: "signature", - }, -}); + {typeName: "DepositData", jsonCase: "eth2"} +); -export const DepositDataRootList = new ListType>({ - elementType: new RootType({expandedType: DepositData}), - limit: 2 ** DEPOSIT_CONTRACT_TREE_DEPTH, -}); +export const DepositDataRootList = new ListCompositeType(Root, 2 ** DEPOSIT_CONTRACT_TREE_DEPTH); -export const DepositEvent = new ContainerType({ - fields: { +export const DepositEvent = new ContainerType( + { depositData: DepositData, - blockNumber: Number64, - index: Number64, - }, - // Custom type, not in the consensus specs - casingMap: { - depositData: "deposit_data", - blockNumber: "block_number", - index: "index", + blockNumber: UintNum64, + index: UintNum64, }, -}); + {typeName: "DepositEvent", jsonCase: "eth2"} +); -export const Eth1Data = new ContainerType({ - fields: { +export const Eth1Data = new ContainerType( + { depositRoot: Root, - depositCount: Number64, + depositCount: UintNum64, blockHash: Bytes32, }, - casingMap: { - depositRoot: "deposit_root", - depositCount: "deposit_count", - blockHash: "block_hash", - }, -}); + {typeName: "Eth1Data", jsonCase: "eth2"} +); -export const Eth1DataVotes = new ListType({ - elementType: Eth1Data, - limit: EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH, -}); +export const Eth1DataVotes = new ListCompositeType(Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH); -export const Eth1DataOrdered = new ContainerType({ - fields: { - // Fields order is strickly preserved - ...Eth1Data.fields, - blockNumber: Number64, +export const Eth1DataOrdered = new ContainerType( + { + depositRoot: Root, + depositCount: UintNum64, + blockHash: Bytes32, + blockNumber: UintNum64, }, - // Custom type, not in the consensus specs - casingMap: { - ...Eth1Data.casingMap, - blockNumber: "block_number", + {typeName: "Eth1DataOrdered", jsonCase: "eth2"} +); + +/** Spec'ed but only used in lodestar as a type */ +export const Eth1Block = new ContainerType( + { + timestamp: UintNum64, + depositRoot: Root, + depositCount: UintNum64, }, -}); + {typeName: "Eth1Block", jsonCase: "eth2"} +); -export const Fork = new ContainerType({ - fields: { +export const Fork = new ContainerType( + { previousVersion: Version, currentVersion: Version, epoch: Epoch, }, - casingMap: { - previousVersion: "previous_version", - currentVersion: "current_version", - epoch: "epoch", - }, -}); + {typeName: "Fork", jsonCase: "eth2"} +); -export const ForkData = new ContainerType({ - fields: { +export const ForkData = new ContainerType( + { currentVersion: Version, genesisValidatorsRoot: Root, }, - casingMap: { - currentVersion: "current_version", - genesisValidatorsRoot: "genesis_validators_root", - }, -}); + {typeName: "ForkData", jsonCase: "eth2"} +); -export const ENRForkID = new ContainerType({ - fields: { +export const ENRForkID = new ContainerType( + { forkDigest: ForkDigest, nextForkVersion: Version, nextForkEpoch: Epoch, }, - casingMap: { - forkDigest: "fork_digest", - nextForkVersion: "next_fork_version", - nextForkEpoch: "next_fork_epoch", - }, -}); + {typeName: "ENRForkID", jsonCase: "eth2"} +); -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: { - blockRoots: "block_roots", - stateRoots: "state_roots", + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); + +/** + * Non-spec'ed helper type to allow efficient hashing in epoch transition. + * This type is like a 'Header' of HistoricalBatch where its fields are hashed. + */ +export const HistoricalBatchRoots = new ContainerType( + { + blockRoots: Root, // Hashed HistoricalBlockRoots + stateRoots: Root, // Hashed HistoricalStateRoots }, -}); + {typeName: "HistoricalBatchRoots", jsonCase: "eth2"} +); -export const Validator = new ContainerLeafNodeStructType({ - fields: { +export const ValidatorContainer = new ContainerType( + { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, - effectiveBalance: Number64, + effectiveBalance: UintNum64, slashed: Boolean, activationEligibilityEpoch: Epoch, activationEpoch: Epoch, exitEpoch: Epoch, withdrawableEpoch: Epoch, }, - casingMap: { - pubkey: "pubkey", - withdrawalCredentials: "withdrawal_credentials", - effectiveBalance: "effective_balance", - slashed: "slashed", - activationEligibilityEpoch: "activation_eligibility_epoch", - activationEpoch: "activation_epoch", - exitEpoch: "exit_epoch", - withdrawableEpoch: "withdrawable_epoch", - }, -}); + {typeName: "Validator", jsonCase: "eth2"} +); + +export const ValidatorNodeStruct = new ContainerNodeStructType(ValidatorContainer.fields, ValidatorContainer.opts); +// The main Validator type is the 'ContainerNodeStructType' version +export const Validator = ValidatorNodeStruct; // Export as stand-alone for direct tree optimizations -export const Validators = new ListType({elementType: Validator, limit: VALIDATOR_REGISTRY_LIMIT}); -export const Balances = new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}); -export const RandaoMixes = new VectorType({elementType: Bytes32, length: EPOCHS_PER_HISTORICAL_VECTOR}); -export const Slashings = new VectorType({elementType: Gwei, length: EPOCHS_PER_SLASHINGS_VECTOR}); -export const JustificationBits = new BitVectorType({length: JUSTIFICATION_BITS_LENGTH}); +export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); +export const Balances = new ListBasicType(UintNum64, VALIDATOR_REGISTRY_LIMIT); +export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR); +export const Slashings = new VectorBasicType(Gwei, EPOCHS_PER_SLASHINGS_VECTOR); +export const JustificationBits = new BitVectorType(JUSTIFICATION_BITS_LENGTH); // Misc dependants -export const AttestationData = new ContainerType({ - fields: { +export const AttestationData = new ContainerType( + { slot: Slot, index: CommitteeIndex, beaconBlockRoot: Root, source: Checkpoint, target: Checkpoint, }, - casingMap: { - slot: "slot", - index: "index", - beaconBlockRoot: "beacon_block_root", - source: "source", - target: "target", - }, -}); + {typeName: "AttestationData", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const IndexedAttestation = new ContainerType({ - fields: { +export const IndexedAttestation = new ContainerType( + { attestingIndices: CommitteeIndices, data: AttestationData, signature: BLSSignature, }, - casingMap: { - attestingIndices: "attesting_indices", - data: "data", - signature: "signature", - }, -}); + {typeName: "IndexedAttestation", jsonCase: "eth2"} +); -export const PendingAttestation = new ContainerType({ - fields: { +export const PendingAttestation = new ContainerType( + { aggregationBits: CommitteeBits, data: AttestationData, inclusionDelay: Slot, proposerIndex: ValidatorIndex, }, - casingMap: { - aggregationBits: "aggregation_bits", - data: "data", - inclusionDelay: "inclusion_delay", - proposerIndex: "proposer_index", - }, -}); + {typeName: "PendingAttestation", jsonCase: "eth2"} +); -export const SigningData = new ContainerType({ - fields: { +export const SigningData = new ContainerType( + { objectRoot: Root, domain: Domain, }, - casingMap: { - objectRoot: "object_root", - domain: "domain", - }, -}); + {typeName: "SigningData", jsonCase: "eth2"} +); // Operations types // ================ -export const Attestation = new ContainerType({ - fields: { +export const Attestation = new ContainerType( + { aggregationBits: CommitteeBits, data: AttestationData, signature: BLSSignature, }, - casingMap: { - aggregationBits: "aggregation_bits", - data: "data", - signature: "signature", - }, -}); + {typeName: "Attestation", jsonCase: "eth2"} +); -export const AttesterSlashing = new ContainerType({ - fields: { +export const AttesterSlashing = new ContainerType( + { attestation1: IndexedAttestation, attestation2: IndexedAttestation, }, - // Declaration time casingMap for toJson/fromJson for container <=> json data - casingMap: { - attestation1: "attestation_1", - attestation2: "attestation_2", - }, -}); + {typeName: "AttesterSlashing", jsonCase: "eth2"} +); -export const Deposit = new ContainerType({ - fields: { - proof: new VectorType({elementType: Bytes32, length: DEPOSIT_CONTRACT_TREE_DEPTH + 1}), +export const Deposit = new ContainerType( + { + proof: new VectorCompositeType(Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1), data: DepositData, }, - expectedCase: "notransform", -}); + {typeName: "Deposit", jsonCase: "eth2"} +); -export const ProposerSlashing = new ContainerType({ - fields: { +export const ProposerSlashing = new ContainerType( + { signedHeader1: SignedBeaconBlockHeader, signedHeader2: SignedBeaconBlockHeader, }, - // Declaration time casingMap for toJson/fromJson for container <=> json data - casingMap: { - signedHeader1: "signed_header_1", - signedHeader2: "signed_header_2", - }, -}); + {typeName: "ProposerSlashing", jsonCase: "eth2"} +); -export const VoluntaryExit = new ContainerType({ - fields: { +export const VoluntaryExit = new ContainerType( + { epoch: Epoch, validatorIndex: ValidatorIndex, }, - casingMap: { - epoch: "epoch", - validatorIndex: "validator_index", - }, -}); + {typeName: "VoluntaryExit", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedVoluntaryExit = new ContainerType({ - fields: { +export const SignedVoluntaryExit = new ContainerType( + { message: VoluntaryExit, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedVoluntaryExit", jsonCase: "eth2"} +); // Block types // =========== -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { randaoReveal: BLSSignature, eth1Data: Eth1Data, graffiti: Bytes32, - proposerSlashings: new ListType({elementType: ProposerSlashing, limit: MAX_PROPOSER_SLASHINGS}), - attesterSlashings: new ListType({elementType: AttesterSlashing, limit: MAX_ATTESTER_SLASHINGS}), - attestations: new ListType({elementType: Attestation, limit: MAX_ATTESTATIONS}), - deposits: new ListType({elementType: Deposit, limit: MAX_DEPOSITS}), - voluntaryExits: new ListType({elementType: SignedVoluntaryExit, limit: MAX_VOLUNTARY_EXITS}), - }, - casingMap: { - randaoReveal: "randao_reveal", - eth1Data: "eth1_data", - graffiti: "graffiti", - proposerSlashings: "proposer_slashings", - attesterSlashings: "attester_slashings", - attestations: "attestations", - deposits: "deposits", - voluntaryExits: "voluntary_exits", - }, -}); - -export const BeaconBlock = new ContainerType({ - fields: { + proposerSlashings: new ListCompositeType(ProposerSlashing, MAX_PROPOSER_SLASHINGS), + attesterSlashings: new ListCompositeType(AttesterSlashing, MAX_ATTESTER_SLASHINGS), + attestations: new ListCompositeType(Attestation, MAX_ATTESTATIONS), + deposits: new ListCompositeType(Deposit, MAX_DEPOSITS), + voluntaryExits: new ListCompositeType(SignedVoluntaryExit, MAX_VOLUNTARY_EXITS), + }, + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: { - slot: "slot", - proposerIndex: "proposer_index", - parentRoot: "parent_root", - stateRoot: "state_root", - body: "body", - }, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); // State types // =========== -export const EpochAttestations = new ListType>({ - elementType: PendingAttestation, - limit: MAX_ATTESTATIONS * SLOTS_PER_EPOCH, -}); +export const EpochAttestations = new ListCompositeType(PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH); -export const BeaconState = new ContainerType({ - fields: { +export const BeaconState = new ContainerType( + { // Misc - genesisTime: Number64, + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: Slot, fork: Fork, @@ -463,14 +362,11 @@ export const BeaconState = new ContainerType({ latestBlockHeader: BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: Eth1Data, eth1DataVotes: Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry validators: Validators, balances: Balances, @@ -486,135 +382,83 @@ export const BeaconState = new ContainerType({ currentJustifiedCheckpoint: Checkpoint, finalizedCheckpoint: Checkpoint, }, - casingMap: { - genesisTime: "genesis_time", - genesisValidatorsRoot: "genesis_validators_root", - slot: "slot", - fork: "fork", - latestBlockHeader: "latest_block_header", - blockRoots: "block_roots", - stateRoots: "state_roots", - historicalRoots: "historical_roots", - eth1Data: "eth1_data", - eth1DataVotes: "eth1_data_votes", - eth1DepositIndex: "eth1_deposit_index", - validators: "validators", - balances: "balances", - randaoMixes: "randao_mixes", - slashings: "slashings", - previousEpochAttestations: "previous_epoch_attestations", - currentEpochAttestations: "current_epoch_attestations", - justificationBits: "justification_bits", - previousJustifiedCheckpoint: "previous_justified_checkpoint", - currentJustifiedCheckpoint: "current_justified_checkpoint", - finalizedCheckpoint: "finalized_checkpoint", - }, -}); + {typeName: "BeaconState", jsonCase: "eth2"} +); // Validator types // =============== -export const CommitteeAssignment = new ContainerType({ - fields: { +export const CommitteeAssignment = new ContainerType( + { validators: CommitteeIndices, committeeIndex: CommitteeIndex, slot: Slot, }, - // Custom type, not in the consensus specs - casingMap: { - validators: "validators", - committeeIndex: "committee_index", - slot: "slot", - }, -}); + {typeName: "CommitteeAssignment", jsonCase: "eth2"} +); -export const AggregateAndProof = new ContainerType({ - fields: { +export const AggregateAndProof = new ContainerType( + { aggregatorIndex: ValidatorIndex, aggregate: Attestation, selectionProof: BLSSignature, }, - casingMap: { - aggregatorIndex: "aggregator_index", - aggregate: "aggregate", - selectionProof: "selection_proof", - }, -}); + {typeName: "AggregateAndProof", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedAggregateAndProof = new ContainerType({ - fields: { +export const SignedAggregateAndProof = new ContainerType( + { message: AggregateAndProof, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} +); // ReqResp types // ============= -export const Status = new ContainerType({ - fields: { +export const Status = new ContainerType( + { forkDigest: ForkDigest, finalizedRoot: Root, finalizedEpoch: Epoch, headRoot: Root, headSlot: Slot, }, - casingMap: { - forkDigest: "fork_digest", - finalizedRoot: "finalized_root", - finalizedEpoch: "finalized_epoch", - headRoot: "head_root", - headSlot: "head_slot", - }, -}); + {typeName: "Status", jsonCase: "eth2"} +); -export const Goodbye = Uint64; +export const Goodbye = UintBn64; -export const Ping = Uint64; +export const Ping = UintBn64; -export const Metadata = new ContainerType({ - fields: { - seqNumber: Uint64, +export const Metadata = new ContainerType( + { + seqNumber: UintBn64, attnets: AttestationSubnets, }, - casingMap: { - seqNumber: "seq_number", - attnets: "attnets", - }, -}); + {typeName: "Metadata", jsonCase: "eth2"} +); -export const BeaconBlocksByRangeRequest = new ContainerType({ - fields: { +export const BeaconBlocksByRangeRequest = new ContainerType( + { startSlot: Slot, - count: Number64, - step: Number64, - }, - casingMap: { - startSlot: "start_slot", - count: "count", - step: "step", + count: UintNum64, + step: UintNum64, }, -}); + {typeName: "BeaconBlocksByRangeRequest", jsonCase: "eth2"} +); -export const BeaconBlocksByRootRequest = new ListType({elementType: Root, limit: MAX_REQUEST_BLOCKS}); +export const BeaconBlocksByRootRequest = new ListCompositeType(Root, MAX_REQUEST_BLOCKS); // Api types // ========= -export const Genesis = new ContainerType({ - fields: { +export const Genesis = new ContainerType( + { genesisValidatorsRoot: Root, - genesisTime: Uint64, + genesisTime: UintNum64, genesisForkVersion: Version, }, - // From beacon-apis - casingMap: { - genesisValidatorsRoot: "genesis_validators_root", - genesisTime: "genesis_time", - genesisForkVersion: "genesis_fork_version", - }, -}); - -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); + {typeName: "Genesis", jsonCase: "eth2"} +); diff --git a/packages/types/src/phase0/types.ts b/packages/types/src/phase0/types.ts index e0312a42c6c4..622996920679 100644 --- a/packages/types/src/phase0/types.ts +++ b/packages/types/src/phase0/types.ts @@ -1,334 +1,42 @@ -import {BitList, List, Vector, BitVector} from "@chainsafe/ssz"; -import { - BLSPubkey, - BLSSignature, - Epoch, - Root, - Number64, - Slot, - ValidatorIndex, - Version, - CommitteeIndex, - Bytes32, - Domain, - ForkDigest, - Gwei, - Uint64, -} from "../primitive/types"; - -export type AttestationSubnets = BitVector; - -export interface BeaconBlockHeader { - slot: Slot; - proposerIndex: ValidatorIndex; - parentRoot: Root; - stateRoot: Root; - bodyRoot: Root; -} - -export interface SignedBeaconBlockHeader { - message: BeaconBlockHeader; - signature: BLSSignature; -} - -export interface Checkpoint { - epoch: Epoch; - root: Root; -} - -export interface DepositMessage { - // BLS pubkey - pubkey: BLSPubkey; - // Withdrawal credentials - withdrawalCredentials: Bytes32; - // Amount in Gwei - amount: Number64; -} - -export interface DepositData { - // BLS pubkey - pubkey: BLSPubkey; - // Withdrawal credentials - withdrawalCredentials: Bytes32; - // Amount in Gwei - amount: Number64; - // Signing over DepositMessage - signature: BLSSignature; -} - -export interface DepositEvent { - depositData: DepositData; - /// The block number of the log that included this `DepositData`. - blockNumber: Number64; - /// The index included with the deposit log. - index: Number64; -} - -export interface Eth1Data { - // Root of the deposit tree - depositRoot: Root; - // Total number of deposits - depositCount: Number64; - // Block hash - blockHash: Bytes32; -} - -export interface Eth1DataOrdered { - // block number for this eth1 data block hash - blockNumber: Number64; - // Root of the deposit tree - depositRoot: Root; - // Total number of deposits - depositCount: Number64; - // Block hash - blockHash: Bytes32; -} - -export interface Fork { - // Previous fork version - previousVersion: Version; - // Current fork version - currentVersion: Version; - // Fork epoch number - epoch: Epoch; -} - -export interface ForkData { - // Current fork version - currentVersion: Version; - // root of genesis validator list - genesisValidatorsRoot: Root; -} - -export interface ENRForkID { - // Current fork digest - forkDigest: ForkDigest; - // next planned fork versin - nextForkVersion: Version; - // next fork epoch - nextForkEpoch: Epoch; -} - -export interface HistoricalBatch { - // Block roots - blockRoots: Vector; - // State roots - stateRoots: Vector; -} - -export interface Validator { - // BLS public key - pubkey: BLSPubkey; - // Commitment to pubkey for withdrawals - withdrawalCredentials: Bytes32; - // Balance at stake - effectiveBalance: Number64; - // Was the validator slashed - slashed: boolean; - // When criteria for activation were met - activationEligibilityEpoch: Epoch; - // Epoch when validator activated - activationEpoch: Epoch; - // Epoch when validator exited - exitEpoch: Epoch; - // When validator can withdraw or transfer funds - withdrawableEpoch: Epoch; -} - -export interface AttestationData { - slot: Slot; - index: CommitteeIndex; - // LMD GHOST vote - beaconBlockRoot: Root; - // FFG vote - source: Checkpoint; - target: Checkpoint; -} - -export interface IndexedAttestation { - // Validator Indices - attestingIndices: List; - // Attestation Data - data: AttestationData; - // Aggregate signature - signature: BLSSignature; -} - -export interface PendingAttestation { - // Attester aggregation bitfield - aggregationBits: BitList; - // Attestation data - data: AttestationData; - // Inclusion delay - inclusionDelay: Slot; - // Proposer index - proposerIndex: ValidatorIndex; -} - -export interface SigningData { - objectRoot: Root; - domain: Domain; -} - -export interface Eth1Block { - // Use blockHash to be consistent with the Eth1Data type - blockHash: Bytes32; - // Use blockNumber to be consistent with DepositEvent type - blockNumber: Number64; - timestamp: Number64; -} - -export interface Attestation { - // Attester participation bitfield - aggregationBits: BitList; - // Attestation data - data: AttestationData; - // BLS aggregate signature - signature: BLSSignature; -} - -export interface AttesterSlashing { - // First attestation - attestation1: IndexedAttestation; - // Second attestation - attestation2: IndexedAttestation; -} - -export interface Deposit { - // Branch in the deposit tree - proof: Vector; - // Deposit data - data: DepositData; -} - -export interface ProposerSlashing { - // First block header - signedHeader1: SignedBeaconBlockHeader; - // Second block header - signedHeader2: SignedBeaconBlockHeader; -} - -export interface VoluntaryExit { - // Minimum epoch for processing exit - epoch: Epoch; - // Index of the exiting validator - validatorIndex: ValidatorIndex; -} - -export interface SignedVoluntaryExit { - message: VoluntaryExit; - // Validator signature - signature: BLSSignature; -} - -export interface BeaconBlockBody { - randaoReveal: BLSSignature; - eth1Data: Eth1Data; - graffiti: Bytes32; - proposerSlashings: List; - attesterSlashings: List; - attestations: List; - deposits: List; - voluntaryExits: List; -} - -export interface BeaconBlock { - // Header - slot: Slot; - proposerIndex: ValidatorIndex; - parentRoot: Root; - stateRoot: Root; - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock { - message: BeaconBlock; - signature: BLSSignature; -} - -export interface BeaconState { - // Misc - genesisTime: Number64; - genesisValidatorsRoot: Root; - slot: Slot; - fork: Fork; // For versioning hard forks - - // History - latestBlockHeader: BeaconBlockHeader; - blockRoots: Vector; - stateRoots: Vector; - historicalRoots: List; - - // Eth1 - eth1Data: Eth1Data; - eth1DataVotes: List; - eth1DepositIndex: Number64; - - // Registry - validators: List; - balances: List; - - // Shuffling - randaoMixes: Vector; - - // Slashings - slashings: Vector; // Balances penalized at every withdrawal period - - // Attestations - previousEpochAttestations: List; - currentEpochAttestations: List; - - // Finality - justificationBits: BitVector; - previousJustifiedCheckpoint: Checkpoint; - currentJustifiedCheckpoint: Checkpoint; - finalizedCheckpoint: Checkpoint; -} - -export interface CommitteeAssignment { - validators: List; - committeeIndex: CommitteeIndex; - slot: Slot; -} - -export interface AggregateAndProof { - aggregatorIndex: ValidatorIndex; - aggregate: Attestation; - selectionProof: BLSSignature; -} - -export interface SignedAggregateAndProof { - message: AggregateAndProof; - signature: BLSSignature; -} - -export interface Status { - forkDigest: ForkDigest; - finalizedRoot: Root; - finalizedEpoch: Epoch; - headRoot: Root; - headSlot: Slot; -} - -export type Goodbye = Uint64; - -export type Ping = Uint64; - -export interface Metadata { - seqNumber: Uint64; - attnets: AttestationSubnets; -} - -export interface BeaconBlocksByRangeRequest { - startSlot: Slot; - count: Number64; - step: Number64; -} - -export type BeaconBlocksByRootRequest = List; - -export interface Genesis { - genesisTime: Uint64; - genesisValidatorsRoot: Root; - genesisForkVersion: Version; -} +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type AttestationSubnets = ValueOf; +export type BeaconBlockHeader = ValueOf; +export type SignedBeaconBlockHeader = ValueOf; +export type Checkpoint = ValueOf; +export type DepositMessage = ValueOf; +export type DepositData = ValueOf; +export type DepositEvent = ValueOf; +export type Eth1Data = ValueOf; +export type Eth1DataOrdered = ValueOf; +export type Eth1Block = ValueOf; +export type Fork = ValueOf; +export type ForkData = ValueOf; +export type ENRForkID = ValueOf; +export type HistoricalBatch = ValueOf; +export type Validator = ValueOf; +export type AttestationData = ValueOf; +export type IndexedAttestation = ValueOf; +export type PendingAttestation = ValueOf; +export type SigningData = ValueOf; +export type Attestation = ValueOf; +export type AttesterSlashing = ValueOf; +export type Deposit = ValueOf; +export type ProposerSlashing = ValueOf; +export type VoluntaryExit = ValueOf; +export type SignedVoluntaryExit = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type CommitteeAssignment = ValueOf; +export type AggregateAndProof = ValueOf; +export type SignedAggregateAndProof = ValueOf; +export type Status = ValueOf; +export type Goodbye = ValueOf; +export type Ping = ValueOf; +export type Metadata = ValueOf; +export type BeaconBlocksByRangeRequest = ValueOf; +export type BeaconBlocksByRootRequest = ValueOf; +export type Genesis = ValueOf; diff --git a/packages/types/src/primitive/sszTypes.ts b/packages/types/src/primitive/sszTypes.ts index db41e0d8db21..95e73c478496 100644 --- a/packages/types/src/primitive/sszTypes.ts +++ b/packages/types/src/primitive/sszTypes.ts @@ -1,46 +1,37 @@ -import { - byteType, - booleanType, - ByteVectorType, - BigIntUintType, - number32Type, - NumberUintType, - RootType, - Number64UintType, -} from "@chainsafe/ssz"; +import {ByteVectorType, UintNumberType, UintBigintType, BooleanType} from "@chainsafe/ssz"; -export const Boolean = booleanType; -export const Bytes4 = new ByteVectorType({length: 4}); -export const Bytes8 = new ByteVectorType({length: 8}); -export const Bytes20 = new ByteVectorType({length: 20}); -export const Bytes32 = new ByteVectorType({length: 32}); -export const Bytes48 = new ByteVectorType({length: 48}); -export const Bytes96 = new ByteVectorType({length: 96}); -export const Uint8 = byteType; -export const Uint16 = new NumberUintType({byteLength: 2}); -export const Uint32 = number32Type; -export const Number64 = new Number64UintType(); -export const Uint64 = new BigIntUintType({byteLength: 8}); -export const Uint128 = new BigIntUintType({byteLength: 16}); -export const Uint256 = new BigIntUintType({byteLength: 32}); +export const Boolean = new BooleanType(); +export const Byte = new UintNumberType(1); +export const Bytes4 = new ByteVectorType(4); +export const Bytes8 = new ByteVectorType(8); +export const Bytes20 = new ByteVectorType(20); +export const Bytes32 = new ByteVectorType(32); +export const Bytes48 = new ByteVectorType(48); +export const Bytes96 = new ByteVectorType(96); +export const Uint8 = new UintNumberType(1); +export const Uint16 = new UintNumberType(2); +export const Uint32 = new UintNumberType(4); +export const UintNum64 = new UintNumberType(8); +export const UintNumInf64 = new UintNumberType(8, {clipInfinity: true}); +export const UintBn64 = new UintBigintType(8); +export const UintBn128 = new UintBigintType(16); +export const UintBn256 = new UintBigintType(32); // Custom types, defined for type hinting and readability -export const Slot = Number64; -export const Epoch = Number64; -export const CommitteeIndex = Number64; -export const SubcommitteeIndex = Number64; -export const ValidatorIndex = Number64; -export const Gwei = Uint64; -export const Root = new RootType({ - expandedType: () => { - throw new Error("Generic Root type has no expanded type"); - }, -}); +export const Slot = UintNumInf64; +export const Epoch = UintNumInf64; +export const SyncPeriod = UintNum64; +export const CommitteeIndex = UintNum64; +export const SubcommitteeIndex = UintNum64; +export const ValidatorIndex = UintNum64; +export const Gwei = UintBn64; +export const Root = new ByteVectorType(32); + export const Version = Bytes4; export const DomainType = Bytes4; export const ForkDigest = Bytes4; export const BLSPubkey = Bytes48; export const BLSSignature = Bytes96; export const Domain = Bytes32; -export const ParticipationFlags = Uint8; +export const ParticipationFlags = new UintNumberType(1, {setBitwiseOR: true}); export const ExecutionAddress = Bytes20; diff --git a/packages/types/src/primitive/types.ts b/packages/types/src/primitive/types.ts index 6a9d275c1657..619f1915abe2 100644 --- a/packages/types/src/primitive/types.ts +++ b/packages/types/src/primitive/types.ts @@ -1,32 +1,34 @@ -import {ByteVector} from "@chainsafe/ssz"; +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; // Each type exported here contains both a compile-time type // (a typescript interface) and a run-time ssz type (a javascript variable) // For more information, see ./index.ts -export type Bytes4 = ByteVector; -export type Bytes8 = ByteVector; -export type Bytes20 = ByteVector; -export type Bytes32 = ByteVector; -export type Bytes48 = ByteVector; -export type Bytes96 = ByteVector; -export type Uint8 = number; -export type Uint16 = number; -export type Uint32 = number; -export type Number64 = number; -export type Uint64 = bigint; -export type Uint128 = bigint; -export type Uint256 = bigint; +export type Bytes4 = ValueOf; +export type Bytes8 = ValueOf; +export type Bytes20 = ValueOf; +export type Bytes32 = ValueOf; +export type Bytes48 = ValueOf; +export type Bytes96 = ValueOf; +export type Uint8 = ValueOf; +export type Uint16 = ValueOf; +export type Uint32 = ValueOf; +export type UintNum64 = ValueOf; +export type UintNumInf64 = ValueOf; +export type UintBn64 = ValueOf; +export type UintBn128 = ValueOf; +export type UintBn256 = ValueOf; // Custom types, defined for type hinting and readability -export type Slot = Number64; -export type Epoch = Number64; -export type SyncPeriod = Number64; -export type CommitteeIndex = Number64; -export type SubcommitteeIndex = Number64; -export type ValidatorIndex = Number64; -export type Gwei = Uint64; +export type Slot = UintNumInf64; +export type Epoch = UintNumInf64; +export type SyncPeriod = UintNum64; +export type CommitteeIndex = UintNum64; +export type SubcommitteeIndex = UintNum64; +export type ValidatorIndex = UintNum64; +export type Gwei = UintBn64; export type Root = Bytes32; export type Version = Bytes4; export type DomainType = Bytes4; @@ -40,3 +42,5 @@ export type ExecutionAddress = Bytes20; /** Common non-spec type to represent roots as strings */ export type RootHex = string; +/** Non-spec type to signal time is represented in seconds */ +export type TimeSeconds = number; diff --git a/packages/types/src/utils/StringType.ts b/packages/types/src/utils/StringType.ts index f94ff80bc5ee..2b3538ba628b 100644 --- a/packages/types/src/utils/StringType.ts +++ b/packages/types/src/utils/StringType.ts @@ -3,36 +3,50 @@ import {BasicType} from "@chainsafe/ssz"; /* eslint-disable @typescript-eslint/naming-convention */ export class StringType extends BasicType { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - struct_getSerializedLength(data?: string): number { - throw new Error("unsupported ssz operation"); - } + readonly typeName = "string"; + byteLength = 0; + fixedSize = 0; + minSize = 0; + maxSize = 0; + defaultValue = "" as T; - struct_convertToJson(value: T): string { - return value; - } + // Serialization + deserialization - struct_convertFromJson(data: string): T { - return data as T; + value_serializeToBytes(): number { + throw Error("Not supported in String type"); } - - struct_assertValidValue(data: unknown): data is T { - throw new Error("unsupported ssz operation"); + value_deserializeFromBytes(): T { + throw Error("Not supported in String type"); } - - serialize(): Uint8Array { - throw new Error("unsupported ssz type for serialization"); + tree_serializeToBytes(): number { + throw Error("Not supported in String type"); + } + tree_deserializeFromBytes(): never { + throw Error("Not supported in String type"); } - struct_serializeToBytes(): number { - throw new Error("unsupported ssz type for serialization"); + // Fast tree opts + + tree_getFromNode(): T { + throw Error("Not supported in String type"); + } + tree_setToNode(): void { + throw Error("Not supported in String type"); + } + tree_getFromPackedNode(): T { + throw Error("Not supported in String type"); } + tree_setToPackedNode(): void { + throw Error("Not supported in String type"); + } + + // JSON - struct_deserializeFromBytes(): T { - throw new Error("unsupported ssz operation"); + fromJson(json: unknown): T { + return json as T; } - struct_defaultValue(): T { - return "something" as T; + toJson(value: T): unknown { + return value; } } diff --git a/packages/types/src/utils/lazyVar.ts b/packages/types/src/utils/lazyVar.ts deleted file mode 100644 index 9378501d095e..000000000000 --- a/packages/types/src/utils/lazyVar.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class LazyVariable { - private var: {set: false} | {set: true; value: T} = {set: false}; - - get(): T { - if (!this.var.set) throw Error("variable not set"); - return this.var.value; - } - - set(value: T): void { - this.var = {set: true, value}; - } -} diff --git a/packages/types/test/unit/constants.test.ts b/packages/types/test/unit/constants.test.ts index 67540b4bc4a4..1ab21a18ac33 100644 --- a/packages/types/test/unit/constants.test.ts +++ b/packages/types/test/unit/constants.test.ts @@ -9,11 +9,11 @@ import {expect} from "chai"; // guarantee that these constants are correct. describe("Lightclient pre-computed constants", () => { - const FINALIZED_ROOT_GINDEX = Number(ssz.altair.BeaconState.getPathGindex(["finalizedCheckpoint", "root"])); + const FINALIZED_ROOT_GINDEX = bnToNum(ssz.altair.BeaconState.getPathInfo(["finalizedCheckpoint", "root"]).gindex); const FINALIZED_ROOT_DEPTH = floorlog2(FINALIZED_ROOT_GINDEX); const FINALIZED_ROOT_INDEX = FINALIZED_ROOT_GINDEX % 2 ** FINALIZED_ROOT_DEPTH; - const NEXT_SYNC_COMMITTEE_GINDEX = Number(ssz.altair.BeaconState.getPathGindex(["nextSyncCommittee"])); + const NEXT_SYNC_COMMITTEE_GINDEX = bnToNum(ssz.altair.BeaconState.getPathInfo(["nextSyncCommittee"]).gindex); const NEXT_SYNC_COMMITTEE_DEPTH = floorlog2(NEXT_SYNC_COMMITTEE_GINDEX); const NEXT_SYNC_COMMITTEE_INDEX = NEXT_SYNC_COMMITTEE_GINDEX % 2 ** NEXT_SYNC_COMMITTEE_DEPTH; @@ -36,3 +36,8 @@ describe("Lightclient pre-computed constants", () => { function floorlog2(num: number): number { return Math.floor(Math.log2(num)); } + +/** Type safe wrapper for Number constructor that takes 'any' */ +function bnToNum(bn: bigint): number { + return Number(bn); +} diff --git a/packages/types/test/unit/ssz.test.ts b/packages/types/test/unit/ssz.test.ts index 5ebf0bc0679e..d6d0e0936255 100644 --- a/packages/types/test/unit/ssz.test.ts +++ b/packages/types/test/unit/ssz.test.ts @@ -3,8 +3,8 @@ import {ssz} from "../../src"; describe("size", function () { it("should calculate correct minSize and maxSize", () => { - const minSize = ssz.phase0.BeaconState.minSize(); - const maxSize = ssz.phase0.BeaconState.maxSize(); + const minSize = ssz.phase0.BeaconState.minSize; + const maxSize = ssz.phase0.BeaconState.maxSize; // https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e expect(minSize).to.be.equal(2687377); expect(maxSize).to.be.equal(141837543039377); @@ -14,41 +14,31 @@ describe("size", function () { describe("container serialization/deserialization field casing(s)", function () { it("AttesterSlashing", function () { const test = { - attestation1: ssz.phase0.IndexedAttestation.defaultValue(), - attestation2: ssz.phase0.IndexedAttestation.defaultValue(), + attestation1: ssz.phase0.IndexedAttestation.defaultValue, + attestation2: ssz.phase0.IndexedAttestation.defaultValue, }; const json = { - attestation_1: ssz.phase0.IndexedAttestation.toJson(test.attestation1, {case: "snake"}), - attestation_2: ssz.phase0.IndexedAttestation.toJson(test.attestation2, {case: "snake"}), + attestation_1: ssz.phase0.IndexedAttestation.toJson(test.attestation1), + attestation_2: ssz.phase0.IndexedAttestation.toJson(test.attestation2), }; - const result = ssz.phase0.AttesterSlashing.fromJson(json, { - case: "snake", - }); - expect(ssz.phase0.AttesterSlashing.equals(test, result)).to.be.true; - const back = ssz.phase0.AttesterSlashing.toJson(result, { - case: "snake", - }); + const result = ssz.phase0.AttesterSlashing.fromJson(json); + const back = ssz.phase0.AttesterSlashing.toJson(result); expect(back).to.be.deep.equal(json); }); it("ProposerSlashing", function () { const test = { - signedHeader1: ssz.phase0.SignedBeaconBlockHeader.defaultValue(), - signedHeader2: ssz.phase0.SignedBeaconBlockHeader.defaultValue(), + signedHeader1: ssz.phase0.SignedBeaconBlockHeader.defaultValue, + signedHeader2: ssz.phase0.SignedBeaconBlockHeader.defaultValue, }; const json = { - signed_header_1: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader1, {case: "snake"}), - signed_header_2: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader2, {case: "snake"}), + signed_header_1: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader1), + signed_header_2: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader2), }; - const result = ssz.phase0.ProposerSlashing.fromJson(json, { - case: "snake", - }); - expect(ssz.phase0.ProposerSlashing.equals(test, result)).to.be.true; - const back = ssz.phase0.ProposerSlashing.toJson(result, { - case: "snake", - }); + const result = ssz.phase0.ProposerSlashing.fromJson(json); + const back = ssz.phase0.ProposerSlashing.toJson(result); expect(back).to.be.deep.equal(json); }); }); diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index f60d4ba35d17..5395c4315d22 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -46,7 +46,7 @@ export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): throw new Error("endianness must be either 'le' or 'be'"); } -export function toHex(buffer: Parameters[0]): string { +export function toHex(buffer: Uint8Array | Parameters[0]): string { if (Buffer.isBuffer(buffer)) { return "0x" + buffer.toString("hex"); } else if (buffer instanceof Uint8Array) { diff --git a/packages/utils/test/unit/json.test.ts b/packages/utils/test/unit/json.test.ts index bb9311396ab2..000e463ee76e 100644 --- a/packages/utils/test/unit/json.test.ts +++ b/packages/utils/test/unit/json.test.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment */ -import {fromHexString, Json} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {expect} from "chai"; import {LodestarError, toJson, toString, CIRCULAR_REFERENCE_TAG} from "../../src"; @@ -140,7 +140,7 @@ describe("Json helper", () => { describe("toString", () => { interface ITestCase { id: string; - json: Json; + json: unknown; output: string; } const testCases: (ITestCase | (() => ITestCase))[] = [ diff --git a/packages/validator/package.json b/packages/validator/package.json index 92a40c84e712..cded35d6cd80 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -54,7 +54,7 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", "bigint-buffer": "^1.1.5", "cross-fetch": "^3.1.4", "strict-event-emitter-types": "^2.0.0" diff --git a/packages/validator/src/repositories/metaDataRepository.ts b/packages/validator/src/repositories/metaDataRepository.ts index 39159ec46fb1..0bbba61941b9 100644 --- a/packages/validator/src/repositories/metaDataRepository.ts +++ b/packages/validator/src/repositories/metaDataRepository.ts @@ -1,5 +1,5 @@ import {Bucket, encodeKey, IDatabaseApiOptions} from "@chainsafe/lodestar-db"; -import {Root, Uint64} from "@chainsafe/lodestar-types"; +import {Root, UintNum64} from "@chainsafe/lodestar-types"; import {ssz} from "@chainsafe/lodestar-types"; import {LodestarValidatorDatabaseController} from "../types"; @@ -22,19 +22,16 @@ export class MetaDataRepository { } async setGenesisValidatorsRoot(genesisValidatorsRoot: Root): Promise { - await this.db.put( - this.encodeKey(GENESIS_VALIDATORS_ROOT), - Buffer.from(genesisValidatorsRoot.valueOf() as Uint8Array) - ); + await this.db.put(this.encodeKey(GENESIS_VALIDATORS_ROOT), Buffer.from(genesisValidatorsRoot)); } - async getGenesisTime(): Promise { + async getGenesisTime(): Promise { const bytes = await this.db.get(this.encodeKey(GENESIS_TIME)); - return bytes ? ssz.Uint64.deserialize(bytes) : null; + return bytes ? ssz.UintNum64.deserialize(bytes) : null; } - async setGenesisTime(genesisTime: Uint64): Promise { - await this.db.put(this.encodeKey(GENESIS_TIME), Buffer.from(ssz.Uint64.serialize(genesisTime))); + async setGenesisTime(genesisTime: UintNum64): Promise { + await this.db.put(this.encodeKey(GENESIS_TIME), Buffer.from(ssz.UintNum64.serialize(genesisTime))); } private encodeKey(key: Uint8Array): Uint8Array { diff --git a/packages/validator/src/services/utils.ts b/packages/validator/src/services/utils.ts index b307d6ff94c7..8fc436cca2d8 100644 --- a/packages/validator/src/services/utils.ts +++ b/packages/validator/src/services/utils.ts @@ -9,10 +9,6 @@ export type SubcommitteeDuty = { selectionProof: SyncSelectionProof["selectionProof"]; }; -export function getAggregationBits(committeeLength: number, validatorIndexInCommittee: number): boolean[] { - return Array.from({length: committeeLength}, (_, i) => i === validatorIndexInCommittee); -} - export function groupAttDutiesByCommitteeIndex(duties: AttDutyAndProof[]): Map { const dutiesByCommitteeIndex = new Map(); diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index cd194035077d..2af97f77e7e8 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -28,11 +28,10 @@ import { ValidatorIndex, ssz, } from "@chainsafe/lodestar-types"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; +import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {routes} from "@chainsafe/lodestar-api"; import {ISlashingProtection} from "../slashingProtection"; import {PubkeyHex} from "../types"; -import {getAggregationBits} from "./utils"; import {externalSignerPostSignature} from "../util/externalSignerClient"; export enum SignerType { @@ -146,7 +145,7 @@ export class ValidatorStore { }); return { - aggregationBits: getAggregationBits(duty.committeeLength, duty.validatorCommitteeIndex) as List, + aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), data: attestationData, signature: await this.getSignature(duty.pubkey, signingRoot), }; @@ -226,7 +225,7 @@ export class ValidatorStore { const domain = this.config.getDomain(DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, slot); const signingData: altair.SyncAggregatorSelectionData = { slot, - subcommitteeIndex: subcommitteeIndex, + subcommitteeIndex, }; const signingRoot = computeSigningRoot(ssz.altair.SyncAggregatorSelectionData, signingData, domain); diff --git a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts index 3a0d4f880f00..083617705235 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts @@ -18,19 +18,11 @@ export class AttestationByTargetRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - sourceEpoch: ssz.Epoch, - targetEpoch: ssz.Epoch, - signingRoot: ssz.Root, - }, - // Custom type, not in the consensus specs - casingMap: { - sourceEpoch: "source_epoch", - targetEpoch: "target_epoch", - signingRoot: "signing_root", - }, - }); + this.type = new ContainerType({ + sourceEpoch: ssz.Epoch, + targetEpoch: ssz.Epoch, + signingRoot: ssz.Root, + }); // casing doesn't matter } async getAll(pubkey: BLSPubkey, limit?: number): Promise { diff --git a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts index 536fedce087a..dd9daee6abf5 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts @@ -21,17 +21,10 @@ export class AttestationLowerBoundRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - minSourceEpoch: ssz.Epoch, - minTargetEpoch: ssz.Epoch, - }, - // Custom type, not in the consensus specs - casingMap: { - minSourceEpoch: "min_source_epoch", - minTargetEpoch: "min_target_epoch", - }, - }); + this.type = new ContainerType({ + minSourceEpoch: ssz.Epoch, + minTargetEpoch: ssz.Epoch, + }); // casing doesn't matter } async get(pubkey: BLSPubkey): Promise { diff --git a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts index c825ac394383..35624fc831b6 100644 --- a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts +++ b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts @@ -18,17 +18,10 @@ export class BlockBySlotRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - slot: ssz.Slot, - signingRoot: ssz.Root, - }, - // Custom type, not in the consensus specs - casingMap: { - slot: "slot", - signingRoot: "signing_root", - }, - }); + this.type = new ContainerType({ + slot: ssz.Slot, + signingRoot: ssz.Root, + }); // casing doesn't matter } async getAll(pubkey: BLSPubkey, limit?: number): Promise { diff --git a/packages/validator/src/slashingProtection/utils.ts b/packages/validator/src/slashingProtection/utils.ts index 0ada547cca39..85981f422eac 100644 --- a/packages/validator/src/slashingProtection/utils.ts +++ b/packages/validator/src/slashingProtection/utils.ts @@ -1,8 +1,8 @@ import {Epoch, Root, ssz} from "@chainsafe/lodestar-types"; -import {fromHexString, toHexString, Vector} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; export const blsPubkeyLen = 48; -export const ZERO_ROOT = ssz.Root.defaultValue(); +export const ZERO_ROOT = ssz.Root.defaultValue; export function isEqualRoot(root1: Root, root2: Root): boolean { return ssz.Root.equals(root1, root2); @@ -31,7 +31,7 @@ export function minEpoch(epochs: Epoch[]): Epoch | null { return epochs.length > 0 ? Math.min(...epochs) : null; } -export function uniqueVectorArr(buffers: Vector[]): Vector[] { +export function uniqueVectorArr(buffers: Uint8Array[]): Uint8Array[] { const bufferStr = new Set(); return buffers.filter((buffer) => { const str = toHexString(buffer); diff --git a/packages/validator/src/util/clock.ts b/packages/validator/src/util/clock.ts index 0dcf23014684..b37eb0c2e869 100644 --- a/packages/validator/src/util/clock.ts +++ b/packages/validator/src/util/clock.ts @@ -2,7 +2,7 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import {ErrorAborted, ILogger, isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; import {GENESIS_SLOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Epoch, Number64, Slot} from "@chainsafe/lodestar-types"; +import {Epoch, Slot, TimeSeconds} from "@chainsafe/lodestar-types"; import {computeEpochAtSlot, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; type RunEveryFn = (slot: Slot, signal: AbortSignal) => Promise; @@ -111,7 +111,7 @@ export class Clock implements IClock { /** * Same to the spec but we use Math.round instead of Math.floor. */ -export function getCurrentSlotAround(config: IChainForkConfig, genesisTime: Number64): Slot { +export function getCurrentSlotAround(config: IChainForkConfig, genesisTime: TimeSeconds): Slot { const diffInSeconds = Date.now() / 1000 - genesisTime; const slotsSinceGenesis = Math.round(diffInSeconds / config.SECONDS_PER_SLOT); return GENESIS_SLOT + slotsSinceGenesis; diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 116ddbf005fa..f25dd572fe5d 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -67,7 +67,7 @@ export class Validator { }) : opts.api; - const clock = new Clock(config, logger, {genesisTime: Number(genesis.genesisTime)}); + const clock = new Clock(config, logger, {genesisTime: genesis.genesisTime}); const validatorStore = new ValidatorStore(config, slashingProtection, signers, genesis); const indicesService = new IndicesService(logger, api, validatorStore); this.emitter = new ValidatorEventEmitter(); diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index 0b21377b402f..3bfcb8116415 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -34,7 +34,7 @@ describe("AttestationDutiesService", function () { index, balance: 32e9, status: "active", - validator: ssz.phase0.Validator.defaultValue(), + validator: ssz.phase0.Validator.defaultValue, }; before(() => { diff --git a/packages/validator/test/unit/services/syncCommitteDuties.test.ts b/packages/validator/test/unit/services/syncCommitteDuties.test.ts index dd3601a91155..08940f360eb6 100644 --- a/packages/validator/test/unit/services/syncCommitteDuties.test.ts +++ b/packages/validator/test/unit/services/syncCommitteDuties.test.ts @@ -38,7 +38,7 @@ describe("SyncCommitteeDutiesService", function () { index: indices[0], balance: 32e9, status: "active", - validator: ssz.phase0.Validator.defaultValue(), + validator: ssz.phase0.Validator.defaultValue, }; before(() => { diff --git a/packages/validator/test/unit/services/syncCommittee.test.ts b/packages/validator/test/unit/services/syncCommittee.test.ts index 6b754a0277e1..dfe75b2733d7 100644 --- a/packages/validator/test/unit/services/syncCommittee.test.ts +++ b/packages/validator/test/unit/services/syncCommittee.test.ts @@ -62,9 +62,9 @@ describe("SyncCommitteeService", function () { ); const beaconBlockRoot = Buffer.alloc(32, 0x4d); - const syncCommitteeSignature = ssz.altair.SyncCommitteeMessage.defaultValue(); - const contribution = ssz.altair.SyncCommitteeContribution.defaultValue(); - const contributionAndProof = ssz.altair.SignedContributionAndProof.defaultValue(); + const syncCommitteeSignature = ssz.altair.SyncCommitteeMessage.defaultValue; + const contribution = ssz.altair.SyncCommitteeContribution.defaultValue; + const contributionAndProof = ssz.altair.SignedContributionAndProof.defaultValue; const duties: SyncDutyAndProofs[] = [ { duty: { diff --git a/packages/validator/test/unit/services/utils.test.ts b/packages/validator/test/unit/services/utils.test.ts deleted file mode 100644 index 543e45a361c4..000000000000 --- a/packages/validator/test/unit/services/utils.test.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {getAggregationBits} from "../../../src/services/utils"; -import {expect} from "chai"; - -describe("getAggregationBits", function () { - it("should return correct bits", function () { - const bits = getAggregationBits(4, 3); - expect(bits).to.deep.equal([false, false, false, true]); - }); -}); diff --git a/packages/validator/test/unit/slashingProtection/interchange/index.test.ts b/packages/validator/test/unit/slashingProtection/interchange/index.test.ts index e33cc61cd968..48f25a1462b1 100644 --- a/packages/validator/test/unit/slashingProtection/interchange/index.test.ts +++ b/packages/validator/test/unit/slashingProtection/interchange/index.test.ts @@ -6,7 +6,7 @@ import {Interchange, parseInterchange, serializeInterchange} from "../../../../s describe("interchange", () => { it("Should parseInterchange and serializeInterchange", () => { - const expectedGenesisValidatorsRoot: Root = ssz.Root.defaultValue(); + const expectedGenesisValidatorsRoot: Root = ssz.Root.defaultValue; const interchange: Interchange = { metadata: { interchange_format: "complete", diff --git a/packages/validator/test/unit/slashingProtection/minMaxSurround/utils.ts b/packages/validator/test/unit/slashingProtection/minMaxSurround/utils.ts index aad76e436994..9f03230e7aaf 100644 --- a/packages/validator/test/unit/slashingProtection/minMaxSurround/utils.ts +++ b/packages/validator/test/unit/slashingProtection/minMaxSurround/utils.ts @@ -1,7 +1,7 @@ import {BLSPubkey, ssz} from "@chainsafe/lodestar-types"; import {IDistanceStore, IDistanceEntry} from "../../../../src/slashingProtection/minMaxSurround"; -export const emptyPubkey = ssz.BLSPubkey.defaultValue(); +export const emptyPubkey = ssz.BLSPubkey.defaultValue; export class DistanceMapStore { map: Map; constructor() { From cbcde08945d0ff4e4e3baed4f58eec83b8523662 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:16:52 +0530 Subject: [PATCH 02/30] Lint and fix spec tests --- packages/lodestar/src/api/impl/beacon/state/index.ts | 1 - packages/lodestar/test/spec/ssz/generic/types.ts | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lodestar/src/api/impl/beacon/state/index.ts b/packages/lodestar/src/api/impl/beacon/state/index.ts index db0ffbdd8f7b..3c77fa839326 100644 --- a/packages/lodestar/src/api/impl/beacon/state/index.ts +++ b/packages/lodestar/src/api/impl/beacon/state/index.ts @@ -3,7 +3,6 @@ import {routes} from "@chainsafe/lodestar-api"; import {Api as IBeaconStateApi} from "@chainsafe/lodestar-api/lib/routes/beacon/state"; import { BeaconStateAllForks, - CachedBeaconStateAllForks, CachedBeaconStateAltair, computeEpochAtSlot, getCurrentEpoch, diff --git a/packages/lodestar/test/spec/ssz/generic/types.ts b/packages/lodestar/test/spec/ssz/generic/types.ts index 31bbeb633cb1..387e198867c1 100644 --- a/packages/lodestar/test/spec/ssz/generic/types.ts +++ b/packages/lodestar/test/spec/ssz/generic/types.ts @@ -126,7 +126,8 @@ export function getTestType(testType: string, testCase: string): Type { // {limit}: the list limit, in bits, of the bitlist. case "bitlist": { // Consider case `bitlist_no_delimiter_empty` - const limit = testCase.includes("no_delimiter") ? 0 : parseSecondNum(testCase, "limit"); + // Set bitLen to a random big value. 0 is invalid and will throw at the constructor + const limit = testCase.includes("no_delimiter") ? 1024 : parseSecondNum(testCase, "limit"); // TODO: memoize return new BitListType(limit); } From 40c2fc59c3a2c8d9238d5aa1aece2350733dafbc Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:58:41 +0530 Subject: [PATCH 03/30] Don't mutate with deepmerge --- .../unit/api/impl/beacon/blocks/getBlockHeaders.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts index f916e61dcabe..2163e56e3f51 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts @@ -6,7 +6,6 @@ import { generateEmptySignedBlock, generateSignedBlock, } from "../../../../../utils/block"; -import deepmerge from "deepmerge"; import {expect} from "chai"; import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; import {toHexString} from "@chainsafe/ssz"; @@ -31,7 +30,11 @@ describe("api - beacon - getBlockHeaders", function () { blockRoot: toHexString(ssz.phase0.BeaconBlock.hashTreeRoot(generateEmptyBlock())), }, ]); - server.dbStub.block.get.resolves(deepmerge(generateEmptySignedBlock(), {message: {slot: 3}})); + + const blockFromDb3 = generateEmptySignedBlock(); + blockFromDb3.message.slot = 3; + server.dbStub.block.get.resolves(blockFromDb3); + server.dbStub.blockArchive.get.resolves(null); const {data: blockHeaders} = await server.blockApi.getBlockHeaders({}); expect(blockHeaders).to.not.be.null; From b0a1d8d3d75396d095b5346bef883e2a5d3b8df7 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:59:10 +0530 Subject: [PATCH 04/30] Use correct casing for receiptRoot --- .../src/bellatrix/block/processExecutionPayload.ts | 2 +- packages/lodestar/src/executionEngine/http.ts | 4 ++-- packages/lodestar/src/executionEngine/mock.ts | 2 +- packages/types/src/bellatrix/sszTypes.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts index 92ef117fde2c..de719e717482 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts @@ -55,7 +55,7 @@ export function processExecutionPayload( parentHash: payload.parentHash, feeRecipient: payload.feeRecipient, stateRoot: payload.stateRoot, - receiptsRoot: payload.receiptsRoot, + receiptRoot: payload.receiptRoot, logsBloom: payload.logsBloom, random: payload.random, blockNumber: payload.blockNumber, diff --git a/packages/lodestar/src/executionEngine/http.ts b/packages/lodestar/src/executionEngine/http.ts index f6b47cd9b829..6322f5f69dfe 100644 --- a/packages/lodestar/src/executionEngine/http.ts +++ b/packages/lodestar/src/executionEngine/http.ts @@ -323,7 +323,7 @@ export function serializeExecutionPayload(data: bellatrix.ExecutionPayload): Exe parentHash: bytesToData(data.parentHash), feeRecipient: bytesToData(data.feeRecipient), stateRoot: bytesToData(data.stateRoot), - receiptsRoot: bytesToData(data.receiptsRoot), + receiptsRoot: bytesToData(data.receiptRoot), logsBloom: bytesToData(data.logsBloom), random: bytesToData(data.random), blockNumber: numToQuantity(data.blockNumber), @@ -342,7 +342,7 @@ export function parseExecutionPayload(data: ExecutionPayloadRpc): bellatrix.Exec parentHash: dataToBytes(data.parentHash, 32), feeRecipient: dataToBytes(data.feeRecipient, 20), stateRoot: dataToBytes(data.stateRoot, 32), - receiptsRoot: dataToBytes(data.receiptsRoot, 32), + receiptRoot: dataToBytes(data.receiptsRoot, 32), logsBloom: dataToBytes(data.logsBloom, BYTES_PER_LOGS_BLOOM), random: dataToBytes(data.random, 32), blockNumber: quantityToNum(data.blockNumber), diff --git a/packages/lodestar/src/executionEngine/mock.ts b/packages/lodestar/src/executionEngine/mock.ts index ede6b0bbf3f3..23e44adcf4f3 100644 --- a/packages/lodestar/src/executionEngine/mock.ts +++ b/packages/lodestar/src/executionEngine/mock.ts @@ -33,7 +33,7 @@ export class ExecutionEngineMock implements IExecutionEngine { parentHash: ZERO_HASH, feeRecipient: Buffer.alloc(20, 0), stateRoot: ZERO_HASH, - receiptsRoot: ZERO_HASH, + receiptRoot: ZERO_HASH, logsBloom: Buffer.alloc(BYTES_PER_LOGS_BLOOM, 0), random: ZERO_HASH, blockNumber: 0, diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 661acbf1e346..2bcac077a0d2 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -31,7 +31,7 @@ const executionPayloadFields = { parentHash: Root, feeRecipient: Bytes20, stateRoot: Bytes32, - receiptsRoot: Bytes32, + receiptRoot: Bytes32, logsBloom: new ByteVectorType(BYTES_PER_LOGS_BLOOM), random: Bytes32, blockNumber: UintNum64, From d7197294d5f312bc64625e95e19c9a7cce88016f Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 12:20:33 +0530 Subject: [PATCH 05/30] Fix receiptsRoot --- packages/lodestar/src/executionEngine/mock.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lodestar/src/executionEngine/mock.ts b/packages/lodestar/src/executionEngine/mock.ts index 23e44adcf4f3..b745fbb01938 100644 --- a/packages/lodestar/src/executionEngine/mock.ts +++ b/packages/lodestar/src/executionEngine/mock.ts @@ -107,7 +107,7 @@ export class ExecutionEngineMock implements IExecutionEngine { parentHash: headBlockHash, feeRecipient: payloadAttributes.suggestedFeeRecipient, stateRoot: crypto.randomBytes(32), - receiptsRoot: crypto.randomBytes(32), + receiptRoot: crypto.randomBytes(32), logsBloom: crypto.randomBytes(BYTES_PER_LOGS_BLOOM), random: payloadAttributes.random, blockNumber: parentPayload.blockNumber + 1, From da8f3587ea1c758fb0407936835da701695757d2 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:11:05 +0530 Subject: [PATCH 06/30] Use bool array in processJustificationAndFinalization --- .../processJustificationAndFinalization.ts | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts index dcda127dcde8..2f62647f7756 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts @@ -1,4 +1,5 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; import {getBlockRoot} from "../../util"; import {CachedBeaconStateAllForks, EpochProcess} from "../../types"; @@ -26,45 +27,48 @@ export function processJustificationAndFinalization( // Process justifications state.previousJustifiedCheckpoint = state.currentJustifiedCheckpoint; - const bits = state.justificationBits; - for (let i = bits.bitLen - 1; i >= 1; i--) { - bits.set(i, bits.get(i - 1)); + const bits = state.justificationBits.toBoolArray(); + + // Rotate bits + for (let i = bits.length - 1; i >= 1; i--) { + bits[i] = bits[i - 1]; } - bits.set(0, false); + bits[0] = false; if (epochProcess.prevEpochUnslashedStake.targetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: previousEpoch, root: getBlockRoot(state, previousEpoch), }); - bits.set(1, true); + bits[1] = true; } if (epochProcess.currEpochUnslashedTargetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: currentEpoch, root: getBlockRoot(state, currentEpoch), }); - bits.set(0, true); + bits[1] = true; } - state.justificationBits = bits; + + state.justificationBits = ssz.phase0.JustificationBits.toViewDU(BitArray.fromBoolArray(bits)); // TODO: Consider rendering bits as array of boolean for faster repeated access here // Process finalizations // The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source - if (bits.get(1) && bits.get(2) && bits.get(3) && oldPreviousJustifiedCheckpoint.epoch + 3 === currentEpoch) { + if (bits[1] && bits[2] && bits[3] && oldPreviousJustifiedCheckpoint.epoch + 3 === currentEpoch) { state.finalizedCheckpoint = oldPreviousJustifiedCheckpoint; } // The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source - if (bits.get(1) && bits.get(2) && oldPreviousJustifiedCheckpoint.epoch + 2 === currentEpoch) { + if (bits[1] && bits[2] && oldPreviousJustifiedCheckpoint.epoch + 2 === currentEpoch) { state.finalizedCheckpoint = oldPreviousJustifiedCheckpoint; } // The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source - if (bits.get(0) && bits.get(1) && bits.get(2) && oldCurrentJustifiedCheckpoint.epoch + 2 === currentEpoch) { + if (bits[0] && bits[1] && bits[2] && oldCurrentJustifiedCheckpoint.epoch + 2 === currentEpoch) { state.finalizedCheckpoint = oldCurrentJustifiedCheckpoint; } // The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source - if (bits.get(0) && bits.get(1) && oldCurrentJustifiedCheckpoint.epoch + 1 === currentEpoch) { + if (bits[0] && bits[1] && oldCurrentJustifiedCheckpoint.epoch + 1 === currentEpoch) { state.finalizedCheckpoint = oldCurrentJustifiedCheckpoint; } } From 1ace537fc53d3f81915feea0d9d0706f0180b747 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:11:21 +0530 Subject: [PATCH 07/30] Simplify SignatureSets computation --- .../allForks/signatureSets/attesterSlashings.ts | 14 +++----------- .../allForks/signatureSets/indexedAttestation.ts | 10 +++------- .../allForks/signatureSets/proposerSlashings.ts | 14 +++----------- .../src/allForks/signatureSets/voluntaryExits.ts | 10 +++------- 4 files changed, 12 insertions(+), 36 deletions(-) diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts index 7680504f5572..cd6952c683a0 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts @@ -18,15 +18,7 @@ export function getAttesterSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - const signatureSets: ISignatureSet[] = []; - - for (const attesterSlashing of signedBlock.message.body.attesterSlashings) { - const attesterSlashingSigSets = getAttesterSlashingSignatureSets(state, attesterSlashing); - - for (const signatureSet of attesterSlashingSigSets) { - signatureSets.push(signatureSet); - } - } - - return signatureSets; + return signedBlock.message.body.attesterSlashings + .map((attesterSlashing) => getAttesterSlashingSignatureSets(state, attesterSlashing)) + .flat(1); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts index 5dc29de7c085..eeaaf4fa5310 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts @@ -50,11 +50,7 @@ export function getAttestationsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - const signatureSets: ISignatureSet[] = []; - - for (const attestation of signedBlock.message.body.attestations) { - signatureSets.push(getIndexedAttestationSignatureSet(state, state.epochCtx.getIndexedAttestation(attestation))); - } - - return signatureSets; + return signedBlock.message.body.attestations.map((attestation) => + getIndexedAttestationSignatureSet(state, state.epochCtx.getIndexedAttestation(attestation)) + ); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts index 5d6c9a11af44..8a939383ac50 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts @@ -32,15 +32,7 @@ export function getProposerSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - const signatureSets: ISignatureSet[] = []; - - for (const proposerSlashing of signedBlock.message.body.proposerSlashings) { - const proposerSlashingSigSets = getProposerSlashingSignatureSets(state, proposerSlashing); - - for (const signatureSet of proposerSlashingSigSets) { - signatureSets.push(signatureSet); - } - } - - return signatureSets; + return signedBlock.message.body.proposerSlashings + .map((proposerSlashing) => getProposerSlashingSignatureSets(state, proposerSlashing)) + .flat(1); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts index bdbbb163efc3..8be8f053eeac 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts @@ -39,11 +39,7 @@ export function getVoluntaryExitsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - const signatureSets: ISignatureSet[] = []; - - for (const voluntaryExit of signedBlock.message.body.voluntaryExits) { - signatureSets.push(getVoluntaryExitSignatureSet(state, voluntaryExit)); - } - - return signatureSets; + return signedBlock.message.body.voluntaryExits.map((voluntaryExit) => + getVoluntaryExitSignatureSet(state, voluntaryExit) + ); } From 5b017d843b9a6771467f3fc69311ac4ed37e2884 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:34:33 +0530 Subject: [PATCH 08/30] Fix typo in processJustificationAndFinalization --- .../src/allForks/epoch/processJustificationAndFinalization.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts index 2f62647f7756..2d0dd10e7d2c 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts @@ -47,7 +47,7 @@ export function processJustificationAndFinalization( epoch: currentEpoch, root: getBlockRoot(state, currentEpoch), }); - bits[1] = true; + bits[0] = true; } state.justificationBits = ssz.phase0.JustificationBits.toViewDU(BitArray.fromBoolArray(bits)); From b2e67d0fcf3d1fba073f626e1aede6145eed432b Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:34:44 +0530 Subject: [PATCH 09/30] Use declarative syntax in processEth1DataReset --- .../src/allForks/epoch/processEth1DataReset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts index 1a2293c292e5..a3aac340e11d 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts @@ -12,6 +12,6 @@ export function processEth1DataReset(state: CachedBeaconStateAllForks, epochProc // reset eth1 data votes if (nextEpoch % EPOCHS_PER_ETH1_VOTING_PERIOD === 0) { - state.eth1DataVotes = ssz.phase0.Eth1DataVotes.defaultViewDU; + state.eth1DataVotes = ssz.phase0.Eth1DataVotes.toViewDU([]); } } From 085bea8ba6cc25def162a0dd38c62b142fe6df6a Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:35:04 +0530 Subject: [PATCH 10/30] Create tree for free in processParticipationFlagUpdates --- .../epoch/processParticipationFlagUpdates.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts index 30dd4967a689..3c08322d8fd5 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts @@ -1,5 +1,5 @@ +import {zeroNode} from "@chainsafe/persistent-merkle-tree"; import {ssz} from "@chainsafe/lodestar-types"; -import {newZeroedArray} from "../../util"; import {CachedBeaconStateAltair} from "../../types"; /** @@ -13,8 +13,15 @@ export function processParticipationFlagUpdates(state: CachedBeaconStateAltair): // Set view and tree from currentEpochParticipation to previousEpochParticipation state.previousEpochParticipation = state.currentEpochParticipation; - // Wipe currentEpochParticipation with an empty value - const currentEpochParticipationArr = newZeroedArray(state.currentEpochParticipation.length); - // TODO: Benchmark the cost of transforming to .toViewDU() - state.currentEpochParticipation = ssz.altair.EpochParticipation.toViewDU(currentEpochParticipationArr); + // We need to replace the node of currentEpochParticipation with a node that represents and empty list of some length. + // SSZ represents a list as = new BranchNode(chunksNode, lengthNode). + // Since the chunks represent all zero'ed data we can re-use the pre-compouted zeroNode at chunkDepth to skip any + // data transformation and create the required tree almost for free. + const currentEpochParticipationNode = ssz.altair.EpochParticipation.tree_setChunksNode( + state.currentEpochParticipation.node, + zeroNode(ssz.altair.EpochParticipation.chunkDepth), + state.currentEpochParticipation.length + ); + + state.currentEpochParticipation = ssz.altair.EpochParticipation.getViewDU(currentEpochParticipationNode); } From 42a162c86e6afb5ddc4d84900ff866adb48ebe40 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:53:11 +0530 Subject: [PATCH 11/30] Fix benchmark title --- .../test/perf/altair/block/processBlock.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts index 6dd70c2523ac..13ceea7b7e3c 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts @@ -102,7 +102,7 @@ describe("altair processBlock", () => { for (const {id, opts} of testCases) { for (const hashState of [false, true]) { itBench({ - id: `altair processBlock - ${perfStateId} ${id}` + hashState ? " hashState" : "", + id: `altair processBlock - ${perfStateId} ${id}` + (hashState ? " hashState" : ""), before: () => { const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; const block = getBlockAltair(state as CachedBeaconStateAltair, opts); From 9f61483297a8292e1f6e17ab5622e1d1b42e8ec0 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 14:56:11 +0530 Subject: [PATCH 12/30] Fix processSyncCommitteeUpdates perf test condition --- .../test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index e6c9b70813dc..3b1a67560699 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -14,7 +14,7 @@ describe("altair processSyncCommitteeUpdates", () => { yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), beforeEach: (state) => { - if (state.epochCtx.epoch + (1 % EPOCHS_PER_SYNC_COMMITTEE_PERIOD) === 0) { + if ((state.epochCtx.epoch + 1) % EPOCHS_PER_SYNC_COMMITTEE_PERIOD === 0) { // OK will run } else { throw Error("processSyncCommitteeUpdates will not rotate syncCommittees"); From e27248afe59fa20f646091cfa18010a588780f83 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 15:07:00 +0530 Subject: [PATCH 13/30] Fix processInactivityUpdates benchmark --- .../test/perf/phase0/epoch/util.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts index 7dfcfe79d624..91c863674dce 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts @@ -1,4 +1,4 @@ -import {AttesterFlags, IAttesterStatus, toAttesterFlags} from "../../../../src"; +import {AttesterFlags, FLAG_ELIGIBLE_ATTESTER, hasMarkers, IAttesterStatus, toAttesterFlags} from "../../../../src"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair, EpochProcess} from "../../../../src/types"; /** @@ -13,8 +13,17 @@ export function generateBalanceDeltasEpochProcess( ): EpochProcess { const vc = state.validators.length; + const statuses = generateStatuses(state.validators.length, flagFactors); + const eligibleValidatorIndices: number[] = []; + for (let i = 0; i < statuses.length; i++) { + if (hasMarkers(statuses[i].flags, FLAG_ELIGIBLE_ATTESTER)) { + eligibleValidatorIndices.push(i); + } + } + const epochProcess: Partial = { - statuses: generateStatuses(state.validators.length, flagFactors), + statuses, + eligibleValidatorIndices, totalActiveStakeByIncrement: vc, baseRewardPerIncrement: 726, prevEpochUnslashedStake: { @@ -32,17 +41,17 @@ export type FlagFactors = Record | number; function generateStatuses(vc: number, flagFactors: FlagFactors): IAttesterStatus[] { const totalProposers = 32; - const statuses: IAttesterStatus[] = []; + const statuses = new Array(vc); for (let i = 0; i < vc; i++) { // Set to number to set all validators to the same value if (typeof flagFactors === "number") { - statuses.push({ + statuses[i] = { flags: flagFactors, proposerIndex: i % totalProposers, inclusionDelay: 1 + (i % 4), active: true, - }); + }; } else { // Use a factor to set some validators to this flag const flagsObj: AttesterFlags = { @@ -55,12 +64,12 @@ function generateStatuses(vc: number, flagFactors: FlagFactors): IAttesterStatus unslashed: i < vc * flagFactors.unslashed, // 6 eligibleAttester: i < vc * flagFactors.eligibleAttester, // 7 }; - statuses.push({ + statuses[i] = { flags: toAttesterFlags(flagsObj), proposerIndex: i % totalProposers, inclusionDelay: 1 + (i % 4), active: true, - }); + }; } } From fb40fd0cab191690edf4aa612877dede3ea3e308 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Feb 2022 16:46:07 +0530 Subject: [PATCH 14/30] Force processSyncCommitteeUpdates perf test to run --- .../epoch/processSyncCommitteeUpdates.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index 3b1a67560699..a8d3fab91ec8 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -14,16 +14,17 @@ describe("altair processSyncCommitteeUpdates", () => { yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), beforeEach: (state) => { - if ((state.epochCtx.epoch + 1) % EPOCHS_PER_SYNC_COMMITTEE_PERIOD === 0) { - // OK will run - } else { - throw Error("processSyncCommitteeUpdates will not rotate syncCommittees"); - } - - return state.clone(); + const stateCloned = state.clone(); + // Force processSyncCommitteeUpdates to run + state.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; + return stateCloned; }, fn: (state) => { + const nextSyncCommitteeBefore = state.nextSyncCommittee; processSyncCommitteeUpdates(state); + if (state.nextSyncCommittee === nextSyncCommitteeBefore) { + throw Error("nextSyncCommittee instance has not changed"); + } }, }); }); From ba5eea1f86ef3a61454deb091138dd83f68d73c4 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 17 Feb 2022 09:50:14 +0530 Subject: [PATCH 15/30] Fix rebase issues --- packages/lodestar/src/chain/blocks/verifyBlock.ts | 4 ++-- packages/lodestar/src/chain/eventHandlers.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/lodestar/src/chain/blocks/verifyBlock.ts b/packages/lodestar/src/chain/blocks/verifyBlock.ts index 6b361b806e6a..5fa267c6dc1d 100644 --- a/packages/lodestar/src/chain/blocks/verifyBlock.ts +++ b/packages/lodestar/src/chain/blocks/verifyBlock.ts @@ -266,10 +266,10 @@ export async function verifyBlockStateTransition( } // Check state root matches - if (!byteArrayEquals(block.message.stateRoot, postState.tree.root)) { + if (!byteArrayEquals(block.message.stateRoot, postState.hashTreeRoot())) { throw new BlockError(block, { code: BlockErrorCode.INVALID_STATE_ROOT, - root: postState.tree.root, + root: postState.hashTreeRoot(), expectedRoot: block.message.stateRoot, preState, postState, diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 829a739d3ded..d9cbe42257c3 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -1,7 +1,7 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import {toHexString} from "@chainsafe/ssz"; import {allForks, Epoch, phase0, Slot, ssz, Version} from "@chainsafe/lodestar-types"; -import {Context, ILogger} from "@chainsafe/lodestar-utils"; +import {ILogger} from "@chainsafe/lodestar-utils"; import {CheckpointWithHex, IProtoBlock} from "@chainsafe/lodestar-fork-choice"; import { BeaconStateAllForks, From b0b7fa06a86efd1249d08612b67ca5dfa1b4cd3b Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 17 Feb 2022 10:07:51 +0530 Subject: [PATCH 16/30] Fix merge issues --- .../cmds/account/cmds/validator/slashingProtection/export.ts | 3 +-- packages/cli/src/cmds/beacon/initBeaconState.ts | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts index 50ee28043199..bb7d3aaa697f 100644 --- a/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts @@ -1,5 +1,4 @@ import {InterchangeFormatVersion} from "@chainsafe/lodestar-validator"; -import {Context} from "@chainsafe/lodestar-utils"; import {ICliCommand, writeFile} from "../../../../../util"; import {IGlobalArgs} from "../../../../../options"; import {IAccountValidatorArgs} from "../options"; @@ -41,7 +40,7 @@ export const exportCmd: ICliCommand Date: Sun, 20 Feb 2022 11:45:29 +0530 Subject: [PATCH 17/30] Use exact commit --- packages/api/package.json | 2 +- packages/beacon-state-transition/package.json | 2 +- packages/cli/package.json | 2 +- packages/config/package.json | 2 +- packages/db/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 2 +- packages/lodestar/package.json | 2 +- packages/types/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 13 ++++++------- 11 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 99cb5f75f9bd..1b8964a95376 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -41,7 +41,7 @@ "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "cross-fetch": "^3.1.4", "eventsource": "^1.1.0", "qs": "^6.10.1" diff --git a/packages/beacon-state-transition/package.json b/packages/beacon-state-transition/package.json index 60af4fd6a5fd..03d1f9240d8f 100644 --- a/packages/beacon-state-transition/package.json +++ b/packages/beacon-state-transition/package.json @@ -43,7 +43,7 @@ "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index 8d585b4b0afb..24c109a15217 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -63,7 +63,7 @@ "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/lodestar-validator": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "@types/lockfile": "^1.0.1", "bip39": "^3.0.2", "deepmerge": "^4.2.2", diff --git a/packages/config/package.json b/packages/config/package.json index eeb25c3d58c0..9809a37624d4 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -44,6 +44,6 @@ "dependencies": { "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" } } diff --git a/packages/db/package.json b/packages/db/package.json index 058087cdcac3..b197f06cbb94 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -38,7 +38,7 @@ "dependencies": { "@chainsafe/lodestar-config": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "@types/levelup": "^4.3.3", "it-all": "^1.0.2", "level": "^7.0.0", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 638d336420e4..4b181076be79 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -42,7 +42,7 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 8a2ee00a3036..965b50d7b10f 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -43,7 +43,7 @@ "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "cross-fetch": "^3.1.4", "mitt": "^3.0.0" }, diff --git a/packages/lodestar/package.json b/packages/lodestar/package.json index a6fa856e0096..6b974584b944 100644 --- a/packages/lodestar/package.json +++ b/packages/lodestar/package.json @@ -76,7 +76,7 @@ "@chainsafe/lodestar-validator": "^0.35.0", "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", "@chainsafe/snappy-stream": "5.0.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "@ethersproject/abi": "^5.0.0", "@types/datastore-level": "^3.0.0", "bl": "^5.0.0", diff --git a/packages/types/package.json b/packages/types/package.json index 2e3b342320f5..9595c38f1bc7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -36,7 +36,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/lodestar-params": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2" + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" }, "keywords": [ "ethereum", diff --git a/packages/validator/package.json b/packages/validator/package.json index cded35d6cd80..993ea86d7820 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -54,7 +54,7 @@ "@chainsafe/lodestar-params": "^0.35.0", "@chainsafe/lodestar-types": "^0.35.0", "@chainsafe/lodestar-utils": "^0.35.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?dapplion/v2", + "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", "bigint-buffer": "^1.1.5", "cross-fetch": "^3.1.4", "strict-event-emitter-types": "^2.0.0" diff --git a/yarn.lock b/yarn.lock index a239582afae6..dc4348ca06a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -488,10 +488,10 @@ protobufjs "^6.11.2" uint8arrays "^3.0.0" -"@chainsafe/persistent-merkle-tree@^0.3.7": +"@chainsafe/persistent-merkle-tree@https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2": version "0.3.7" - resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.3.7.tgz#d257674fcacf1770e54df1e7e476bb55efcb3ed4" - integrity sha512-UXXzvCN8P4eQWCsaTaHRrNa+2sTwY3NB0Vf+uWgM4cU1qf8LOsTU7aU2CeXFAGJag5W/xEQfVGNh463kXTYAgg== + uid "089475892f3e5aae15f767c9f360b2e23af65638" + resolved "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2#089475892f3e5aae15f767c9f360b2e23af65638" dependencies: "@chainsafe/as-sha256" "^0.2.3" @@ -512,13 +512,12 @@ buffer-from "^1.1.1" snappy "^6.3.5" -"@chainsafe/ssz@^0.8.20": +"@chainsafe/ssz@https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2": version "0.8.20" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.8.20.tgz#2c8e70e173a2540c8e4e0bad7c44fd836b8b256b" - integrity sha512-SjD/GqkByWbC5JC2W4cvaFjok6kfZeZDLLzr9k0Kn4sIOZPBfuRTumvV6ihqgn7ItOFpZRXMz0u4DgH+lr28WQ== + resolved "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2#dd44c159620efafcd31a0aecfd257f57bdc085d8" dependencies: "@chainsafe/as-sha256" "^0.2.4" - "@chainsafe/persistent-merkle-tree" "^0.3.7" + "@chainsafe/persistent-merkle-tree" "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2" case "^1.6.3" "@cspotcode/source-map-consumer@0.8.0": From 9e29e4e6c5aa07d42bf55917bbba06e9433eba7c Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:19:09 +0530 Subject: [PATCH 18/30] Fix perf test altair processSyncCommitteeUpdates --- .../test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index a8d3fab91ec8..b93ac7104fcd 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -16,7 +16,7 @@ describe("altair processSyncCommitteeUpdates", () => { beforeEach: (state) => { const stateCloned = state.clone(); // Force processSyncCommitteeUpdates to run - state.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; + stateCloned.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; return stateCloned; }, fn: (state) => { From c12dc8aa980e26399ffceeebe7e6a56048d7f0bd Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 14:19:44 +0530 Subject: [PATCH 19/30] Rename receiptRoot to receiptsRoot --- .../src/bellatrix/block/processExecutionPayload.ts | 2 +- packages/lodestar/src/executionEngine/http.ts | 4 ++-- packages/lodestar/src/executionEngine/mock.ts | 4 ++-- packages/types/src/bellatrix/sszTypes.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts index de719e717482..92ef117fde2c 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts @@ -55,7 +55,7 @@ export function processExecutionPayload( parentHash: payload.parentHash, feeRecipient: payload.feeRecipient, stateRoot: payload.stateRoot, - receiptRoot: payload.receiptRoot, + receiptsRoot: payload.receiptsRoot, logsBloom: payload.logsBloom, random: payload.random, blockNumber: payload.blockNumber, diff --git a/packages/lodestar/src/executionEngine/http.ts b/packages/lodestar/src/executionEngine/http.ts index 6322f5f69dfe..f6b47cd9b829 100644 --- a/packages/lodestar/src/executionEngine/http.ts +++ b/packages/lodestar/src/executionEngine/http.ts @@ -323,7 +323,7 @@ export function serializeExecutionPayload(data: bellatrix.ExecutionPayload): Exe parentHash: bytesToData(data.parentHash), feeRecipient: bytesToData(data.feeRecipient), stateRoot: bytesToData(data.stateRoot), - receiptsRoot: bytesToData(data.receiptRoot), + receiptsRoot: bytesToData(data.receiptsRoot), logsBloom: bytesToData(data.logsBloom), random: bytesToData(data.random), blockNumber: numToQuantity(data.blockNumber), @@ -342,7 +342,7 @@ export function parseExecutionPayload(data: ExecutionPayloadRpc): bellatrix.Exec parentHash: dataToBytes(data.parentHash, 32), feeRecipient: dataToBytes(data.feeRecipient, 20), stateRoot: dataToBytes(data.stateRoot, 32), - receiptRoot: dataToBytes(data.receiptsRoot, 32), + receiptsRoot: dataToBytes(data.receiptsRoot, 32), logsBloom: dataToBytes(data.logsBloom, BYTES_PER_LOGS_BLOOM), random: dataToBytes(data.random, 32), blockNumber: quantityToNum(data.blockNumber), diff --git a/packages/lodestar/src/executionEngine/mock.ts b/packages/lodestar/src/executionEngine/mock.ts index b745fbb01938..ede6b0bbf3f3 100644 --- a/packages/lodestar/src/executionEngine/mock.ts +++ b/packages/lodestar/src/executionEngine/mock.ts @@ -33,7 +33,7 @@ export class ExecutionEngineMock implements IExecutionEngine { parentHash: ZERO_HASH, feeRecipient: Buffer.alloc(20, 0), stateRoot: ZERO_HASH, - receiptRoot: ZERO_HASH, + receiptsRoot: ZERO_HASH, logsBloom: Buffer.alloc(BYTES_PER_LOGS_BLOOM, 0), random: ZERO_HASH, blockNumber: 0, @@ -107,7 +107,7 @@ export class ExecutionEngineMock implements IExecutionEngine { parentHash: headBlockHash, feeRecipient: payloadAttributes.suggestedFeeRecipient, stateRoot: crypto.randomBytes(32), - receiptRoot: crypto.randomBytes(32), + receiptsRoot: crypto.randomBytes(32), logsBloom: crypto.randomBytes(BYTES_PER_LOGS_BLOOM), random: payloadAttributes.random, blockNumber: parentPayload.blockNumber + 1, diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 2bcac077a0d2..661acbf1e346 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -31,7 +31,7 @@ const executionPayloadFields = { parentHash: Root, feeRecipient: Bytes20, stateRoot: Bytes32, - receiptRoot: Bytes32, + receiptsRoot: Bytes32, logsBloom: new ByteVectorType(BYTES_PER_LOGS_BLOOM), random: Bytes32, blockNumber: UintNum64, From 593e7a9b246c36f96dd6cfbbeb106d0fd302057f Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 15:20:18 +0530 Subject: [PATCH 20/30] Remove unnecessary dif --- .../cli/src/cmds/account/cmds/validator/create.ts | 2 +- .../cli/src/cmds/account/cmds/validator/recover.ts | 2 +- packages/light-client/src/utils/utils.ts | 13 ++----------- packages/light-client/test/getGenesisData.ts | 2 +- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/cmds/account/cmds/validator/create.ts b/packages/cli/src/cmds/account/cmds/validator/create.ts index 1ef8939ab097..ef9ae8ec3584 100644 --- a/packages/cli/src/cmds/account/cmds/validator/create.ts +++ b/packages/cli/src/cmds/account/cmds/validator/create.ts @@ -80,7 +80,7 @@ and pre-computed deposit RPL data", const {name, passphraseFile, storeWithdrawalKeystore, count} = args; const accountPaths = getAccountPaths(args); const maxEffectiveBalance = MAX_EFFECTIVE_BALANCE; - const depositGwei = args.depositGwei !== undefined ? parseInt(args.depositGwei, 10) : maxEffectiveBalance; + const depositGwei = Number(args.depositGwei || 0) || maxEffectiveBalance; if (depositGwei > maxEffectiveBalance) throw new YargsError(`depositGwei ${depositGwei} is higher than MAX_EFFECTIVE_BALANCE ${maxEffectiveBalance}`); diff --git a/packages/cli/src/cmds/account/cmds/validator/recover.ts b/packages/cli/src/cmds/account/cmds/validator/recover.ts index f1366dc69dd3..08e163632834 100644 --- a/packages/cli/src/cmds/account/cmds/validator/recover.ts +++ b/packages/cli/src/cmds/account/cmds/validator/recover.ts @@ -65,7 +65,7 @@ export const recover: ICliCommand(pubkeys: T[], bits: BitArray): T[] { - if (bits.bitLen > pubkeys.length) { - throw Error(`syncCommittee bitLen ${bits.bitLen} > pubkeys.length ${pubkeys.length}`); - } - + // BitArray.intersectValues() checks the length is correct return bits.intersectValues(pubkeys); } diff --git a/packages/light-client/test/getGenesisData.ts b/packages/light-client/test/getGenesisData.ts index 7c812420174e..2679932e901b 100644 --- a/packages/light-client/test/getGenesisData.ts +++ b/packages/light-client/test/getGenesisData.ts @@ -19,7 +19,7 @@ async function getGenesisData(): Promise { const api = getClient(config, {baseUrl}); const {data: genesis} = await api.beacon.getGenesis(); console.log(network, { - genesisTime: genesis.genesisTime, + genesisTime: Number(genesis.genesisTime), genesisValidatorsRoot: "0x" + Buffer.from(genesis.genesisValidatorsRoot as Uint8Array).toString("hex"), }); } From a7dff5772cb9c6e073b8de37bef80dda9681cb6d Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 15:47:04 +0530 Subject: [PATCH 21/30] Remove unnecessary type casting --- .../src/altair/block/index.ts | 8 +++---- .../src/altair/block/processAttestation.ts | 8 ++----- .../altair/block/processAttesterSlashing.ts | 9 ++------ .../src/altair/block/processDeposit.ts | 4 ++-- .../altair/block/processProposerSlashing.ts | 9 ++------ .../src/altair/block/processSyncCommittee.ts | 6 ++--- .../src/altair/block/processVoluntaryExit.ts | 4 ++-- .../src/altair/epoch/index.ts | 14 +++++------ .../altair/epoch/processInactivityUpdates.ts | 2 +- .../src/altair/epoch/processSlashings.ts | 4 ++-- .../src/altair/upgradeState.ts | 4 ++-- .../src/bellatrix/block/index.ts | 10 ++++---- .../block/processAttesterSlashing.ts | 9 ++------ .../block/processProposerSlashing.ts | 9 ++------ .../src/phase0/block/index.ts | 8 +++---- .../src/phase0/block/processAttestation.ts | 10 ++------ .../phase0/block/processAttesterSlashing.ts | 9 ++------ .../src/phase0/block/processDeposit.ts | 4 ++-- .../phase0/block/processProposerSlashing.ts | 9 ++------ .../src/phase0/block/processVoluntaryExit.ts | 4 ++-- .../src/phase0/epoch/index.ts | 16 ++++++------- .../src/phase0/epoch/processSlashings.ts | 4 ++-- .../perf/allForks/util/epochContext.test.ts | 2 +- .../perf/allForks/util/shufflings.test.ts | 2 +- .../altair/block/processAttestation.test.ts | 2 +- .../perf/altair/block/processBlock.test.ts | 4 ++-- .../perf/altair/block/processEth1Data.test.ts | 2 +- .../test/perf/altair/epoch/epoch.test.ts | 2 +- .../test/perf/analyzeEpochs.ts | 3 +-- .../perf/phase0/block/processBlock.test.ts | 4 ++-- .../test/perf/phase0/block/util.ts | 2 +- .../phase0/epoch/afterProcessEpoch.test.ts | 4 ++-- .../phase0/epoch/beforeProcessEpoch.test.ts | 4 ++-- .../test/perf/phase0/epoch/epoch.test.ts | 2 +- .../processEffectiveBalanceUpdates.test.ts | 4 ++-- .../epoch/processRegistryUpdates.test.ts | 2 +- .../epoch/processSlashingsAllForks.test.ts | 2 +- .../test/perf/sanityCheck.test.ts | 4 ++-- .../beacon-state-transition/test/perf/util.ts | 9 ++++---- .../test/utils/state.ts | 2 +- .../fork-choice/src/forkChoice/forkChoice.ts | 9 ++++---- .../perf/protoArray/computeDeltas.test.ts | 4 +--- .../lodestar/src/chain/blocks/verifyBlock.ts | 2 +- packages/lodestar/src/chain/eventHandlers.ts | 8 ++----- .../lodestar/src/chain/factory/block/body.ts | 2 +- packages/lodestar/src/chain/initState.ts | 2 +- packages/lodestar/src/chain/regen/regen.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 7 ++---- .../lodestar/test/spec/allForks/sanity.ts | 2 +- .../test/spec/altair/operations.test.ts | 9 ++------ .../test/spec/bellatrix/operations.test.ts | 23 ++++++++++++------- .../test/spec/phase0/operations.test.ts | 9 ++------ .../unit/api/impl/validator/utils.test.ts | 2 +- .../unit/network/attestationService.test.ts | 2 +- packages/lodestar/test/utils/state.ts | 2 +- .../test/utils/validationData/attestation.ts | 3 +-- 56 files changed, 126 insertions(+), 182 deletions(-) diff --git a/packages/beacon-state-transition/src/altair/block/index.ts b/packages/beacon-state-transition/src/altair/block/index.ts index 610deee9d23d..8cf54be8cc02 100644 --- a/packages/beacon-state-transition/src/altair/block/index.ts +++ b/packages/beacon-state-transition/src/altair/block/index.ts @@ -1,6 +1,6 @@ import {altair} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processAttestations, RootCache} from "./processAttestation"; @@ -22,9 +22,9 @@ export { }; export function processBlock(state: CachedBeaconStateAltair, block: altair.BeaconBlock, verifySignatures = true): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); + processBlockHeader(state, block); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); processSyncAggregate(state, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processAttestation.ts b/packages/beacon-state-transition/src/altair/block/processAttestation.ts index a773f6135797..0215ba8d79f8 100644 --- a/packages/beacon-state-transition/src/altair/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/altair/block/processAttestation.ts @@ -41,7 +41,7 @@ export function processAttestations( for (const attestation of attestations) { const data = attestation.data; - validateAttestation(state as CachedBeaconStateAllForks, attestation); + validateAttestation(state, attestation); // Retrieve the validator indices from the attestation participation bitfield const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); @@ -51,11 +51,7 @@ export function processAttestations( // TODO: Why should we verify an indexed attestation that we just created? If it's just for the signature // we can verify only that and nothing else. if (verifySignature) { - const sigSet = getAttestationWithIndicesSignatureSet( - state as CachedBeaconStateAllForks, - attestation, - attestingIndices - ); + const sigSet = getAttestationWithIndicesSignatureSet(state, attestation, attestingIndices); if (!verifySignatureSet(sigSet)) { throw new Error("Attestation signature is not valid"); } diff --git a/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts index 5459a51575d3..a840f07e5131 100644 --- a/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -8,10 +8,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.altair, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.altair, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processDeposit.ts b/packages/beacon-state-transition/src/altair/block/processDeposit.ts index 59c42232924d..967b1640e303 100644 --- a/packages/beacon-state-transition/src/altair/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/altair/block/processDeposit.ts @@ -1,9 +1,9 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processDeposit as processDepositAllForks} from "../../allForks/block"; export function processDeposit(state: CachedBeaconStateAltair, deposit: phase0.Deposit): void { - processDepositAllForks(ForkName.altair, state as CachedBeaconStateAllForks, deposit); + processDepositAllForks(ForkName.altair, state, deposit); } diff --git a/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts index 87ff108168a2..40f5d8aa7d3e 100644 --- a/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.altair, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.altair, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts index 5c29b531dfe9..3164849864d1 100644 --- a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts +++ b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts @@ -3,12 +3,12 @@ import {DOMAIN_SYNC_COMMITTEE} from "@chainsafe/lodestar-params"; import {byteArrayEquals} from "@chainsafe/ssz"; import {computeSigningRoot, getBlockRootAtSlot, ISignatureSet, SignatureSetType, verifySignatureSet} from "../../util"; -import {CachedBeaconStateAltair} from "../../types"; +import {CachedBeaconStateAllForks} from "../../types"; import {G2_POINT_AT_INFINITY} from "../../constants"; import {getUnparticipantValues} from "../../util/array"; export function processSyncAggregate( - state: CachedBeaconStateAltair, + state: CachedBeaconStateAllForks, block: altair.BeaconBlock, verifySignatures = true ): void { @@ -45,7 +45,7 @@ export function processSyncAggregate( } export function getSyncCommitteeSignatureSet( - state: CachedBeaconStateAltair, + state: CachedBeaconStateAllForks, block: altair.BeaconBlock, /** Optional parameter to prevent computing it twice */ participantIndices?: number[] diff --git a/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts index 1d2184b05f4c..6d8fd35c3282 100644 --- a/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processVoluntaryExitAllForks} from "../../allForks/block"; export function processVoluntaryExit( @@ -7,5 +7,5 @@ export function processVoluntaryExit( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - processVoluntaryExitAllForks(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature); + processVoluntaryExitAllForks(state, signedVoluntaryExit, verifySignature); } diff --git a/packages/beacon-state-transition/src/altair/epoch/index.ts b/packages/beacon-state-transition/src/altair/epoch/index.ts index 946e616bc6e4..35249a419c05 100644 --- a/packages/beacon-state-transition/src/altair/epoch/index.ts +++ b/packages/beacon-state-transition/src/altair/epoch/index.ts @@ -26,16 +26,16 @@ export { }; export function processEpoch(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processJustificationAndFinalization(state as CachedBeaconStateAllForks, epochProcess); + processJustificationAndFinalization(state, epochProcess); processInactivityUpdates(state, epochProcess); processRewardsAndPenalties(state, epochProcess); - processRegistryUpdates(state as CachedBeaconStateAllForks, epochProcess); + processRegistryUpdates(state, epochProcess); processSlashings(state, epochProcess); - processEth1DataReset(state as CachedBeaconStateAllForks, epochProcess); - processEffectiveBalanceUpdates(state as CachedBeaconStateAllForks, epochProcess); - processSlashingsReset(state as CachedBeaconStateAllForks, epochProcess); - processRandaoMixesReset(state as CachedBeaconStateAllForks, epochProcess); - processHistoricalRootsUpdate(state as CachedBeaconStateAllForks, epochProcess); + processEth1DataReset(state, epochProcess); + processEffectiveBalanceUpdates(state, epochProcess); + processSlashingsReset(state, epochProcess); + processRandaoMixesReset(state, epochProcess); + processHistoricalRootsUpdate(state, epochProcess); processParticipationFlagUpdates(state); processSyncCommitteeUpdates(state); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts index 2ac7cc2de243..3363645444a2 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts @@ -25,7 +25,7 @@ export function processInactivityUpdates(state: CachedBeaconStateAltair, epochPr const {config, inactivityScores} = state; const {INACTIVITY_SCORE_BIAS, INACTIVITY_SCORE_RECOVERY_RATE} = config; const {statuses, eligibleValidatorIndices} = epochProcess; - const inActivityLeak = isInInactivityLeak(state as CachedBeaconStateAllForks); + const inActivityLeak = isInInactivityLeak(state); // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers} = attesterStatusUtil; diff --git a/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts b/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts index d30ff90a9015..0a4b13070197 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts @@ -1,7 +1,7 @@ import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import {processSlashingsAllForks} from "../../allForks/epoch/processSlashings"; export function processSlashings(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processSlashingsAllForks(ForkName.altair, state as CachedBeaconStateAllForks, epochProcess); + processSlashingsAllForks(ForkName.altair, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/altair/upgradeState.ts b/packages/beacon-state-transition/src/altair/upgradeState.ts index 8f8af9e3aa5e..9080c507375f 100644 --- a/packages/beacon-state-transition/src/altair/upgradeState.ts +++ b/packages/beacon-state-transition/src/altair/upgradeState.ts @@ -1,5 +1,5 @@ import {ssz} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../types"; +import {CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../types"; import {newZeroedArray} from "../util"; import {getAttestationParticipationStatus, RootCache} from "./block/processAttestation"; import {getNextSyncCommittee} from "../util/syncCommittee"; @@ -102,7 +102,7 @@ function translateParticipation( pendingAttesations: CompositeViewDU ): void { const {epochCtx} = state; - const rootCache = new RootCache(state as CachedBeaconStateAllForks); + const rootCache = new RootCache(state); const epochParticipation = state.previousEpochParticipation; for (const attestation of pendingAttesations.getAllReadonly()) { diff --git a/packages/beacon-state-transition/src/bellatrix/block/index.ts b/packages/beacon-state-transition/src/bellatrix/block/index.ts index abbcd2d1048c..8c7e3d720c0e 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/index.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/index.ts @@ -1,6 +1,6 @@ import {bellatrix} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processSyncAggregate} from "../../altair/block/processSyncCommittee"; @@ -18,15 +18,15 @@ export function processBlock( verifySignatures = true, executionEngine: ExecutionEngine | null ): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); + processBlockHeader(state, block); // The call to the process_execution_payload must happen before the call to the process_randao as the former depends // on the randao_mix computed with the reveal of the previous block. if (isExecutionEnabled(state, block.body)) { processExecutionPayload(state, block.body.executionPayload, executionEngine); } - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); - processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block, verifySignatures); + processSyncAggregate(state, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts index 7a04b955ba0b..bb6a0b3a7b2e 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -8,10 +8,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.bellatrix, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.bellatrix, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts index 8c1aa07cd579..d652d3d83a9f 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.bellatrix, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.bellatrix, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/index.ts b/packages/beacon-state-transition/src/phase0/block/index.ts index 76da378f08da..e80bec82d5c1 100644 --- a/packages/beacon-state-transition/src/phase0/block/index.ts +++ b/packages/beacon-state-transition/src/phase0/block/index.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processAttestation, validateAttestation} from "./processAttestation"; @@ -22,8 +22,8 @@ export { }; export function processBlock(state: CachedBeaconStatePhase0, block: phase0.BeaconBlock, verifySignatures = true): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body.eth1Data); + processBlockHeader(state, block); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts index b4bcef6151d1..1f4d98ae21d2 100644 --- a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts @@ -22,7 +22,7 @@ export function processAttestation( const slot = state.slot; const data = attestation.data; - validateAttestation(state as CachedBeaconStateAllForks, attestation); + validateAttestation(state, attestation); const pendingAttestation = ssz.phase0.PendingAttestation.toViewDU({ data: data, @@ -51,13 +51,7 @@ export function processAttestation( state.previousEpochAttestations.push(pendingAttestation); } - if ( - !isValidIndexedAttestation( - state as CachedBeaconStateAllForks, - epochCtx.getIndexedAttestation(attestation), - verifySignature - ) - ) { + if (!isValidIndexedAttestation(state, epochCtx.getIndexedAttestation(attestation), verifySignature)) { throw new Error("Attestation is not valid"); } } diff --git a/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts index b2a44d873f3d..58d7593b9a4f 100644 --- a/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts @@ -1,7 +1,7 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -9,10 +9,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.phase0, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.phase0, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processDeposit.ts b/packages/beacon-state-transition/src/phase0/block/processDeposit.ts index 8695899c5950..aec62bb0de1b 100644 --- a/packages/beacon-state-transition/src/phase0/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/phase0/block/processDeposit.ts @@ -1,9 +1,9 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processDeposit as processDepositAllForks} from "../../allForks/block"; export function processDeposit(state: CachedBeaconStatePhase0, deposit: phase0.Deposit): void { - processDepositAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, deposit); + processDepositAllForks(ForkName.phase0, state, deposit); } diff --git a/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts index 865bb09f7a69..b9275251a6e7 100644 --- a/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.phase0, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.phase0, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts index 8c8d6ac921c6..97adf7173a9e 100644 --- a/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processVoluntaryExitAllForks} from "../../allForks/block"; export function processVoluntaryExit( @@ -7,5 +7,5 @@ export function processVoluntaryExit( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - processVoluntaryExitAllForks(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature); + processVoluntaryExitAllForks(state, signedVoluntaryExit, verifySignature); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/index.ts b/packages/beacon-state-transition/src/phase0/epoch/index.ts index ba80adaa7f58..1fe9a87e39ed 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/index.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/index.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; import { processJustificationAndFinalization, processRegistryUpdates, @@ -16,15 +16,15 @@ import {processParticipationRecordUpdates} from "./processParticipationRecordUpd export {processRewardsAndPenalties, processSlashings, getAttestationDeltas, processParticipationRecordUpdates}; export function processEpoch(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processJustificationAndFinalization(state as CachedBeaconStateAllForks, epochProcess); + processJustificationAndFinalization(state, epochProcess); processRewardsAndPenalties(state, epochProcess); - processRegistryUpdates(state as CachedBeaconStateAllForks, epochProcess); + processRegistryUpdates(state, epochProcess); processSlashings(state, epochProcess); // inline processFinalUpdates() to follow altair and for clarity - processEth1DataReset(state as CachedBeaconStateAllForks, epochProcess); - processEffectiveBalanceUpdates(state as CachedBeaconStateAllForks, epochProcess); - processSlashingsReset(state as CachedBeaconStateAllForks, epochProcess); - processRandaoMixesReset(state as CachedBeaconStateAllForks, epochProcess); - processHistoricalRootsUpdate(state as CachedBeaconStateAllForks, epochProcess); + processEth1DataReset(state, epochProcess); + processEffectiveBalanceUpdates(state, epochProcess); + processSlashingsReset(state, epochProcess); + processRandaoMixesReset(state, epochProcess); + processHistoricalRootsUpdate(state, epochProcess); processParticipationRecordUpdates(state); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts b/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts index 2882e6d9da51..96369abeb63b 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts @@ -1,7 +1,7 @@ import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; import {processSlashingsAllForks} from "../../allForks/epoch/processSlashings"; export function processSlashings(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processSlashingsAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, epochProcess); + processSlashingsAllForks(ForkName.phase0, state, epochProcess); } diff --git a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts index 602a6d82d901..2f3ba9bcefaa 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts @@ -15,7 +15,7 @@ describe("epochCtx.getCommitteeAssignments", () => { before(function () { this.timeout(60 * 1000); - state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + state = generatePerfTestCachedStatePhase0(); epoch = computeEpochAtSlot(state.slot); // Sanity check to ensure numValidators doesn't go stale diff --git a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts index c0e279845818..e0ba1ebdb1ee 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts @@ -15,7 +15,7 @@ describe("epoch shufflings", () => { before(function () { this.timeout(60 * 1000); - state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + state = generatePerfTestCachedStatePhase0(); nextEpoch = computeEpochAtSlot(state.slot) + 1; // Sanity check to ensure numValidators doesn't go stale diff --git a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts index ea3c2f113782..7589569c357a 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts @@ -59,7 +59,7 @@ describe("altair processAttestation", () => { itBench({ id: `altair processAttestation - ${perfStateId} ${id}`, before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; + const state = generatePerfTestCachedStateAltair(); const block = getBlockAltair(state as CachedBeaconStateAltair, opts); return {state, attestations: block.message.body.attestations as phase0.Attestation[]}; }, diff --git a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts index 13ceea7b7e3c..f8f45fa46ae3 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts @@ -10,7 +10,7 @@ import { PresetName, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; -import {allForks, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; +import {allForks, CachedBeaconStateAltair} from "../../../../src"; import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../../util"; import {BlockAltairOpts, getBlockAltair} from "../../phase0/block/util"; import {StateBlock} from "../../types"; @@ -104,7 +104,7 @@ describe("altair processBlock", () => { itBench({ id: `altair processBlock - ${perfStateId} ${id}` + (hashState ? " hashState" : ""), before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; + const state = generatePerfTestCachedStateAltair(); const block = getBlockAltair(state as CachedBeaconStateAltair, opts); // Populate permanent root caches of the block ssz.altair.BeaconBlock.hashTreeRoot(block.message); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts index 31ac1fb98013..47f19bc36e49 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts @@ -26,7 +26,7 @@ describe("altair processEth1Data", () => { itBench({ id: `altair processEth1Data - ${perfStateId} ${id}`, before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; + const state = generatePerfTestCachedStateAltair(); const block = getBlockAltair(state as CachedBeaconStateAltair, { proposerSlashingLen: 0, attesterSlashingLen: 0, diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts index 0f86c4c2a5b4..b27d9ae991f3 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts @@ -28,7 +28,7 @@ describe(`altair processEpoch - ${stateId}`, () => { itBench({ id: `altair processEpoch - ${stateId}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - beforeEach: () => stateOg.value.clone() as CachedBeaconStateAllForks, + beforeEach: () => stateOg.value.clone(), fn: (state) => { const epochProcess = beforeProcessEpoch(state); altair.processEpoch(state as CachedBeaconStateAltair, epochProcess); diff --git a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts index 9455de75fcee..0751d922f289 100644 --- a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts +++ b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts @@ -8,7 +8,6 @@ import { allForks, computeEpochAtSlot, computeStartSlotAtEpoch, - CachedBeaconStateAllForks, AttesterFlags, beforeProcessEpoch, parseAttesterFlags, @@ -101,7 +100,7 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< const postState = createCachedBeaconStateTest(stateTB, config); const epochProcess = beforeProcessEpoch(postState); - allForks.processSlots(postState as CachedBeaconStateAllForks, nextEpochSlot, null); + allForks.processSlots(postState, nextEpochSlot, null); const validatorCount = state.validators.length; diff --git a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts index 27c9c04beb1f..8c11050fe88e 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts @@ -8,7 +8,7 @@ import { MAX_VOLUNTARY_EXITS, PresetName, } from "@chainsafe/lodestar-params"; -import {allForks, CachedBeaconStateAllForks} from "../../../../src"; +import {allForks} from "../../../../src"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; import {BlockOpts, getBlockPhase0} from "./util"; import {StateBlock} from "../../types"; @@ -101,7 +101,7 @@ describe("phase0 processBlock", () => { itBench({ id: `phase0 processBlock - ${perfStateId} ${id}`, before: () => { - const state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + const state = generatePerfTestCachedStatePhase0(); const block = getBlockPhase0(state, opts); state.hashTreeRoot(); return {block, state}; diff --git a/packages/beacon-state-transition/test/perf/phase0/block/util.ts b/packages/beacon-state-transition/test/perf/phase0/block/util.ts index 31bd1c953c3b..610ebda6a2c8 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/util.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/util.ts @@ -162,7 +162,7 @@ export function getBlockPhase0( */ export function getBlockAltair(preState: CachedBeaconStateAltair, opts: BlockAltairOpts): altair.SignedBeaconBlock { const emptySig = Buffer.alloc(96); - const phase0Block = getBlockPhase0(preState as CachedBeaconStateAllForks, opts); + const phase0Block = getBlockPhase0(preState, opts); const stateEpoch = computeEpochAtSlot(preState.slot); for (const attestation of phase0Block.message.body.attestations) { const attEpoch = computeEpochAtSlot(attestation.data.slot); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts index c10941424471..dadd10022ea6 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {CachedBeaconStateAllForks, beforeProcessEpoch} from "../../../../src"; +import {beforeProcessEpoch} from "../../../../src"; import {StateEpoch} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -13,7 +13,7 @@ describe("phase0 afterProcessEpoch", () => { before: () => { const state = generatePerfTestCachedStatePhase0({goBackOneSlot: true}); const epochProcess = beforeProcessEpoch(state); - return {state: state as CachedBeaconStateAllForks, epochProcess}; + return {state: state, epochProcess}; }, beforeEach: ({state, epochProcess}) => ({state: state.clone(), epochProcess}), fn: ({state, epochProcess}) => { diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts index ef9b6905f151..b0984acc7cad 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {beforeProcessEpoch, CachedBeaconStateAllForks} from "../../../../src"; +import {beforeProcessEpoch} from "../../../../src"; import {State} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -12,7 +12,7 @@ describe("phase0 beforeProcessEpoch", () => { itBench({ id: `phase0 beforeProcessEpoch - ${perfStateId}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - before: () => generatePerfTestCachedStatePhase0({goBackOneSlot: true}) as CachedBeaconStateAllForks, + before: () => generatePerfTestCachedStatePhase0({goBackOneSlot: true}), beforeEach: (state) => state.clone(), fn: (state) => { beforeProcessEpoch(state); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts index 1cdfb25e6aa2..76259bea469b 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts @@ -28,7 +28,7 @@ describe(`phase0 processEpoch - ${stateId}`, () => { itBench({ id: `phase0 processEpoch - ${stateId}`, - beforeEach: () => stateOg.value.clone() as CachedBeaconStateAllForks, + beforeEach: () => stateOg.value.clone(), fn: (state) => { const epochProcess = beforeProcessEpoch(state); phase0.processEpoch(state as CachedBeaconStatePhase0, epochProcess); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts index 03ba47dc45ec..b0b26b9e615e 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts @@ -74,11 +74,11 @@ function getEffectiveBalanceTestData( stateTree.commit(); const cachedBeaconState = createCachedBeaconStateTest(stateTree, config, {skipSyncPubkeys: true}); - const epochProcess = beforeProcessEpoch(cachedBeaconState as CachedBeaconStateAllForks); + const epochProcess = beforeProcessEpoch(cachedBeaconState); epochProcess.balances = balances; return { - state: cachedBeaconState as CachedBeaconStateAllForks, + state: cachedBeaconState, epochProcess: epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts index 7d76a2ae9bc9..e9943b22988b 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts @@ -90,7 +90,7 @@ function getRegistryUpdatesTestData( epochProcess.indicesEligibleForActivation = linspace(lengths.indicesEligibleForActivation); return { - state: state as CachedBeaconStateAllForks, + state, epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts index 572bc6b5cfe2..880d2e42283a 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts @@ -54,7 +54,7 @@ function getProcessSlashingsTestData( epochProcess.indicesToSlash = linspace(indicesToSlashLen); return { - state: state as CachedBeaconStateAllForks, + state, epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/sanityCheck.test.ts b/packages/beacon-state-transition/test/perf/sanityCheck.test.ts index 225f786f8fcd..384193ca226d 100644 --- a/packages/beacon-state-transition/test/perf/sanityCheck.test.ts +++ b/packages/beacon-state-transition/test/perf/sanityCheck.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; import {ACTIVE_PRESET, EFFECTIVE_BALANCE_INCREMENT, PresetName} from "@chainsafe/lodestar-params"; -import {beforeProcessEpoch, CachedBeaconStateAllForks} from "../../src"; +import {beforeProcessEpoch} from "../../src"; import {generatePerfTestCachedStateAltair, generatePerfTestCachedStatePhase0, perfStateId} from "./util"; describe("Perf test sanity check", function () { @@ -30,7 +30,7 @@ describe("Perf test sanity check", function () { it("targetStake is in the same range", () => { const phase0State = generatePerfTestCachedStatePhase0(); - const epochProcess = beforeProcessEpoch(phase0State as CachedBeaconStateAllForks); + const epochProcess = beforeProcessEpoch(phase0State); expect( BigInt(epochProcess.prevEpochUnslashedStake.targetStakeByIncrement) * BigInt(EFFECTIVE_BALANCE_INCREMENT) > targetStake diff --git a/packages/beacon-state-transition/test/perf/util.ts b/packages/beacon-state-transition/test/perf/util.ts index 849fb2f882f5..893e9acb3419 100644 --- a/packages/beacon-state-transition/test/perf/util.ts +++ b/packages/beacon-state-transition/test/perf/util.ts @@ -21,7 +21,6 @@ import { CachedBeaconStateAltair, BeaconStatePhase0, BeaconStateAltair, - BeaconStateAllForks, } from "../../src/types"; import {profilerLogger} from "../utils/logger"; import {interopPubkeysCached} from "../utils/interop"; @@ -202,7 +201,7 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean } if (!phase0CachedState23638) { phase0CachedState23638 = allForks.processSlots( - phase0CachedState23637 as CachedBeaconStateAllForks, + phase0CachedState23637, phase0CachedState23637.slot + 1 ) as CachedBeaconStatePhase0; phase0CachedState23638.slot += 1; @@ -243,7 +242,7 @@ export function generatePerfTestCachedStateAltair(opts?: {goBackOneSlot: boolean } if (!altairCachedState23638) { altairCachedState23638 = allForks.processSlots( - altairCachedState23637 as CachedBeaconStateAllForks, + altairCachedState23637, altairCachedState23637.slot + 1 ) as CachedBeaconStateAltair; altairCachedState23638.slot += 1; @@ -445,7 +444,7 @@ export function generateTestCachedBeaconStateOnlyValidators({ config: createIBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - }) as CachedBeaconStateAllForks; + }); } const initialValue = null; @@ -518,5 +517,5 @@ export async function getNetworkCachedState( } const stateView = config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz); - return createCachedBeaconStateTest(stateView as BeaconStateAllForks, config) as CachedBeaconStateAllForks; + return createCachedBeaconStateTest(stateView, config); } diff --git a/packages/beacon-state-transition/test/utils/state.ts b/packages/beacon-state-transition/test/utils/state.ts index f4edc6bde94a..0219d04c53f0 100644 --- a/packages/beacon-state-transition/test/utils/state.ts +++ b/packages/beacon-state-transition/test/utils/state.ts @@ -96,7 +96,7 @@ export function generateCachedState( // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - }) as CachedBeaconStateAllForks; + }); } export function createCachedBeaconStateTest( diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 2de9aa8626bc..0813dcf6d99a 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -9,7 +9,6 @@ import { ZERO_HASH, bellatrix, EffectiveBalanceIncrements, - BeaconStateBellatrix, BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; @@ -331,9 +330,9 @@ export class ForkChoice implements IForkChoice { if ( preCachedData?.isMergeTransitionBlock || - (bellatrix.isBellatrixStateType(state as BeaconStateBellatrix) && + (bellatrix.isBellatrixStateType(state) && bellatrix.isBellatrixBlockBodyType(block.body) && - bellatrix.isMergeTransitionBlock(state as BeaconStateBellatrix, block.body)) + bellatrix.isMergeTransitionBlock(state, block.body)) ) assertValidTerminalPowBlock(this.config, (block as unknown) as bellatrix.BeaconBlock, preCachedData); @@ -415,8 +414,8 @@ export class ForkChoice implements IForkChoice { finalizedRoot: toHexString(state.finalizedCheckpoint.root), ...(bellatrix.isBellatrixBlockBodyType(block.body) && - bellatrix.isBellatrixStateType(state as BeaconStateBellatrix) && - bellatrix.isExecutionEnabled(state as BeaconStateBellatrix, block.body) + bellatrix.isBellatrixStateType(state) && + bellatrix.isExecutionEnabled(state, block.body) ? { executionPayloadBlockHash: toHexString(block.body.executionPayload.blockHash), executionStatus: this.getPostMergeExecStatus(preCachedData), diff --git a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts index 4c6747ec16c3..d9666f1fb72f 100644 --- a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts +++ b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts @@ -30,9 +30,7 @@ describe("computeDeltas", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = (generatePerfTestCachedStateAltair({ - goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); diff --git a/packages/lodestar/src/chain/blocks/verifyBlock.ts b/packages/lodestar/src/chain/blocks/verifyBlock.ts index 99ee663a7072..f35c94ff7477 100644 --- a/packages/lodestar/src/chain/blocks/verifyBlock.ts +++ b/packages/lodestar/src/chain/blocks/verifyBlock.ts @@ -161,7 +161,7 @@ export async function verifyBlockStateTransition( if (useBlsBatchVerify && !validSignatures) { const signatureSets = validProposerSignature ? allForks.getAllBlockSignatureSetsExceptProposer(postState, block) - : allForks.getAllBlockSignatureSets(postState as CachedBeaconStateAllForks, block); + : allForks.getAllBlockSignatureSets(postState, block); if (signatureSets.length > 0 && !(await chain.bls.verifySignatureSets(signatureSets))) { throw new BlockError(block, {code: BlockErrorCode.INVALID_SIGNATURE, state: postState}); diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 4666b1af4175..9fae7140a265 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -3,11 +3,7 @@ import {toHexString} from "@chainsafe/ssz"; import {allForks, Epoch, phase0, Slot, ssz, Version} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; import {CheckpointWithHex, IProtoBlock} from "@chainsafe/lodestar-fork-choice"; -import { - BeaconStateAllForks, - CachedBeaconStateAllForks, - computeStartSlotAtEpoch, -} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition"; import {AttestationError, BlockError, BlockErrorCode} from "./errors"; import {ChainEvent, IChainEvents} from "./emitter"; import {BeaconChain} from "./chain"; @@ -148,7 +144,7 @@ export async function onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWit // TODO: Improve using regen here const headState = this.stateCache.get(this.forkChoice.getHead().stateRoot); if (headState) { - this.opPool.pruneAll(headState as BeaconStateAllForks); + this.opPool.pruneAll(headState); } } diff --git a/packages/lodestar/src/chain/factory/block/body.ts b/packages/lodestar/src/chain/factory/block/body.ts index e54bf90fe35a..aa4eb8ec87fb 100644 --- a/packages/lodestar/src/chain/factory/block/body.ts +++ b/packages/lodestar/src/chain/factory/block/body.ts @@ -61,7 +61,7 @@ export async function assembleBody( const [attesterSlashings, proposerSlashings, voluntaryExits] = chain.opPool.getSlashingsAndExits(currentState); const attestations = chain.aggregatedAttestationPool.getAttestationsForBlock(currentState); - const {eth1Data, deposits} = await chain.eth1.getEth1DataAndDeposits(currentState as CachedBeaconStateAllForks); + const {eth1Data, deposits} = await chain.eth1.getEth1DataAndDeposits(currentState); const blockBody: phase0.BeaconBlockBody = { randaoReveal, diff --git a/packages/lodestar/src/chain/initState.ts b/packages/lodestar/src/chain/initState.ts index d5d6c9d1892d..50bdde123d79 100644 --- a/packages/lodestar/src/chain/initState.ts +++ b/packages/lodestar/src/chain/initState.ts @@ -157,7 +157,7 @@ export async function initStateFromDb( stateRoot: toHexString(state.hashTreeRoot()), }); - return state as BeaconStateAllForks; + return state; } /** diff --git a/packages/lodestar/src/chain/regen/regen.ts b/packages/lodestar/src/chain/regen/regen.ts index f0ef414901fa..0543607abf93 100644 --- a/packages/lodestar/src/chain/regen/regen.ts +++ b/packages/lodestar/src/chain/regen/regen.ts @@ -192,7 +192,7 @@ export class StateRegenerator implements IStateRegenerator { } } - return state as CachedBeaconStateAllForks; + return state; } private findFirstStateBlock(stateRoot: RootHex): IProtoBlock { diff --git a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 2735d9c83f4e..01166df9e9b2 100644 --- a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -2,7 +2,6 @@ import {itBench} from "@dapplion/benchmark"; import {expect} from "chai"; import { CachedBeaconStateAltair, - CachedBeaconStateAllForks, computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, @@ -27,9 +26,7 @@ describe("getAttestationsForBlock", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = (generatePerfTestCachedStateAltair({ - goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); @@ -46,7 +43,7 @@ describe("getAttestationsForBlock", () => { beforeEach: () => getAggregatedAttestationPool(originalState), fn: (pool) => { // logger.info("Number of attestations in pool", pool.getAll().length); - pool.getAttestationsForBlock(originalState as CachedBeaconStateAllForks); + pool.getAttestationsForBlock(originalState); }, }); }); diff --git a/packages/lodestar/test/spec/allForks/sanity.ts b/packages/lodestar/test/spec/allForks/sanity.ts index 9e8f2da8c685..b7c689b598c2 100644 --- a/packages/lodestar/test/spec/allForks/sanity.ts +++ b/packages/lodestar/test/spec/allForks/sanity.ts @@ -50,7 +50,7 @@ export function sanityBlock(fork: ForkName, testPath: string): void { `${ACTIVE_PRESET}/${fork}/sanity/blocks`, join(SPEC_TEST_LOCATION, testPath), (testcase) => { - const stateTB = testcase.pre as BeaconStateAllForks; + const stateTB = testcase.pre; let wrappedState = createCachedBeaconStateTest(stateTB, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { diff --git a/packages/lodestar/test/spec/altair/operations.test.ts b/packages/lodestar/test/spec/altair/operations.test.ts index 5c9086aa3a48..a93b4012eb0e 100644 --- a/packages/lodestar/test/spec/altair/operations.test.ts +++ b/packages/lodestar/test/spec/altair/operations.test.ts @@ -1,9 +1,4 @@ -import { - CachedBeaconStateAllForks, - CachedBeaconStateAltair, - allForks, - altair, -} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedBeaconStateAltair, allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; @@ -35,7 +30,7 @@ operations(ForkName.altair, { }, block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { diff --git a/packages/lodestar/test/spec/bellatrix/operations.test.ts b/packages/lodestar/test/spec/bellatrix/operations.test.ts index 18ef54410d61..cc961bf3fe32 100644 --- a/packages/lodestar/test/spec/bellatrix/operations.test.ts +++ b/packages/lodestar/test/spec/bellatrix/operations.test.ts @@ -26,12 +26,12 @@ const sync_aggregate: BlockProcessFn = ( block.slot = state.slot; block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); - altair.processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block); + altair.processSyncAggregate((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, block); }; operations(ForkName.bellatrix, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { - altair.processAttestations((state as unknown) as CachedBeaconStateAltair, [testCase.attestation]); + altair.processAttestations((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, [testCase.attestation]); }, attester_slashing: (state, testCase: IBaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { @@ -39,11 +39,11 @@ operations(ForkName.bellatrix, { }, block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { - altair.processDeposit((state as unknown) as CachedBeaconStateAltair, testCase.deposit); + altair.processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, testCase.deposit); }, proposer_slashing: (state, testCase: IBaseSpecTest & {proposer_slashing: phase0.ProposerSlashing}) => { @@ -54,15 +54,22 @@ operations(ForkName.bellatrix, { sync_aggregate_random: sync_aggregate, voluntary_exit: (state, testCase: IBaseSpecTest & {voluntary_exit: phase0.SignedVoluntaryExit}) => { - altair.processVoluntaryExit((state as unknown) as CachedBeaconStateAltair, testCase.voluntary_exit); + altair.processVoluntaryExit( + (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, + testCase.voluntary_exit + ); }, execution_payload: ( state, testCase: IBaseSpecTest & {execution_payload: bellatrix.ExecutionPayload; execution: {execution_valid: boolean}} ) => { - processExecutionPayload((state as unknown) as CachedBeaconStateBellatrix, testCase.execution_payload, { - notifyNewPayload: () => testCase.execution.execution_valid, - }); + processExecutionPayload( + (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix, + testCase.execution_payload, + { + notifyNewPayload: () => testCase.execution.execution_valid, + } + ); }, }); diff --git a/packages/lodestar/test/spec/phase0/operations.test.ts b/packages/lodestar/test/spec/phase0/operations.test.ts index 3e540df86b24..c1cbe1ff29cd 100644 --- a/packages/lodestar/test/spec/phase0/operations.test.ts +++ b/packages/lodestar/test/spec/phase0/operations.test.ts @@ -1,9 +1,4 @@ -import { - CachedBeaconStateAllForks, - CachedBeaconStatePhase0, - allForks, - phase0, -} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedBeaconStatePhase0, allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; import {operations} from "../allForks/operations"; @@ -21,7 +16,7 @@ operations(ForkName.phase0, { }, block_header: (state, testCase: IBaseSpecTest & {block: phase0.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { diff --git a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts index 9579f32c410e..4bf17f8461df 100644 --- a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts @@ -11,7 +11,7 @@ describe("api / impl / validator / utils", () => { const indexes: ValidatorIndex[] = []; let state: BeaconStateAllForks; before("Prepare state", () => { - state = ssz.phase0.BeaconState.defaultViewDU as BeaconStateAllForks; + state = ssz.phase0.BeaconState.defaultViewDU; const validator = ssz.phase0.Validator.defaultValue; const validators = state.validators; for (let i = 0; i < vc; i++) { diff --git a/packages/lodestar/test/unit/network/attestationService.test.ts b/packages/lodestar/test/unit/network/attestationService.test.ts index d24b3c350b0f..fa38138f217b 100644 --- a/packages/lodestar/test/unit/network/attestationService.test.ts +++ b/packages/lodestar/test/unit/network/attestationService.test.ts @@ -64,7 +64,7 @@ describe("AttnetsService", function () { genesisTime: Math.floor(Date.now() / 1000), chainId: 0, networkId: BigInt(0), - state: state as BeaconStateAllForks, + state, config, }); // load getCurrentSlot first, vscode not able to debug without this diff --git a/packages/lodestar/test/utils/state.ts b/packages/lodestar/test/utils/state.ts index 1f35427045c7..4194729c2451 100644 --- a/packages/lodestar/test/utils/state.ts +++ b/packages/lodestar/test/utils/state.ts @@ -144,7 +144,7 @@ export function generateCachedState( // This is a performance test, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - }) as CachedBeaconStateAllForks; + }); } /** diff --git a/packages/lodestar/test/utils/validationData/attestation.ts b/packages/lodestar/test/utils/validationData/attestation.ts index 45722e8ca9c4..678f789412af 100644 --- a/packages/lodestar/test/utils/validationData/attestation.ts +++ b/packages/lodestar/test/utils/validationData/attestation.ts @@ -1,5 +1,4 @@ import { - CachedBeaconStateAllForks, computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch, @@ -107,7 +106,7 @@ export function getAttestationValidData( // Add state to regen const regen = ({ - getState: async () => (state as unknown) as CachedBeaconStateAllForks, + getState: async () => state, } as Partial) as IStateRegenerator; const chain = ({ From 60605fd35f9a9d00070c2d9ebd797697bd28f6bb Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 18:52:12 +0530 Subject: [PATCH 22/30] Review PR --- .../test/perf/analyzeEpochs.ts | 4 +--- .../beacon-state-transition/test/utils/state.ts | 17 ++++++++--------- .../lodestar/src/chain/factory/block/body.ts | 10 +++++----- packages/lodestar/src/eth1/utils/eth1Vote.ts | 2 +- packages/lodestar/src/executionEngine/mock.ts | 2 +- .../lodestar/src/metrics/metrics/lodestar.ts | 4 ++-- .../lodestar/src/network/peers/peerManager.ts | 4 ++-- packages/lodestar/src/node/nodejs.ts | 2 +- .../test/spec/bellatrix/operations.test.ts | 4 +--- .../test/spec/ssz/generic/index.test.ts | 10 ---------- .../utils/replaceUintTypeWithUintBigintType.ts | 3 --- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../encodingStrategies/sszSnappy/testData.ts | 10 +++++----- packages/lodestar/test/utils/block.ts | 10 +++++----- packages/lodestar/test/utils/state.ts | 10 +++++----- packages/validator/src/validator.ts | 2 +- 16 files changed, 39 insertions(+), 57 deletions(-) diff --git a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts index 0751d922f289..5aaf1f8b3444 100644 --- a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts +++ b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts @@ -171,9 +171,7 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< function countAttBits(atts: phase0.PendingAttestation[]): number { let totalBits = 0; for (const att of atts) { - const indexes = Array.from({length: att.aggregationBits.bitLen}, () => 0); - const yesCount = att.aggregationBits.intersectValues(indexes).length; - totalBits += yesCount; + totalBits += att.aggregationBits.getTrueBitIndexes().length; } return totalBits / atts.length; } diff --git a/packages/beacon-state-transition/test/utils/state.ts b/packages/beacon-state-transition/test/utils/state.ts index 0219d04c53f0..6f631d678868 100644 --- a/packages/beacon-state-transition/test/utils/state.ts +++ b/packages/beacon-state-transition/test/utils/state.ts @@ -1,4 +1,3 @@ -import {BitArray} from "@chainsafe/ssz"; import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; import { EPOCHS_PER_HISTORICAL_VECTOR, @@ -7,7 +6,7 @@ import { GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT, } from "@chainsafe/lodestar-params"; -import {phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/default"; import {ZERO_HASH} from "../../src/constants"; @@ -55,21 +54,21 @@ export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { }, blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: [] as Root[], + historicalRoots: [], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: [] as phase0.Eth1Data[], + eth1DataVotes: [], eth1DepositIndex: 0, - validators: [] as phase0.Validator[], - balances: [] as number[], + validators: [], + balances: [], randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: newZeroedBigIntArray(EPOCHS_PER_SLASHINGS_VECTOR), - previousEpochAttestations: [] as phase0.PendingAttestation[], - currentEpochAttestations: [] as phase0.PendingAttestation[], - justificationBits: BitArray.fromBitLen(4), + previousEpochAttestations: [], + currentEpochAttestations: [], + justificationBits: ssz.phase0.JustificationBits.defaultValue, previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, root: ZERO_HASH, diff --git a/packages/lodestar/src/chain/factory/block/body.ts b/packages/lodestar/src/chain/factory/block/body.ts index aa4eb8ec87fb..d30dc7cf28fa 100644 --- a/packages/lodestar/src/chain/factory/block/body.ts +++ b/packages/lodestar/src/chain/factory/block/body.ts @@ -67,11 +67,11 @@ export async function assembleBody( randaoReveal, graffiti, eth1Data, - proposerSlashings: proposerSlashings, - attesterSlashings: attesterSlashings, - attestations: attestations, - deposits: deposits, - voluntaryExits: voluntaryExits, + proposerSlashings, + attesterSlashings, + attestations, + deposits, + voluntaryExits, }; const blockEpoch = computeEpochAtSlot(blockSlot); diff --git a/packages/lodestar/src/eth1/utils/eth1Vote.ts b/packages/lodestar/src/eth1/utils/eth1Vote.ts index 4fdb1b2993e8..38d663feb286 100644 --- a/packages/lodestar/src/eth1/utils/eth1Vote.ts +++ b/packages/lodestar/src/eth1/utils/eth1Vote.ts @@ -126,7 +126,7 @@ function getEth1DataKey(eth1Data: phase0.Eth1Data): string { } /** - * Returns the array of keys with max value. May return 0, 1 or more keys + * Serialize eth1Data types to a unique string ID. It is only used for comparison. */ export function fastSerializeEth1Data(eth1Data: phase0.Eth1Data): string { return toHex(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHex(eth1Data.depositRoot); diff --git a/packages/lodestar/src/executionEngine/mock.ts b/packages/lodestar/src/executionEngine/mock.ts index ede6b0bbf3f3..c7d5ff8ba390 100644 --- a/packages/lodestar/src/executionEngine/mock.ts +++ b/packages/lodestar/src/executionEngine/mock.ts @@ -132,7 +132,7 @@ export class ExecutionEngineMock implements IExecutionEngine { * 3. Client software MAY stop the corresponding building process after serving this call. */ async getPayload(payloadId: PayloadId): Promise { - const payloadIdNbr = parseInt(payloadId, 10); + const payloadIdNbr = Number(payloadId); const payload = this.preparingPayloads.get(payloadIdNbr); if (!payload) { throw Error(`Unknown payloadId ${payloadId}`); diff --git a/packages/lodestar/src/metrics/metrics/lodestar.ts b/packages/lodestar/src/metrics/metrics/lodestar.ts index d5e4be15526d..12f8b95c9111 100644 --- a/packages/lodestar/src/metrics/metrics/lodestar.ts +++ b/packages/lodestar/src/metrics/metrics/lodestar.ts @@ -1,4 +1,4 @@ -import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks} from "@chainsafe/lodestar-types"; import {RegistryMetricCreator} from "../utils/registryMetricCreator"; import {IMetricsOptions} from "../options"; @@ -11,7 +11,7 @@ export type ILodestarMetrics = ReturnType; export function createLodestarMetrics( register: RegistryMetricCreator, metadata: IMetricsOptions["metadata"], - anchorState?: BeaconStateAllForks + anchorState?: Pick ) { if (metadata) { register.static<"semver" | "branch" | "commit" | "version" | "network">({ diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index 089e8f530248..60b0e1a9f67b 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -1,6 +1,8 @@ import LibP2p, {Connection} from "libp2p"; import PeerId from "peer-id"; import {IDiscv5DiscoveryInputOptions} from "@chainsafe/discv5"; +import {BitArray} from "@chainsafe/ssz"; +import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {allForks, altair, phase0} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; @@ -22,8 +24,6 @@ import { renderIrrelevantPeerType, } from "./utils"; import {SubnetType} from "../metadata"; -import {BitArray} from "@chainsafe/ssz"; -import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; /** heartbeat performs regular updates such as updating reputations and performing discovery requests */ const HEARTBEAT_INTERVAL_MS = 30 * 1000; diff --git a/packages/lodestar/src/node/nodejs.ts b/packages/lodestar/src/node/nodejs.ts index b18ea3d801cf..8535d50a72a6 100644 --- a/packages/lodestar/src/node/nodejs.ts +++ b/packages/lodestar/src/node/nodejs.ts @@ -10,6 +10,7 @@ import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {phase0} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; import {Api} from "@chainsafe/lodestar-api"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconDb} from "../db"; import {INetwork, Network, getReqRespHandlers} from "../network"; @@ -22,7 +23,6 @@ import {initializeExecutionEngine} from "../executionEngine"; import {initializeEth1ForBlockProduction} from "../eth1"; import {IBeaconNodeOptions} from "./options"; import {runNodeNotifier} from "./notifier"; -import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; export * from "./options"; diff --git a/packages/lodestar/test/spec/bellatrix/operations.test.ts b/packages/lodestar/test/spec/bellatrix/operations.test.ts index cc961bf3fe32..03e576a5b57d 100644 --- a/packages/lodestar/test/spec/bellatrix/operations.test.ts +++ b/packages/lodestar/test/spec/bellatrix/operations.test.ts @@ -67,9 +67,7 @@ operations(ForkName.bellatrix, { processExecutionPayload( (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix, testCase.execution_payload, - { - notifyNewPayload: () => testCase.execution.execution_valid, - } + {notifyNewPayload: () => testCase.execution.execution_valid} ); }, }); diff --git a/packages/lodestar/test/spec/ssz/generic/index.test.ts b/packages/lodestar/test/spec/ssz/generic/index.test.ts index 7ce4bd79a753..7fd0755cf011 100644 --- a/packages/lodestar/test/spec/ssz/generic/index.test.ts +++ b/packages/lodestar/test/spec/ssz/generic/index.test.ts @@ -14,11 +14,6 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { describe(`${testType} invalid`, () => { const invalidCasesPath = path.join(testTypePath, "invalid"); for (const invalidCase of fs.readdirSync(invalidCasesPath)) { - const onlyId = process.env.ONLY_ID; - if (onlyId && !invalidCase.includes(onlyId)) { - continue; - } - it(invalidCase, () => { // TODO: Strong type errors and assert that the entire it() throws known errors if (invalidCase.endsWith("_0")) { @@ -53,11 +48,6 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { continue; } - const onlyId = process.env.ONLY_ID; - if (onlyId && !validCase.includes(onlyId)) { - continue; - } - it(validCase, () => { const type = getTestType(testType, validCase); const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); diff --git a/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts index 01a8aaf683cd..57e7f3ac3354 100644 --- a/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts +++ b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts @@ -1,9 +1,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment , @typescript-eslint/no-explicit-any */ -// ####################################### -// # MUST NOT IMPORT FROM @chainsafe/ssz # -// ####################################### import { Type, UintNumberType, diff --git a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 0f04b15a7bdb..9c1e25d60213 100644 --- a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,4 +1,5 @@ import {bls, SecretKey} from "@chainsafe/bls"; +import {BitArray} from "@chainsafe/ssz"; import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {createIChainForkConfig, defaultChainConfig} from "@chainsafe/lodestar-config"; import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; @@ -15,7 +16,6 @@ import {generateAttestation, generateEmptyAttestation} from "../../../utils/atte import {generateCachedState} from "../../../utils/state"; import {renderBitArray} from "../../../utils/render"; import sinon from "sinon"; -import {BitArray} from "@chainsafe/ssz"; describe("AggregatedAttestationPool", function () { let pool: AggregatedAttestationPool; diff --git a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts index b0aa73f339e7..1ec5e32fb909 100644 --- a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts +++ b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts @@ -57,11 +57,11 @@ export const sszSnappySignedBeaconBlockPhase0: ISszSnappyTestData ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: [] as Root[], + historicalRoots: [], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: [] as phase0.Eth1Data[], + eth1DataVotes: [], eth1DepositIndex: 0, validators: validators, balances: Array.from({length: numValidators}, () => MAX_EFFECTIVE_BALANCE), randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: Array.from({length: EPOCHS_PER_SLASHINGS_VECTOR}, () => BigInt(0)), - previousEpochAttestations: [] as phase0.PendingAttestation[], - currentEpochAttestations: [] as phase0.PendingAttestation[], + previousEpochAttestations: [], + currentEpochAttestations: [], justificationBits: BitArray.fromBitLen(4), previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index f25dd572fe5d..116ddbf005fa 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -67,7 +67,7 @@ export class Validator { }) : opts.api; - const clock = new Clock(config, logger, {genesisTime: genesis.genesisTime}); + const clock = new Clock(config, logger, {genesisTime: Number(genesis.genesisTime)}); const validatorStore = new ValidatorStore(config, slashingProtection, signers, genesis); const indicesService = new IndicesService(logger, api, validatorStore); this.emitter = new ValidatorEventEmitter(); From 15cb1ebba761d2bd58c6dd0eebd8005052db2bb3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 21:38:33 +0530 Subject: [PATCH 23/30] Remove unused imports --- packages/beacon-state-transition/src/altair/epoch/index.ts | 2 +- .../src/altair/epoch/processInactivityUpdates.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-state-transition/src/altair/epoch/index.ts b/packages/beacon-state-transition/src/altair/epoch/index.ts index 35249a419c05..1623f86d633c 100644 --- a/packages/beacon-state-transition/src/altair/epoch/index.ts +++ b/packages/beacon-state-transition/src/altair/epoch/index.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import { processJustificationAndFinalization, processRegistryUpdates, diff --git a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts index 3363645444a2..adec4cfdaf6b 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts @@ -1,5 +1,5 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import * as attesterStatusUtil from "../../util/attesterStatus"; import {isInInactivityLeak} from "../../util"; From 83efb0a9464bcdab492032fc9b2d782e2584a6cf Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 24 Mar 2022 12:00:56 -0500 Subject: [PATCH 24/30] Use published versions of ssz --- packages/api/package.json | 4 +-- packages/beacon-state-transition/package.json | 4 +-- packages/cli/package.json | 2 +- packages/config/package.json | 2 +- packages/db/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 4 +-- packages/lodestar/package.json | 4 +-- packages/types/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 28 +++++++++++-------- 11 files changed, 31 insertions(+), 25 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 31dcc8233f63..2467699f8424 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,8 +40,8 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/persistent-merkle-tree": "^0.4.0", + "@chainsafe/ssz": "^0.9.0", "cross-fetch": "^3.1.4", "eventsource": "^1.1.0", "qs": "^6.10.1" diff --git a/packages/beacon-state-transition/package.json b/packages/beacon-state-transition/package.json index b39bbb6111c6..3e4a68c5fc34 100644 --- a/packages/beacon-state-transition/package.json +++ b/packages/beacon-state-transition/package.json @@ -41,9 +41,9 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", + "@chainsafe/persistent-merkle-tree": "^0.4.0", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/ssz": "^0.9.0", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index 80787c577c02..42288f5f843f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -63,7 +63,7 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/ssz": "^0.9.0", "@types/lockfile": "^1.0.1", "bip39": "^3.0.2", "deepmerge": "^4.2.2", diff --git a/packages/config/package.json b/packages/config/package.json index cf28568b79e4..5a3bdfdad2ae 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -44,6 +44,6 @@ "dependencies": { "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" + "@chainsafe/ssz": "^0.9.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index a0ed60e03331..434067cddda1 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -38,7 +38,7 @@ "dependencies": { "@chainsafe/lodestar-config": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/ssz": "^0.9.0", "@types/levelup": "^4.3.3", "it-all": "^1.0.2", "level": "^7.0.0", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 498ff6a72bd2..b5a3ec802591 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -42,7 +42,7 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" + "@chainsafe/ssz": "^0.9.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index bcc2f9e0db18..186badb564de 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -42,8 +42,8 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/persistent-merkle-tree": "^0.4.0", + "@chainsafe/ssz": "^0.9.0", "cross-fetch": "^3.1.4", "mitt": "^3.0.0" }, diff --git a/packages/lodestar/package.json b/packages/lodestar/package.json index efb6a20965cf..f0f95dea5cac 100644 --- a/packages/lodestar/package.json +++ b/packages/lodestar/package.json @@ -74,9 +74,9 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2", + "@chainsafe/persistent-merkle-tree": "^0.4.0", "@chainsafe/snappy-stream": "5.0.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/ssz": "^0.9.0", "@ethersproject/abi": "^5.0.0", "@types/datastore-level": "^3.0.0", "bl": "^5.0.0", diff --git a/packages/types/package.json b/packages/types/package.json index 6d728fbf7d58..0737796ff705 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -36,7 +36,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/lodestar-params": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2" + "@chainsafe/ssz": "^0.9.0" }, "keywords": [ "ethereum", diff --git a/packages/validator/package.json b/packages/validator/package.json index ec81e0737ca0..4461143b4642 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -54,7 +54,7 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2", + "@chainsafe/ssz": "^0.9.0", "bigint-buffer": "^1.1.5", "cross-fetch": "^3.1.4", "strict-event-emitter-types": "^2.0.0" diff --git a/yarn.lock b/yarn.lock index 3e435ff08246..86633482f1da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -385,7 +385,7 @@ dependencies: event-target-shim "^5.0.0" -"@chainsafe/as-sha256@^0.2.3", "@chainsafe/as-sha256@^0.2.4": +"@chainsafe/as-sha256@^0.2.4": version "0.2.4" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.2.4.tgz#17e78d32c71c836160bb55fd79c3daad93d9abf0" integrity sha512-rYfIOaQm0OlFcHdJFUu5VyYOA1HVeQXxOivUsawBjd7WXc3lMQ0bXMfCgN50gPPLWT92G4ioZ0EZz8RnH+YT/g== @@ -393,6 +393,11 @@ "@assemblyscript/loader" "^0.9.2" buffer "^5.4.3" +"@chainsafe/as-sha256@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.0.tgz#ba7d4b85a77c51a50c071856ea6ead4066583b74" + integrity sha512-X0Yaz1H9ByEelnrG2Q76aEmLmlsjZe8905n6Eez8wmV2E1TvBnY2AFYYy2U52dkaGrWrG4MHYKLo4NJcSqkFeg== + "@chainsafe/bls-hd-key@^0.2.0": version "0.2.1" resolved "https://registry.yarnpkg.com/@chainsafe/bls-hd-key/-/bls-hd-key-0.2.1.tgz#3387a63f36f3cac20276bef40bb16a46294e6724" @@ -488,12 +493,12 @@ protobufjs "^6.11.2" uint8arrays "^3.0.0" -"@chainsafe/persistent-merkle-tree@https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2": - version "0.3.7" - uid "089475892f3e5aae15f767c9f360b2e23af65638" - resolved "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2#089475892f3e5aae15f767c9f360b2e23af65638" +"@chainsafe/persistent-merkle-tree@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.4.0.tgz#249abef9623d45889d71f0f7e807d3aa93472d21" + integrity sha512-xf/DfSAAeAnFOIlNG+m0IvaVK3ccc47vSLEPi0IgbOslXWa540Zxrr33PHbByKI4OhKBp+Zi58WHZAe5O/s2nQ== dependencies: - "@chainsafe/as-sha256" "^0.2.3" + "@chainsafe/as-sha256" "^0.3.0" "@chainsafe/persistent-ts@^0.19.1": version "0.19.1" @@ -512,12 +517,13 @@ buffer-from "^1.1.1" snappy "^6.3.5" -"@chainsafe/ssz@https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2": - version "0.8.20" - resolved "https://gitpkg.now.sh/dapplion/ssz/packages/ssz?e01ce05824485c0045fb2d1fb99cd1e898f49af2#dd44c159620efafcd31a0aecfd257f57bdc085d8" +"@chainsafe/ssz@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.9.0.tgz#27e40fa3316ef8736bbae898bdf90c1dce4a7045" + integrity sha512-QTLxxcknbiFrpgK2FplAxqxAyi0AGDNoAyxaUUJGdG0mXU4WWIKKLXtYATAXOHVruTzKZeydHatkrpKyYRdyjw== dependencies: - "@chainsafe/as-sha256" "^0.2.4" - "@chainsafe/persistent-merkle-tree" "https://gitpkg.now.sh/dapplion/ssz/packages/persistent-merkle-tree?dapplion/v2" + "@chainsafe/as-sha256" "^0.3.0" + "@chainsafe/persistent-merkle-tree" "^0.4.0" case "^1.6.3" "@cspotcode/source-map-consumer@0.8.0": From 68bdd0e6c278e65e2b09589787a959b641ac4c2f Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 24 Mar 2022 16:11:58 -0500 Subject: [PATCH 25/30] Fix sha256 usage --- .../src/allForks/block/processRandao.ts | 7 ++----- .../beacon-state-transition/src/util/aggregator.ts | 4 ++-- packages/beacon-state-transition/src/util/seed.ts | 14 +++++++------- .../beacon-state-transition/src/util/shuffle.ts | 12 ++++++------ packages/lodestar/src/network/gossip/gossipsub.ts | 4 ++-- packages/utils/src/verifyMerkleBranch.ts | 8 ++++---- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/beacon-state-transition/src/allForks/block/processRandao.ts b/packages/beacon-state-transition/src/allForks/block/processRandao.ts index 8287b9abcd6c..9696ad53ae0d 100644 --- a/packages/beacon-state-transition/src/allForks/block/processRandao.ts +++ b/packages/beacon-state-transition/src/allForks/block/processRandao.ts @@ -1,5 +1,5 @@ import xor from "buffer-xor"; -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {allForks} from "@chainsafe/lodestar-types"; import {getRandaoMix} from "../../util"; import {verifyRandaoSignature} from "../signatureSets"; @@ -28,9 +28,6 @@ export function processRandao( } // mix in RANDAO reveal - const randaoMix = xor( - Buffer.from(getRandaoMix(state, epoch) as Uint8Array), - Buffer.from(SHA256.digest(randaoReveal)) - ); + const randaoMix = xor(Buffer.from(getRandaoMix(state, epoch) as Uint8Array), Buffer.from(digest(randaoReveal))); state.randaoMixes.set(epoch % EPOCHS_PER_HISTORICAL_VECTOR, randaoMix); } diff --git a/packages/beacon-state-transition/src/util/aggregator.ts b/packages/beacon-state-transition/src/util/aggregator.ts index eef4ad87f1e3..2595ea456789 100644 --- a/packages/beacon-state-transition/src/util/aggregator.ts +++ b/packages/beacon-state-transition/src/util/aggregator.ts @@ -1,4 +1,4 @@ -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {BLSSignature} from "@chainsafe/lodestar-types"; import {intDiv, bytesToBigInt} from "@chainsafe/lodestar-utils"; import { @@ -29,5 +29,5 @@ export function isAggregatorFromCommitteeLength(committeeLength: number, slotSig * Using bytesToInt() may cause isSelectionProofValid() to always return false. */ export function isSelectionProofValid(sig: BLSSignature, modulo: number): boolean { - return bytesToBigInt(SHA256.digest(sig).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; + return bytesToBigInt(digest(sig).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; } diff --git a/packages/beacon-state-transition/src/util/seed.ts b/packages/beacon-state-transition/src/util/seed.ts index dde0d3f2366c..86f30bc293a7 100644 --- a/packages/beacon-state-transition/src/util/seed.ts +++ b/packages/beacon-state-transition/src/util/seed.ts @@ -2,7 +2,7 @@ * @module chain/stateTransition/util */ -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {Epoch, Bytes32, DomainType, ValidatorIndex} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt, intToBytes} from "@chainsafe/lodestar-utils"; import { @@ -37,7 +37,7 @@ export function computeProposers( computeProposerIndex( effectiveBalanceIncrements, shuffling.activeIndices, - SHA256.digest(Buffer.concat([epochSeed, intToBytes(slot, 8)])) + digest(Buffer.concat([epochSeed, intToBytes(slot, 8)])) ) ); } @@ -66,7 +66,7 @@ export function computeProposerIndex( /* eslint-disable-next-line no-constant-condition */ while (true) { const candidateIndex = indices[computeShuffledIndex(i % indices.length, indices.length, seed)]; - const randByte = SHA256.digest( + const randByte = digest( Buffer.concat([ seed, // @@ -113,7 +113,7 @@ export function getNextSyncCommitteeIndices( while (syncCommitteeIndices.length < SYNC_COMMITTEE_SIZE) { const shuffledIndex = computeShuffledIndex(i % activeValidatorCount, activeValidatorCount, seed); const candidateIndex = activeValidatorIndices[shuffledIndex]; - const randByte = SHA256.digest( + const randByte = digest( Buffer.concat([ seed, // @@ -146,11 +146,11 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By const _seed = seed; for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) { const pivot = Number( - bytesToBigInt(SHA256.digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) + bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) ); const flip = (pivot + indexCount - permuted) % indexCount; const position = Math.max(permuted, flip); - const source = SHA256.digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); + const source = digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); const byte = source[Math.floor((position % 256) / 8)]; const bit = (byte >> position % 8) % 2; permuted = bit ? flip : permuted; @@ -171,5 +171,5 @@ export function getRandaoMix(state: BeaconStateAllForks, epoch: Epoch): Bytes32 export function getSeed(state: BeaconStateAllForks, epoch: Epoch, domainType: DomainType): Uint8Array { const mix = getRandaoMix(state, epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1); - return SHA256.digest(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix])); + return digest(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix])); } diff --git a/packages/beacon-state-transition/src/util/shuffle.ts b/packages/beacon-state-transition/src/util/shuffle.ts index c8da6cb94bb9..15462a899f94 100644 --- a/packages/beacon-state-transition/src/util/shuffle.ts +++ b/packages/beacon-state-transition/src/util/shuffle.ts @@ -1,7 +1,7 @@ /** * @module util/objects */ -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {SHUFFLE_ROUND_COUNT} from "@chainsafe/lodestar-params"; import {ValidatorIndex, Bytes32} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt} from "@chainsafe/lodestar-utils"; @@ -111,7 +111,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): buf[_SHUFFLE_H_SEED_SIZE] = r; // Seed is already in place, now just hash the correct part of the buffer, and take a uint64 from it, // and modulo it to get a pivot within range. - const h = SHA256.digest(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); + const h = digest(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); const pivot = Number(bytesToBigInt(h.slice(0, 8)) % BigInt(listSize)) >>> 0; // Split up the for-loop in two: @@ -134,7 +134,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start from the pivot position, and work back to the mirror position (of the part left to the pivot). // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(pivot >> 8, buf); // already using first pivot byte below. - source = SHA256.digest(buf); + source = digest(buf); byteV = source[(pivot & 0xff) >> 3]; for (let i = 0, j; i < mirror; i++) { @@ -145,7 +145,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = SHA256.digest(buf); + source = digest(buf); } // Same trick with byte retrieval. Only every 8th. @@ -171,7 +171,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start at the end, and work back to the mirror point. // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(end >> 8, buf); - source = SHA256.digest(buf); + source = digest(buf); byteV = source[(end & 0xff) >> 3]; for (let i = pivot + 1, j; i < mirror; i++) { j = end - i + pivot + 1; @@ -181,7 +181,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = SHA256.digest(buf); + source = digest(buf); } // Same trick with byte retrieval. Only every 8th. diff --git a/packages/lodestar/src/network/gossip/gossipsub.ts b/packages/lodestar/src/network/gossip/gossipsub.ts index 5bc72f756ea8..192f74de230a 100644 --- a/packages/lodestar/src/network/gossip/gossipsub.ts +++ b/packages/lodestar/src/network/gossip/gossipsub.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import Gossipsub from "libp2p-gossipsub"; import {messageIdToString} from "libp2p-gossipsub/src/utils/messageIdToString"; -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {ERR_TOPIC_VALIDATOR_IGNORE, ERR_TOPIC_VALIDATOR_REJECT} from "libp2p-gossipsub/src/constants"; import {InMessage, utils} from "libp2p-interfaces/src/pubsub"; import Libp2p from "libp2p"; @@ -89,7 +89,7 @@ export class Eth2Gossipsub extends Gossipsub { Dlazy: 6, scoreParams: computeGossipPeerScoreParams(modules), scoreThresholds: gossipScoreThresholds, - fastMsgIdFn: (msg: InMessage) => Buffer.from(SHA256.digest(msg.data)).toString("hex"), + fastMsgIdFn: (msg: InMessage) => Buffer.from(digest(msg.data)).toString("hex"), }); const {config, logger, metrics, signal, gossipHandlers} = modules; this.config = config; diff --git a/packages/utils/src/verifyMerkleBranch.ts b/packages/utils/src/verifyMerkleBranch.ts index 45299b7201f1..e109813dd29e 100644 --- a/packages/utils/src/verifyMerkleBranch.ts +++ b/packages/utils/src/verifyMerkleBranch.ts @@ -1,7 +1,7 @@ -import SHA256 from "@chainsafe/as-sha256"; +import {digest, digest64} from "@chainsafe/as-sha256"; export function hash(...inputs: Uint8Array[]): Uint8Array { - return SHA256.digest(Buffer.concat(inputs)); + return digest(Buffer.concat(inputs)); } /** @@ -18,9 +18,9 @@ export function verifyMerkleBranch( let value = leaf; for (let i = 0; i < depth; i++) { if (Math.floor(index / 2 ** i) % 2) { - value = SHA256.digest64(Buffer.concat([proof[i], value])); + value = digest64(Buffer.concat([proof[i], value])); } else { - value = SHA256.digest64(Buffer.concat([value, proof[i]])); + value = digest64(Buffer.concat([value, proof[i]])); } } return Buffer.from(value).equals(root); From 2a00e80b4fc20b9c890c8a8d8dcef924ddca4789 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 24 Mar 2022 16:18:19 -0500 Subject: [PATCH 26/30] Fix readme examples --- packages/types/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/types/README.md b/packages/types/README.md index 75205af91bf5..870090a8beca 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -45,7 +45,7 @@ import {ssz, Epoch} from "@chainsafe/lodestar-types"; const EpochType: Type = ssz.Epoch; -const e = EpochType.defaultValue; +const e = EpochType.defaultValue(); ``` ### By fork @@ -55,8 +55,8 @@ Lodestar types support multiple different consensus forks. In order to easily di ```typescript import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; -const phase0State: phase0.BeaconState = ssz.phase0.BeaconState.defaultValue; -const altairState: altair.BeaconState = ssz.altair.BeaconState.defaultValue; +const phase0State: phase0.BeaconState = ssz.phase0.BeaconState.defaultValue(); +const altairState: altair.BeaconState = ssz.altair.BeaconState.defaultValue(); ``` Primitive types are directly available without a namespace. @@ -64,7 +64,7 @@ Primitive types are directly available without a namespace. ```typescript import {Epoch, ssz} from "@chainsafe/lodestar-types"; -const epoch: Epoch = ssz.Epoch.defaultValue; +const epoch: Epoch = ssz.Epoch.defaultValue(); ``` In some cases, we need interfaces that accept types across all forks, eg: when the fork is not known ahead of time. Typescript interfaces for this purpose are exported under the `allForks` namespace. SSZ Types typed to these interfaces are also provided under an `allForks` namespace, but keyed by `ForkName`. @@ -73,7 +73,7 @@ In some cases, we need interfaces that accept types across all forks, eg: when t import {ForkName} from "@chainsafe/lodestar-params"; import {allForks, ssz} from "@chainsafe/lodestar-types"; -const state: allForks.BeaconState = ssz.allForks[ForkName.phase0].BeaconState.defaultValue; +const state: allForks.BeaconState = ssz.allForks[ForkName.phase0].BeaconState.defaultValue(); ``` ## License From 37e1f9c038bfd44d48353a4d8903af0372ba8096 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 24 Mar 2022 16:31:50 -0500 Subject: [PATCH 27/30] Simplify toViewDU calls --- .../src/allForks/epoch/processEth1DataReset.ts | 2 +- .../beacon-state-transition/src/bellatrix/upgradeState.ts | 4 +--- .../src/phase0/epoch/processParticipationRecordUpdates.ts | 2 +- packages/lodestar/src/chain/genesis/genesis.ts | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts index a3aac340e11d..3998e3aef0e1 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts @@ -12,6 +12,6 @@ export function processEth1DataReset(state: CachedBeaconStateAllForks, epochProc // reset eth1 data votes if (nextEpoch % EPOCHS_PER_ETH1_VOTING_PERIOD === 0) { - state.eth1DataVotes = ssz.phase0.Eth1DataVotes.toViewDU([]); + state.eth1DataVotes = ssz.phase0.Eth1DataVotes.defaultViewDU(); } } diff --git a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts index 21238eac3737..e1703ef3e1e9 100644 --- a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts +++ b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts @@ -53,9 +53,7 @@ export function upgradeState(stateAltair: CachedBeaconStateAltair): CachedBeacon }); // Execution-layer - stateBellatrix.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU( - ssz.bellatrix.ExecutionPayloadHeader.defaultValue() - ); + stateBellatrix.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU(); // Commit new added fields ViewDU to the root node stateBellatrix.commit(); diff --git a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts index 9023488a5384..1a6d3569bb7f 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts @@ -10,5 +10,5 @@ export function processParticipationRecordUpdates(state: CachedBeaconStatePhase0 state.previousEpochAttestations = state.currentEpochAttestations; // Reset list to empty - state.currentEpochAttestations = ssz.phase0.EpochAttestations.toViewDU([]); + state.currentEpochAttestations = ssz.phase0.EpochAttestations.defaultViewDU(); } diff --git a/packages/lodestar/src/chain/genesis/genesis.ts b/packages/lodestar/src/chain/genesis/genesis.ts index 13163473de9f..a495f4f519f1 100644 --- a/packages/lodestar/src/chain/genesis/genesis.ts +++ b/packages/lodestar/src/chain/genesis/genesis.ts @@ -86,7 +86,7 @@ export class GenesisBuilder implements IGenesisBuilder { ssz.phase0.Eth1Data.defaultValue(), getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) ); - this.depositTree = ssz.phase0.DepositDataRootList.toViewDU(ssz.phase0.DepositDataRootList.defaultValue()); + this.depositTree = ssz.phase0.DepositDataRootList.defaultViewDU(); this.fromBlock = this.eth1Provider.deployBlock; } From ab8bd7e0f0881478409a6be7f2d7407c1a10efd5 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 4 Apr 2022 11:02:14 -0500 Subject: [PATCH 28/30] Fix merge issues --- packages/api/src/keymanager/routes.ts | 6 ++-- .../lodestar/src/network/peers/peerManager.ts | 3 +- yarn.lock | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/api/src/keymanager/routes.ts b/packages/api/src/keymanager/routes.ts index 55288fb5ffc7..a278145c4360 100644 --- a/packages/api/src/keymanager/routes.ts +++ b/packages/api/src/keymanager/routes.ts @@ -154,8 +154,8 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { return { - listKeys: jsonType(), - importKeystores: jsonType(), - deleteKeystores: jsonType(), + listKeys: jsonType("camel"), + importKeystores: jsonType("camel"), + deleteKeystores: jsonType("camel"), }; } diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index aedb0db41ee5..cbe80c6f29f8 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -25,7 +25,6 @@ import { renderIrrelevantPeerType, } from "./utils"; import {SubnetType} from "../metadata"; -import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; /** heartbeat performs regular updates such as updating reputations and performing discovery requests */ const HEARTBEAT_INTERVAL_MS = 30 * 1000; @@ -637,7 +636,7 @@ function countAttnets(peerData?: PeerData): number { let count = 0; for (let i = 0; i < ATTESTATION_SUBNET_COUNT; i++) { - if (attNets[i]) count++; + if (attNets.get(i)) count++; } return count; diff --git a/yarn.lock b/yarn.lock index fd67721761ac..dfbcb48e63c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -59,6 +59,11 @@ resolved "https://registry.yarnpkg.com/@actions/io/-/io-1.1.1.tgz#4a157406309e212ab27ed3ae30e8c1d641686a66" integrity sha512-Qi4JoKXjmE0O67wAOH6y0n26QXhMKMFo7GD/4IXNVcrtLjUlGjGuVys6pQgwF3ArfGTQu0XpqaNr0YhED2RaRA== +"@assemblyscript/loader@^0.9.2": + version "0.9.4" + resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.9.4.tgz#a483c54c1253656bb33babd464e3154a173e1577" + integrity sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA== + "@azure/abort-controller@^1.0.0": version "1.0.4" resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.4.tgz#fd3c4d46c8ed67aace42498c8e2270960250eafd" @@ -380,6 +385,14 @@ dependencies: event-target-shim "^5.0.0" +"@chainsafe/as-sha256@^0.2.3", "@chainsafe/as-sha256@^0.2.4": + version "0.2.4" + resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.2.4.tgz#17e78d32c71c836160bb55fd79c3daad93d9abf0" + integrity sha512-rYfIOaQm0OlFcHdJFUu5VyYOA1HVeQXxOivUsawBjd7WXc3lMQ0bXMfCgN50gPPLWT92G4ioZ0EZz8RnH+YT/g== + dependencies: + "@assemblyscript/loader" "^0.9.2" + buffer "^5.4.3" + "@chainsafe/as-sha256@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.0.tgz#ba7d4b85a77c51a50c071856ea6ead4066583b74" @@ -481,6 +494,13 @@ protobufjs "^6.11.2" uint8arrays "^3.0.0" +"@chainsafe/persistent-merkle-tree@^0.3.7": + version "0.3.7" + resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.3.7.tgz#d257674fcacf1770e54df1e7e476bb55efcb3ed4" + integrity sha512-UXXzvCN8P4eQWCsaTaHRrNa+2sTwY3NB0Vf+uWgM4cU1qf8LOsTU7aU2CeXFAGJag5W/xEQfVGNh463kXTYAgg== + dependencies: + "@chainsafe/as-sha256" "^0.2.3" + "@chainsafe/persistent-merkle-tree@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.4.0.tgz#249abef9623d45889d71f0f7e807d3aa93472d21" @@ -505,6 +525,15 @@ buffer-from "^1.1.1" snappy "^6.3.5" +"@chainsafe/ssz@^0.8.20": + version "0.8.20" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.8.20.tgz#2c8e70e173a2540c8e4e0bad7c44fd836b8b256b" + integrity sha512-SjD/GqkByWbC5JC2W4cvaFjok6kfZeZDLLzr9k0Kn4sIOZPBfuRTumvV6ihqgn7ItOFpZRXMz0u4DgH+lr28WQ== + dependencies: + "@chainsafe/as-sha256" "^0.2.4" + "@chainsafe/persistent-merkle-tree" "^0.3.7" + case "^1.6.3" + "@chainsafe/ssz@^0.9.0": version "0.9.0" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.9.0.tgz#27e40fa3316ef8736bbae898bdf90c1dce4a7045" From c83991f7b5ae52760f14860f12474995774b5391 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 4 Apr 2022 11:13:18 -0500 Subject: [PATCH 29/30] Fix type error --- packages/lodestar/test/e2e/keymanager/keymanager.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lodestar/test/e2e/keymanager/keymanager.test.ts b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts index fa976e1fa701..827aa8dd385b 100644 --- a/packages/lodestar/test/e2e/keymanager/keymanager.test.ts +++ b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts @@ -9,7 +9,7 @@ import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default" import {HttpClient} from "@chainsafe/lodestar-api/src"; import {getClient} from "@chainsafe/lodestar-api/src/keymanager/client"; import {ISlashingProtection, Validator} from "@chainsafe/lodestar-validator"; -import {ByteVector, fromHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {WinstonLogger} from "@chainsafe/lodestar-utils"; import {ssz} from "@chainsafe/lodestar-types"; import {LogLevel, testLogger, TestLoggerOpts} from "../../utils/logger"; @@ -483,7 +483,7 @@ function dirContainFileWithPubkeyInFilename(dir: string, pubkeys: string[]): boo } // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -function createAttesterDuty(pubkey: ByteVector, slot: number, committeeIndex: number, validatorIndex: number) { +function createAttesterDuty(pubkey: Uint8Array, slot: number, committeeIndex: number, validatorIndex: number) { return { slot: slot, committeeIndex: committeeIndex, From cb36a548ec1b917e12db798e933abad67d811d55 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 4 Apr 2022 11:31:08 -0500 Subject: [PATCH 30/30] Update ssz in keymanager server --- packages/keymanager-server/package.json | 2 +- yarn.lock | 29 ------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/packages/keymanager-server/package.json b/packages/keymanager-server/package.json index 741a30e50f15..e63cd8842aae 100644 --- a/packages/keymanager-server/package.json +++ b/packages/keymanager-server/package.json @@ -52,7 +52,7 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "lockfile": "^1.0.4", "fastify": "3.15.1", "fastify-cors": "^6.0.1", diff --git a/yarn.lock b/yarn.lock index dfbcb48e63c8..fd67721761ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -59,11 +59,6 @@ resolved "https://registry.yarnpkg.com/@actions/io/-/io-1.1.1.tgz#4a157406309e212ab27ed3ae30e8c1d641686a66" integrity sha512-Qi4JoKXjmE0O67wAOH6y0n26QXhMKMFo7GD/4IXNVcrtLjUlGjGuVys6pQgwF3ArfGTQu0XpqaNr0YhED2RaRA== -"@assemblyscript/loader@^0.9.2": - version "0.9.4" - resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.9.4.tgz#a483c54c1253656bb33babd464e3154a173e1577" - integrity sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA== - "@azure/abort-controller@^1.0.0": version "1.0.4" resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.4.tgz#fd3c4d46c8ed67aace42498c8e2270960250eafd" @@ -385,14 +380,6 @@ dependencies: event-target-shim "^5.0.0" -"@chainsafe/as-sha256@^0.2.3", "@chainsafe/as-sha256@^0.2.4": - version "0.2.4" - resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.2.4.tgz#17e78d32c71c836160bb55fd79c3daad93d9abf0" - integrity sha512-rYfIOaQm0OlFcHdJFUu5VyYOA1HVeQXxOivUsawBjd7WXc3lMQ0bXMfCgN50gPPLWT92G4ioZ0EZz8RnH+YT/g== - dependencies: - "@assemblyscript/loader" "^0.9.2" - buffer "^5.4.3" - "@chainsafe/as-sha256@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.0.tgz#ba7d4b85a77c51a50c071856ea6ead4066583b74" @@ -494,13 +481,6 @@ protobufjs "^6.11.2" uint8arrays "^3.0.0" -"@chainsafe/persistent-merkle-tree@^0.3.7": - version "0.3.7" - resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.3.7.tgz#d257674fcacf1770e54df1e7e476bb55efcb3ed4" - integrity sha512-UXXzvCN8P4eQWCsaTaHRrNa+2sTwY3NB0Vf+uWgM4cU1qf8LOsTU7aU2CeXFAGJag5W/xEQfVGNh463kXTYAgg== - dependencies: - "@chainsafe/as-sha256" "^0.2.3" - "@chainsafe/persistent-merkle-tree@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.4.0.tgz#249abef9623d45889d71f0f7e807d3aa93472d21" @@ -525,15 +505,6 @@ buffer-from "^1.1.1" snappy "^6.3.5" -"@chainsafe/ssz@^0.8.20": - version "0.8.20" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.8.20.tgz#2c8e70e173a2540c8e4e0bad7c44fd836b8b256b" - integrity sha512-SjD/GqkByWbC5JC2W4cvaFjok6kfZeZDLLzr9k0Kn4sIOZPBfuRTumvV6ihqgn7ItOFpZRXMz0u4DgH+lr28WQ== - dependencies: - "@chainsafe/as-sha256" "^0.2.4" - "@chainsafe/persistent-merkle-tree" "^0.3.7" - case "^1.6.3" - "@chainsafe/ssz@^0.9.0": version "0.9.0" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.9.0.tgz#27e40fa3316ef8736bbae898bdf90c1dce4a7045"