Merge branch 'dev' into release-instructions

.circleci/config.yml

@@ -35,14 +35,14 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v30-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v32-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v30-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v32-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}
           venv_path: ./venv
 jobs:
   checkout_specs:

Makefile

@@ -16,7 +16,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
 
 # A list of fake targets.
 .PHONY: \
-	check_toc     \
 	clean         \
 	coverage      \
 	detect_errors \
@@ -39,7 +38,6 @@ NORM = $(shell tput sgr0)
 
 # Print target descriptions.
 help:
-	@echo "make $(BOLD)check_toc$(NORM)     -- check table of contents"
 	@echo "make $(BOLD)clean$(NORM)         -- delete all untracked files"
 	@echo "make $(BOLD)coverage$(NORM)      -- run pyspec tests with coverage"
 	@echo "make $(BOLD)detect_errors$(NORM) -- detect generator errors"
@@ -63,10 +61,9 @@ PIP_VENV = $(VENV)/bin/pip3
 CODESPELL_VENV = $(VENV)/bin/codespell
 
 # Make a virtual environment will all of the necessary dependencies.
-$(VENV): requirements_preinstallation.txt
+$(VENV): pyproject.toml
 	@echo "Creating virtual environment"
 	@python3 -m venv $(VENV)
-	@$(PIP_VENV) install -r requirements_preinstallation.txt
 
 ###############################################################################
 # Specification
@@ -85,7 +82,7 @@ $(ETH2SPEC): setup.py | $(VENV)
 
 # Force rebuild/install the eth2spec package.
 eth2spec:
-	$(MAKE) --always-make $(ETH2SPEC)
+	@$(MAKE) --always-make $(ETH2SPEC)
 
 # Create the pyspec for all phases.
 pyspec: $(VENV) setup.py
@@ -99,6 +96,8 @@ pyspec: $(VENV) setup.py
 TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 
 # Run pyspec tests.
+# Note: for debugging output to show, print to stderr.
+#
 # To run a specific test, append k=<test>, eg:
 #   make test k=test_verify_kzg_proof
 # To run tests for a specific fork, append fork=<fork>, eg:
@@ -117,6 +116,7 @@ test: $(ETH2SPEC) pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
+		--capture=no \
 		$(MAYBE_TEST) \
 		$(MAYBE_FORK) \
 		$(PRESET) \
@@ -193,10 +193,6 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-# Check all files and error if any ToC were modified.
-check_toc: $(MARKDOWN_FILES:=.toc)
-	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
-
 # Generate ToC sections & save copy of original if modified.
 %.toc:
 	@cp $* $*.tmp; \
@@ -209,8 +205,12 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 		echo "\033[1;34m See $*.tmp\033[0m"; \
 	fi
 
+# Check all files and error if any ToC were modified.
+_check_toc: $(MARKDOWN_FILES:=.toc)
+	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
+
 # Check for mistakes.
-lint: $(ETH2SPEC) pyspec check_toc
+lint: $(ETH2SPEC) pyspec _check_toc
 	@$(CODESPELL_VENV) . --skip "./.git,$(VENV),$(PYSPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(PYSPEC_DIR)/eth2spec
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(TEST_GENERATORS_DIR)
@@ -235,17 +235,19 @@ gen_list:
 	done
 
 # Run one generator.
+# This will forcibly rebuild eth2spec just in case.
 # To check modules for a generator, append modcheck=true, eg:
 #   make gen_genesis modcheck=true
 gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
-gen_%: $(ETH2SPEC) pyspec
+gen_%: eth2spec
 	@mkdir -p $(TEST_VECTOR_DIR)
 	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
 		--output $(TEST_VECTOR_DIR) \
 		$(MAYBE_MODCHECK)
 
 # Run all generators then check for errors.
-gen_all: $(GENERATOR_TARGETS) detect_errors
+gen_all: $(GENERATOR_TARGETS)
+	@$(MAKE) detect_errors
 
 # Detect errors in generators.
 detect_errors: $(TEST_VECTOR_DIR)

README.md

@@ -81,7 +81,6 @@ Documentation on the different components used during spec writing can be found
 
 Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).
 
-
 ## Installation and Usage
 The consensus-specs repo can be used by running the tests locally or inside a docker container.
 

configs/mainnet.yaml

@@ -115,15 +115,13 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa
 # Networking
 # ---------------------------------------------------------------
 # `10 * 2**20` (= 10485760, 10 MiB)
-GOSSIP_MAX_SIZE: 10485760
+MAX_PAYLOAD_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024
-# `10 * 2**20` (=10485760, 10 MiB)
-MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -173,8 +171,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 # `Epoch(2**8)`

configs/minimal.yaml

@@ -116,15 +116,13 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
 # Networking
 # ---------------------------------------------------------------
 # `10 * 2**20` (= 10485760, 10 MiB)
-GOSSIP_MAX_SIZE: 10485760
+MAX_PAYLOAD_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
-# `10 * 2**20` (=10485760, 10 MiB)
-MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -174,8 +172,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4

docker/README.md

@@ -1,18 +1,20 @@
 ## Docker related information
 
 This dockerfile sets up the dependencies required to run consensus-spec tests. The docker image can be locally built with:
-- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
+- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
 Handy commands:
+
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
 - `docker run $IMAGE_NAME make test` will run the make test command inside the docker container
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
-The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
+The `scripts/build_run_docker_tests.sh` script will cover most use cases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
+
 - `./build_run_docker_tests.sh --p mainnet` will run the mainnet preset tests
 - `./build_run_docker_tests.sh --a` will run all the tests across all the forks
 - `./build_run_docker_tests.sh --f deneb` will only run deneb tests

docs/docs/new-feature.md

@@ -1,8 +1,9 @@
 # How to add a new feature proposal in consensus-specs
 
+## Table of contents
+
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-## Table of Contents
 
 - [A. Make it executable for linter checks](#a-make-it-executable-for-linter-checks)
   - [1. Create a folder under `./specs/_features`](#1-create-a-folder-under-specs_features)
@@ -23,7 +24,6 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
-
 ## A. Make it executable for linter checks
 
 ### 1. Create a folder under `./specs/_features`
@@ -35,6 +35,7 @@ For example, if it's an `EIP-9999` CL spec, you can create a `./specs/_features/
 For example, if the latest fork is Capella, use `./specs/capella` content as your "previous fork".
 
 ### 3. Write down your proposed `beacon-chain.md` change
+
 - You can either use [Beacon Chain Spec Template](./templates/beacon-chain-template.md), or make a copy of the latest fork content and then edit it.
 - Tips:
     - We use [`doctoc`](https://www.npmjs.com/package/doctoc) tool to generate the table of content.
@@ -50,8 +51,11 @@ For example, if the latest fork is Capella, use `./specs/capella` content as you
         - Use simple Python rather than the fancy Python dark magic.
 
 ### 4. Add `fork.md`
+
 You can refer to the previous fork's `fork.md` file.
+
 ### 5. Make it executable
+
 - Update Pyspec [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name.
 - Update helpers for [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py) for building the spec:
     - Update [`pysetup/constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/constants.py) with the new feature name as Pyspec `constants.py` defined.
@@ -63,17 +67,21 @@ You can refer to the previous fork's `fork.md` file.
 ## B: Make it executable for pytest and test generator
 
 ### 1. [Optional] Add `light-client/*` docs if you updated the content of `BeaconBlock`
+
 - You can refer to the previous fork's `light-client/*` file.
 - Add the path of the new markdown files in [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py)'s `get_md_doc_paths` function.
 
 ### 2. Add the mainnet and minimal presets and update the configs
+
 - Add presets: `presets/mainnet/<new-feature-name>.yaml` and `presets/minimal/<new-feature-name>.yaml`
 - Update configs: `configs/mainnet.yaml` and `configs/minimal.yaml`
 
 ### 3. Update [`context.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/context.py)
+
 - [Optional] Add `with_<new-feature-name>_and_later` decorator for writing pytest cases. e.g., `with_capella_and_later`.
 
 ### 4. Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py)
+
 - Add `<NEW_FEATURE>` to `ALL_PHASES` and `TESTGEN_FORKS`
 
 ### 5. Update [`genesis.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/genesis.py):
@@ -94,6 +102,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold):
 - If the given feature changes `ExecutionPayload` fields, you have to set the initial values by updating `get_sample_genesis_execution_payload_header` helper.
 
 ### 6. Update CI configurations
+
 - Update [GitHub Actions config](https://github.com/ethereum/consensus-specs/blob/dev/.github/workflows/run-tests.yml)
     - Update `pyspec-tests.strategy.matrix.version` list by adding new feature to it
 - Update [CircleCI config](https://github.com/ethereum/consensus-specs/blob/dev/.circleci/config.yml)
@@ -102,7 +111,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold):
 ## Others
 
 ### Bonus
+
 - Add `validator.md` if honest validator behavior changes with the new feature.
 
 ### Need help?
+
 You can tag spec elves for cleaning up your PR. 🧚

docs/docs/templates/beacon-chain-template.md

@@ -3,15 +3,14 @@
 # <FORK_NAME> -- The Beacon Chain
 
 ## Table of contents
+
 <!-- TOC -->
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
 
-
-
 ## Introduction
 
 ## Notation
@@ -28,7 +27,6 @@
 
 ## Preset
 
-
 ### [CATEGORY OF PRESETS]
 
 | Name | Value |
@@ -64,5 +62,4 @@ class CONTAINER_NAME(Container):
 
 ### Epoch processing
 
-
 ### Block processing

fork_choice/safe-block.md

@@ -1,6 +1,7 @@
 # Fork Choice -- Safe Block
 
 ## Table of contents
+
 <!-- TOC -->
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->

pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = [
+  "marko==1.0.2",
+  "ruamel.yaml==0.17.21",
+  "setuptools==75.8.0",
+  "wheel==0.45.1",
+]
+
+[project]
+name = "eth2spec"
+dynamic = ["version"]
+authors = [{ name = "ethereum" }]
+description = "Ethereum consensus layer specifications package"
+readme = { file = "README.md", content-type = "text/markdown" }
+requires-python = ">=3.9,<4.0"
+dependencies = [
+  "curdleproofs==0.1.2",
+  "eth-typing==3.5.2",
+  "eth-utils==2.3.2",
+  "lru-dict==1.2.0",
+  "marko==1.0.2",
+  "milagro_bls_binding==1.9.0",
+  "py_arkworks_bls12381==0.3.8",
+  "py_ecc==6.0.0",
+  "pycryptodome==3.21.0",
+  "remerkleable==0.1.28",
+  "ruamel.yaml==0.17.21",
+  "setuptools==75.8.0",
+  "trie==3.0.1",
+]
+
+[project.optional-dependencies]
+test = [
+  "pytest-cov==6.0.0",
+  "pytest-xdist==3.6.1",
+  "pytest==8.3.4",
+]
+lint = [
+  "codespell==2.4.0",
+  "flake8==5.0.4",
+  "mypy==0.981",
+  "pylint==3.3.1",
+]
+generator = [
+  "filelock==3.17.0",
+  "pathos==0.3.0",
+  "pytest==8.3.4",
+  "python-snappy==0.7.3",
+]
+docs = [
+  "mdx-truly-sane-lists==1.3",
+  "mkdocs-awesome-pages-plugin==2.8.0",
+  "mkdocs-material==9.1.5",
+  "mkdocs==1.4.2",
+]