diff --git a/Justfile b/Justfile index 159aa3ce..c6339655 100644 --- a/Justfile +++ b/Justfile @@ -102,23 +102,25 @@ dora: fi # manually send a preconfirmation to the bolt devnet -send-preconf: +send-preconf count='1': cd bolt-spammer && RUST_LOG=info cargo run -- \ --provider-url $(kurtosis port print bolt-devnet el-1-geth-lighthouse rpc) \ --beacon-client-url $(kurtosis port print bolt-devnet cl-1-lighthouse-geth http) \ --bolt-sidecar-url http://$(kurtosis port print bolt-devnet mev-sidecar-api api) \ --private-key 53321db7c1e331d93a11a41d16f004d7ff63972ec8ec7c25db329728ceeb1710 \ - --slot head + --slot head \ + --count {{count}} # manually send a blob preconfirmation to the bolt devnet -send-blob-preconf: +send-blob-preconf count='1': cd bolt-spammer && RUST_LOG=info cargo run -- \ --provider-url $(kurtosis port print bolt-devnet el-1-geth-lighthouse rpc) \ --beacon-client-url $(kurtosis port print bolt-devnet cl-1-lighthouse-geth http) \ --bolt-sidecar-url http://$(kurtosis port print bolt-devnet mev-sidecar-api api) \ --private-key 53321db7c1e331d93a11a41d16f004d7ff63972ec8ec7c25db329728ceeb1710 \ --slot head \ - --blob + --blob \ + --count {{count}} \ # build all the docker images locally build-images: diff --git a/bolt-sidecar/src/config/mod.rs b/bolt-sidecar/src/config/mod.rs index 06b409bb..bbf93e13 100644 --- a/bolt-sidecar/src/config/mod.rs +++ b/bolt-sidecar/src/config/mod.rs @@ -146,7 +146,7 @@ pub struct Limits { impl Default for Limits { fn default() -> Self { Self { - max_commitments_per_slot: NonZero::new(6).expect("Valid non-zero"), + max_commitments_per_slot: NonZero::new(128).expect("Valid non-zero"), max_committed_gas_per_slot: NonZero::new(10_000_000).expect("Valid non-zero"), } } diff --git a/bolt-spammer/src/main.rs b/bolt-spammer/src/main.rs index 779ec598..5f1c14c5 100644 --- a/bolt-spammer/src/main.rs +++ b/bolt-spammer/src/main.rs @@ -33,6 +33,8 @@ struct Opts { blob: bool, #[clap(short = 's', long, default_value = "head")] slot: String, + #[clap(short = 'C', long, default_value_t = 1)] + count: u64, } #[tokio::main] @@ -53,45 +55,47 @@ async fn main() -> Result<()> { let current_slot = current_slot(&beacon_api_client).await?; let target_slot = if opts.slot == "head" { current_slot + 2 } else { opts.slot.parse()? }; - let mut tx = if opts.blob { generate_random_blob_tx() } else { generate_random_tx() }; - tx.set_from(sender); - tx.set_nonce(provider.get_transaction_count(sender).await?); - - let tx_signed = tx.build(&transaction_signer).await?; - let tx_hash = tx_signed.tx_hash().to_string(); - let tx_rlp = hex::encode(tx_signed.encoded_2718()); - - let message_digest = { - let mut data = Vec::new(); - data.extend_from_slice(&target_slot.to_le_bytes()); - data.extend_from_slice(hex::decode(tx_hash.trim_start_matches("0x"))?.as_slice()); - keccak256(data) - }; - - let signature = wallet.sign_hash(&message_digest).await?; - let signature = hex::encode(signature.as_bytes()); - - let request = prepare_rpc_request( - "bolt_inclusionPreconfirmation", - vec![serde_json::json!({ - "slot": target_slot, - "tx": tx_rlp, - "signature": signature, - })], - ); - - info!("Transaction hash: {}", tx_hash); - info!("body: {}", serde_json::to_string(&request)?); - - let client = reqwest::Client::new(); - let response = client - .post(&opts.bolt_sidecar_url) - .header("content-type", "application/json") - .body(serde_json::to_string(&request)?) - .send() - .await?; - - info!("Response: {:?}", response.text().await?); + for i in 0..opts.count { + let mut tx = if opts.blob { generate_random_blob_tx() } else { generate_random_tx() }; + tx.set_from(sender); + tx.set_nonce(provider.get_transaction_count(sender).await? + i); + + let tx_signed = tx.build(&transaction_signer).await?; + let tx_hash = tx_signed.tx_hash().to_string(); + let tx_rlp = hex::encode(tx_signed.encoded_2718()); + + let message_digest = { + let mut data = Vec::new(); + data.extend_from_slice(&target_slot.to_le_bytes()); + data.extend_from_slice(hex::decode(tx_hash.trim_start_matches("0x"))?.as_slice()); + keccak256(data) + }; + + let signature = wallet.sign_hash(&message_digest).await?; + let signature = hex::encode(signature.as_bytes()); + + let request = prepare_rpc_request( + "bolt_inclusionPreconfirmation", + vec![serde_json::json!({ + "slot": target_slot, + "tx": tx_rlp, + "signature": signature, + })], + ); + + info!("Transaction hash: {}", tx_hash); + info!("body: {}", serde_json::to_string(&request)?); + + let client = reqwest::Client::new(); + let response = client + .post(&opts.bolt_sidecar_url) + .header("content-type", "application/json") + .body(serde_json::to_string(&request)?) + .send() + .await?; + + info!("Response: {:?}", response.text().await?); + } Ok(()) } diff --git a/bolt-spammer/src/utils.rs b/bolt-spammer/src/utils.rs index 386fdfbb..89edf80b 100644 --- a/bolt-spammer/src/utils.rs +++ b/bolt-spammer/src/utils.rs @@ -19,7 +19,7 @@ pub fn generate_random_tx() -> TransactionRequest { .with_to(Address::from_str(DEAD_ADDRESS).unwrap()) .with_chain_id(KURTOSIS_CHAIN_ID) .with_value(U256::from(thread_rng().gen_range(1..100))) - .with_gas_limit(1_000_000u128) + .with_gas_limit(21_000u128) .with_gas_price(NOICE_GAS_PRICE) } @@ -38,7 +38,7 @@ pub fn generate_random_blob_tx() -> TransactionRequest { .with_max_fee_per_blob_gas(100u128) .max_fee_per_gas(NOICE_GAS_PRICE) .max_priority_fee_per_gas(NOICE_GAS_PRICE / 10) - .with_gas_limit(1_000_000u128) + .with_gas_limit(42_000u128) .with_blob_sidecar(sidecar) .with_input(random_bytes) } diff --git a/builder/Dockerfile b/builder/Dockerfile index 522d5360..c808c9d9 100644 --- a/builder/Dockerfile +++ b/builder/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.21-alpine AS builder +FROM golang:1.22-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/builder/Dockerfile.alltools b/builder/Dockerfile.alltools index 4800421c..ddffb8ee 100644 --- a/builder/Dockerfile.alltools +++ b/builder/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.21-alpine AS builder +FROM golang:1.22-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/builder/builder/utils.go b/builder/builder/utils.go index 753e2177..59bd040d 100644 --- a/builder/builder/utils.go +++ b/builder/builder/utils.go @@ -156,8 +156,11 @@ func EmitBoltDemoEvent(message string) { func CalculateMerkleMultiProofs( payloadTransactions types.Transactions, - constraints types.HashToConstraintDecoded, + HashToConstraintDecoded types.HashToConstraintDecoded, ) (inclusionProof *common.InclusionProof, rootNode *ssz.Node, err error) { + constraintsOrderedByIndex, constraintsWithoutIndex, _, _ := types.ParseConstraintsDecoded(HashToConstraintDecoded) + constraints := slices.Concat(constraintsOrderedByIndex, constraintsWithoutIndex) + // BOLT: generate merkle tree from payload transactions (we need raw RLP bytes for this) rawTxs := make([]bellatrix.Transaction, len(payloadTransactions)) for i, tx := range payloadTransactions { @@ -185,21 +188,20 @@ func CalculateMerkleMultiProofs( baseGeneralizedIndex := int(math.Pow(float64(2), float64(21))) generalizedIndexes := make([]int, len(constraints)) transactionHashes := make([]common.Hash, len(constraints)) - i := 0 - for hash := range constraints { + for i, constraint := range constraints { + tx := constraint.Tx // get the index of the preconfirmed transaction in the block - preconfIndex := slices.IndexFunc(payloadTransactions, func(tx *types.Transaction) bool { return tx.Hash() == hash }) + preconfIndex := slices.IndexFunc(payloadTransactions, func(payloadTx *types.Transaction) bool { return payloadTx.Hash() == tx.Hash() }) if preconfIndex == -1 { - log.Error(fmt.Sprintf("Preconfirmed transaction %s not found in block", hash)) + log.Error(fmt.Sprintf("Preconfirmed transaction %s not found in block", tx.Hash())) log.Error(fmt.Sprintf("block has %v transactions", len(payloadTransactions))) continue } generalizedIndex := baseGeneralizedIndex + preconfIndex generalizedIndexes[i] = generalizedIndex - transactionHashes[i] = hash - i++ + transactionHashes[i] = tx.Hash() } log.Info(fmt.Sprintf("[BOLT]: Calculating merkle multiproof for %d preconfirmed transaction", diff --git a/builder/core/types/constraints.go b/builder/core/types/constraints.go index 4cefe15f..e587b475 100644 --- a/builder/core/types/constraints.go +++ b/builder/core/types/constraints.go @@ -1,6 +1,10 @@ package types -import "github.com/ethereum/go-ethereum/common" +import ( + "sort" + + "github.com/ethereum/go-ethereum/common" +) // NOTE: not the greatest place for this type but given that it uses // `common.Hash`, `Transaction` and it's used in both the builder @@ -13,3 +17,46 @@ type ( Tx *Transaction } ) + +// ParseConstraintsDecoded receives a map of constraints and returns +// - a slice of constraints sorted by index +// - a slice of constraints without index sorted by nonce and hash +// - the total gas required by the constraints +// - the total blob gas required by the constraints +func ParseConstraintsDecoded(constraints HashToConstraintDecoded) ([]*ConstraintDecoded, []*ConstraintDecoded, uint64, uint64) { + // Here we initialize and track the constraints left to be executed along + // with their gas requirements + constraintsOrderedByIndex := make([]*ConstraintDecoded, 0, len(constraints)) + constraintsWithoutIndex := make([]*ConstraintDecoded, 0, len(constraints)) + constraintsTotalGasLeft := uint64(0) + constraintsTotalBlobGasLeft := uint64(0) + + for _, constraint := range constraints { + if constraint.Index == nil { + constraintsWithoutIndex = append(constraintsWithoutIndex, constraint) + } else { + constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint) + } + constraintsTotalGasLeft += constraint.Tx.Gas() + constraintsTotalBlobGasLeft += constraint.Tx.BlobGas() + } + + // Sorts the constraints by index ascending + sort.Slice(constraintsOrderedByIndex, func(i, j int) bool { + // By assumption, all constraints here have a non-nil index + return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index + }) + + // Sorts the unindexed constraints by nonce ascending and by hash + sort.Slice(constraintsWithoutIndex, func(i, j int) bool { + iNonce := constraintsWithoutIndex[i].Tx.Nonce() + jNonce := constraintsWithoutIndex[j].Tx.Nonce() + // Sort by hash + if iNonce == jNonce { + return constraintsWithoutIndex[i].Tx.Hash().Cmp(constraintsWithoutIndex[j].Tx.Hash()) < 0 + } + return iNonce < jNonce + }) + + return constraintsOrderedByIndex, constraintsWithoutIndex, constraintsTotalGasLeft, constraintsTotalBlobGasLeft +} diff --git a/builder/go.mod b/builder/go.mod index e125adca..dfe1cc15 100644 --- a/builder/go.mod +++ b/builder/go.mod @@ -1,6 +1,6 @@ module github.com/ethereum/go-ethereum -go 1.20 +go 1.22 require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 @@ -26,7 +26,7 @@ require ( github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.15.0 - github.com/ferranbt/fastssz v0.1.3 + github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.2 github.com/flashbots/go-boost-utils v1.8.0 @@ -86,6 +86,7 @@ require ( ) require ( + github.com/emicklei/dot v1.6.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/getsentry/sentry-go v0.18.0 // indirect github.com/goccy/go-yaml v1.11.2 // indirect diff --git a/builder/go.sum b/builder/go.sum index f1abe31c..1ab78598 100644 --- a/builder/go.sum +++ b/builder/go.sum @@ -137,6 +137,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= +github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -150,8 +152,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= -github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 h1:k70X5h1haHaSbpD/9fcjtvAUEVlRlOKtdpvN7Mzhcv4= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= diff --git a/builder/miner/worker.go b/builder/miner/worker.go index 6400861a..c845edbd 100644 --- a/builder/miner/worker.go +++ b/builder/miner/worker.go @@ -1033,26 +1033,10 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac // Here we initialize and track the constraints left to be executed along // with their gas requirements - constraintsOrderedByIndex := make([]*types.ConstraintDecoded, 0, len(constraints)) - constraintsWithoutIndex := make([]*types.ConstraintDecoded, 0, len(constraints)) - constraintsTotalGasLeft := uint64(0) - constraintsTotalBlobGasLeft := uint64(0) - - for _, constraint := range constraints { - if constraint.Index == nil { - constraintsWithoutIndex = append(constraintsWithoutIndex, constraint) - } else { - constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint) - } - constraintsTotalGasLeft += constraint.Tx.Gas() - constraintsTotalBlobGasLeft += constraint.Tx.BlobGas() - } - - // Sorts the constraints by index ascending - sort.Slice(constraintsOrderedByIndex, func(i, j int) bool { - // By assumption, all constraints here have a non-nil index - return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index - }) + constraintsOrderedByIndex, + constraintsWithoutIndex, + constraintsTotalGasLeft, + constraintsTotalBlobGasLeft := types.ParseConstraintsDecoded(constraints) for { // `env.tcount` starts from 0 so it's correct to use it as the current index @@ -1177,7 +1161,7 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac // As such, we can safely exist break } - candidate = candidateTx{tx: common.Pop(&constraintsWithoutIndex).Tx, isConstraint: true} + candidate = candidateTx{tx: common.Shift(&constraintsWithoutIndex).Tx, isConstraint: true} } } diff --git a/mev-boost-relay/go.mod b/mev-boost-relay/go.mod index 8fb9f5aa..37f3e488 100644 --- a/mev-boost-relay/go.mod +++ b/mev-boost-relay/go.mod @@ -45,6 +45,7 @@ require ( github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/emicklei/dot v1.6.2 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect @@ -91,7 +92,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/ferranbt/fastssz v0.1.3 + github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect diff --git a/mev-boost-relay/go.sum b/mev-boost-relay/go.sum index f61dcb9b..3d75db6a 100644 --- a/mev-boost-relay/go.sum +++ b/mev-boost-relay/go.sum @@ -116,6 +116,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= +github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -131,6 +133,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 h1:k70X5h1haHaSbpD/9fcjtvAUEVlRlOKtdpvN7Mzhcv4= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/flashbots/go-boost-utils v1.8.0 h1:z3K1hw+Fbl9AGMNQKnK7Bvf0M/rKgjfruAEvra+Z8Mg= github.com/flashbots/go-boost-utils v1.8.0/go.mod h1:Ry1Rw8Lx5v1rpAR0+IvR4sV10jYAeQaGVM3vRD8mYdM= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= diff --git a/mev-boost-relay/services/api/proofs.go b/mev-boost-relay/services/api/proofs.go index b9638616..6505b0cb 100644 --- a/mev-boost-relay/services/api/proofs.go +++ b/mev-boost-relay/services/api/proofs.go @@ -6,7 +6,6 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/ethereum/go-ethereum/core/types" fastSsz "github.com/ferranbt/fastssz" "github.com/flashbots/mev-boost-relay/common" "github.com/sirupsen/logrus" @@ -20,43 +19,31 @@ var ( ) // verifyInclusionProof verifies the proofs against the constraints, and returns an error if the proofs are invalid. -func verifyInclusionProof(log *logrus.Entry, transactionsRoot phase0.Root, proof *common.InclusionProof, constraints map[phase0.Hash32]*Constraint) error { +// +// NOTE: assumes constraints transactions are already without blobs +func verifyInclusionProof(log *logrus.Entry, transactionsRoot phase0.Root, proof *common.InclusionProof, hashToConstraints HashToConstraintDecoded) error { if proof == nil { return ErrNilProof } + constraints := ParseConstraintsDecoded(hashToConstraints) + leaves := make([][]byte, len(constraints)) - i := 0 - for hash, constraint := range constraints { + for i, constraint := range constraints { if constraint == nil { return ErrNilConstraint } - if len(constraint.Tx) == 0 { - log.Warnf("[BOLT]: Raw tx is empty for constraint tx hash %s", hash) - continue - } - // Compute the hash tree root for the raw preconfirmed transaction // and use it as "Leaf" in the proof to be verified against - - // TODO: this is pretty inefficient, we should work with the transaction already - // parsed without the blob here to avoid unmarshalling and marshalling again - transaction := new(types.Transaction) - err := transaction.UnmarshalBinary(constraint.Tx) - if err != nil { - log.WithError(err).Error("error unmarshalling transaction while verifying proofs") - return err - } - - withoutBlob, err := transaction.WithoutBlobTxSidecar().MarshalBinary() + encoded, err := constraint.Tx.MarshalBinary() if err != nil { log.WithError(err).Error("error marshalling transaction without blob tx sidecar") return err } - tx := Transaction(withoutBlob) + tx := Transaction(encoded) txHashTreeRoot, err := tx.HashTreeRoot() if err != nil { return ErrInvalidRoot diff --git a/mev-boost-relay/services/api/service.go b/mev-boost-relay/services/api/service.go index 9d0a3fbc..7430a35b 100644 --- a/mev-boost-relay/services/api/service.go +++ b/mev-boost-relay/services/api/service.go @@ -2158,10 +2158,9 @@ func (api *RelayAPI) updateRedisBidWithProofs( api.RespondError(opts.w, http.StatusBadRequest, err.Error()) return nil, nil, false } - constraints := make(map[phase0.Hash32]*Constraint) + constraints := make(HashToConstraintDecoded) for _, signedConstraints := range *slotConstraints { for _, constraint := range signedConstraints.Message.Constraints { - // TODO: just compute the hash instead of decoding the entire tx just for this. decoded := new(types.Transaction) if err := decoded.UnmarshalBinary(constraint.Tx); err != nil { api.log.WithError(err).Error("could not decode transaction") @@ -2169,7 +2168,7 @@ func (api *RelayAPI) updateRedisBidWithProofs( return nil, nil, false } api.log.Infof("Decoded tx hash %s", decoded.Hash().String()) - constraints[phase0.Hash32(decoded.Hash().Bytes())] = constraint + constraints[decoded.Hash()] = &ConstraintDecoded{Tx: decoded.WithoutBlobTxSidecar(), Index: constraint.Index} } } diff --git a/mev-boost-relay/services/api/types.go b/mev-boost-relay/services/api/types.go index 8b1972a4..2454a0b5 100644 --- a/mev-boost-relay/services/api/types.go +++ b/mev-boost-relay/services/api/types.go @@ -4,8 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "sort" "github.com/attestantio/go-eth2-client/spec/phase0" + gethCommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" boostTypes "github.com/flashbots/go-boost-utils/types" ) @@ -39,3 +43,51 @@ func (c *ConstraintSubscriptionAuth) String() string { } return string(buf) } + +type ( + HashToConstraintDecoded = map[gethCommon.Hash]*ConstraintDecoded + ConstraintDecoded struct { + Index *Index + Tx *types.Transaction + } +) + +// ParseConstraintsDecoded receives a map of constraints and +// - creates a slice of constraints sorted by index +// - creates a slice of constraints without index sorted by nonce and hash +// Returns the concatenation of the slices +func ParseConstraintsDecoded(constraints HashToConstraintDecoded) []*ConstraintDecoded { + // Here we initialize and track the constraints left to be executed along + // with their gas requirements + constraintsOrderedByIndex := make([]*ConstraintDecoded, 0, len(constraints)) + constraintsWithoutIndex := make([]*ConstraintDecoded, 0, len(constraints)) + + for _, constraint := range constraints { + if constraint.Index == nil { + constraintsWithoutIndex = append(constraintsWithoutIndex, constraint) + } else { + constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint) + } + } + + // Sorts the constraints by index ascending + sort.Slice(constraintsOrderedByIndex, func(i, j int) bool { + // By assumption, all constraints here have a non-nil index + return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index + }) + + // Sorts the unindexed constraints by nonce ascending and by hash + sort.Slice(constraintsWithoutIndex, func(i, j int) bool { + iNonce := constraintsWithoutIndex[i].Tx.Nonce() + jNonce := constraintsWithoutIndex[j].Tx.Nonce() + // Sort by hash + if iNonce == jNonce { + return constraintsWithoutIndex[i].Tx.Hash().Cmp(constraintsWithoutIndex[j].Tx.Hash()) < 0 + } + return iNonce < jNonce + }) + + constraintsConcat := slices.Concat(constraintsOrderedByIndex, constraintsWithoutIndex) + + return constraintsConcat +} diff --git a/mev-boost/Dockerfile b/mev-boost/Dockerfile index e3fabfd6..495f5045 100644 --- a/mev-boost/Dockerfile +++ b/mev-boost/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM golang:1.21 AS builder +FROM golang:1.22-alpine AS builder ARG VERSION WORKDIR /build @@ -10,10 +10,10 @@ RUN go mod download ADD . . RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOOS=linux go build \ - -trimpath \ - -v \ - -ldflags "-w -s -X 'github.com/flashbots/mev-boost/config.Version=$VERSION'" \ - -o mev-boost . + -trimpath \ + -v \ + -ldflags "-w -s -X 'github.com/flashbots/mev-boost/config.Version=$VERSION'" \ + -o mev-boost . FROM alpine WORKDIR /app diff --git a/mev-boost/go.mod b/mev-boost/go.mod index cf25a570..d7451641 100644 --- a/mev-boost/go.mod +++ b/mev-boost/go.mod @@ -1,6 +1,6 @@ module github.com/flashbots/mev-boost -go 1.21 +go 1.22 require ( github.com/ethereum/go-ethereum v1.13.10 @@ -28,6 +28,7 @@ require ( github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/emicklei/dot v1.6.2 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect @@ -68,7 +69,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/ferranbt/fastssz v0.1.3 + github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect diff --git a/mev-boost/go.sum b/mev-boost/go.sum index 05d217a3..63d6c536 100644 --- a/mev-boost/go.sum +++ b/mev-boost/go.sum @@ -88,6 +88,8 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= +github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -103,6 +105,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 h1:k70X5h1haHaSbpD/9fcjtvAUEVlRlOKtdpvN7Mzhcv4= +github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/flashbots/go-boost-utils v1.8.0 h1:z3K1hw+Fbl9AGMNQKnK7Bvf0M/rKgjfruAEvra+Z8Mg= github.com/flashbots/go-boost-utils v1.8.0/go.mod h1:Ry1Rw8Lx5v1rpAR0+IvR4sV10jYAeQaGVM3vRD8mYdM= github.com/flashbots/go-utils v0.5.0 h1:ldjWta9B9//DJU2QcwRbErez3+1aKhSn6EoFc6d5kPY= diff --git a/mev-boost/server/constraints.go b/mev-boost/server/constraints.go index e5c21c22..e6edec36 100644 --- a/mev-boost/server/constraints.go +++ b/mev-boost/server/constraints.go @@ -1,8 +1,11 @@ package server import ( + "slices" + "sort" + "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/ethereum/go-ethereum/common" + gethCommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" lru "github.com/hashicorp/golang-lru/v2" ) @@ -40,13 +43,13 @@ func (c *Constraint) String() string { // ConstraintCache is a cache for constraints. type ConstraintCache struct { // map of slots to all constraints for that slot - constraints *lru.Cache[uint64, map[common.Hash]*Constraint] + constraints *lru.Cache[uint64, map[gethCommon.Hash]*Constraint] } // NewConstraintCache creates a new constraint cache. // cap is the maximum number of slots to store constraints for. func NewConstraintCache(cap int) *ConstraintCache { - constraints, _ := lru.New[uint64, map[common.Hash]*Constraint](cap) + constraints, _ := lru.New[uint64, map[gethCommon.Hash]*Constraint](cap) return &ConstraintCache{ constraints: constraints, } @@ -55,7 +58,7 @@ func NewConstraintCache(cap int) *ConstraintCache { // AddInclusionConstraint adds an inclusion constraint to the cache at the given slot for the given transaction. func (c *ConstraintCache) AddInclusionConstraint(slot uint64, tx Transaction, index *uint64) error { if _, exists := c.constraints.Get(slot); !exists { - c.constraints.Add(slot, make(map[common.Hash]*Constraint)) + c.constraints.Add(slot, make(map[gethCommon.Hash]*Constraint)) } // parse transaction to get its hash and store it in the cache @@ -78,7 +81,7 @@ func (c *ConstraintCache) AddInclusionConstraint(slot uint64, tx Transaction, in // AddInclusionConstraints adds multiple inclusion constraints to the cache at the given slot func (c *ConstraintCache) AddInclusionConstraints(slot uint64, constraints []*Constraint) error { if _, exists := c.constraints.Get(slot); !exists { - c.constraints.Add(slot, make(map[common.Hash]*Constraint)) + c.constraints.Add(slot, make(map[gethCommon.Hash]*Constraint)) } m, _ := c.constraints.Get(slot) @@ -95,12 +98,12 @@ func (c *ConstraintCache) AddInclusionConstraints(slot uint64, constraints []*Co } // Get gets the constraints at the given slot. -func (c *ConstraintCache) Get(slot uint64) (map[common.Hash]*Constraint, bool) { +func (c *ConstraintCache) Get(slot uint64) (map[gethCommon.Hash]*Constraint, bool) { return c.constraints.Get(slot) } // FindTransactionByHash finds the constraint for the given transaction hash and returns it. -func (c *ConstraintCache) FindTransactionByHash(txHash common.Hash) (*Constraint, bool) { +func (c *ConstraintCache) FindTransactionByHash(txHash gethCommon.Hash) (*Constraint, bool) { for _, hashToConstraint := range c.constraints.Values() { if constraint, exists := hashToConstraint[txHash]; exists { return constraint, true @@ -108,3 +111,51 @@ func (c *ConstraintCache) FindTransactionByHash(txHash common.Hash) (*Constraint } return nil, false } + +type ( + HashToConstraintDecoded = map[gethCommon.Hash]*ConstraintDecoded + ConstraintDecoded struct { + Index *uint64 + Tx *types.Transaction + } +) + +// ParseConstraintsDecoded receives a map of constraints and +// - creates a slice of constraints sorted by index +// - creates a slice of constraints without index sorted by nonce and hash +// Returns the concatenation of the slices +func ParseConstraintsDecoded(constraints HashToConstraintDecoded) []*ConstraintDecoded { + // Here we initialize and track the constraints left to be executed along + // with their gas requirements + constraintsOrderedByIndex := make([]*ConstraintDecoded, 0, len(constraints)) + constraintsWithoutIndex := make([]*ConstraintDecoded, 0, len(constraints)) + + for _, constraint := range constraints { + if constraint.Index == nil { + constraintsWithoutIndex = append(constraintsWithoutIndex, constraint) + } else { + constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint) + } + } + + // Sorts the constraints by index ascending + sort.Slice(constraintsOrderedByIndex, func(i, j int) bool { + // By assumption, all constraints here have a non-nil index + return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index + }) + + // Sorts the unindexed constraints by nonce ascending and by hash + sort.Slice(constraintsWithoutIndex, func(i, j int) bool { + iNonce := constraintsWithoutIndex[i].Tx.Nonce() + jNonce := constraintsWithoutIndex[j].Tx.Nonce() + // Sort by hash + if iNonce == jNonce { + return constraintsWithoutIndex[i].Tx.Hash().Cmp(constraintsWithoutIndex[j].Tx.Hash()) < 0 + } + return iNonce < jNonce + }) + + constraintsConcat := slices.Concat(constraintsOrderedByIndex, constraintsWithoutIndex) + + return constraintsConcat +} diff --git a/mev-boost/server/constraints_test.go b/mev-boost/server/constraints_test.go new file mode 100644 index 00000000..56e3f811 --- /dev/null +++ b/mev-boost/server/constraints_test.go @@ -0,0 +1,52 @@ +package server + +import ( + "encoding/hex" + "testing" + + gethTypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func Test_ParseContraintsDecoded(t *testing.T) { + rawTxs := []string{ + // These two will have index set, and nonce 367, 368 + "f86882016f84042343e082520894deaddeaddeaddeaddeaddeaddeaddeaddeaddead07808360306ba0a5b07edf4e7074a679b08cfc474364f3378e87006d82843c8bf306fc1c6e9e57a07927c7f92ac2f9a5166433e2b9bbc5f48ebf9d366d437c568c465cdf9ac148d8", + "f86882017084042343e082520894deaddeaddeaddeaddeaddeaddeaddeaddeaddead3b808360306ba082d4f1a817f12d59d21bbf1b156715bc1ab307f160b4a3e1527ec915a7757273a073a51224caa582e0cb34388ff188a68d022a6a283fcb9b4e6dfecece8ccf21e6", + // These three will not + // The first two will have same nonce 369, but one is to aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa just to have different hash: 0x678a4d09b8dd43ebd675b9e3f1983185f5a31f7b44e3f5815436a8fae647d1f9 + "f86882017184042343e082520894aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa27808360306ca00499b985ef217b1f9c03ef039190ae877b4578114e0b03a1ebbae977d5ca7d5fa0086a6d280a7dd5fdb64c4ddc727329f0ba1fa8c49deab9cafe71207f21dbf81b", + // This has hash 0x1c8e21622617cc02111389c67f542b5059cf5b024b265c5fdbcac529ae7ab7e0, so it will appear first + "f86882017184042343e082520894deaddeaddeaddeaddeaddeaddeaddeaddeaddead27808360306ca00499b985ef217b1f9c03ef039190ae877b4578114e0b03a1ebbae977d5ca7d5fa0086a6d280a7dd5fdb64c4ddc727329f0ba1fa8c49deab9cafe71207f21dbf81b", + // This will have nonce 370 + "f86882017284042343e082520894deaddeaddeaddeaddeaddeaddeaddeaddeaddead11808360306ba0891ff5261562c21a3f89f12d95391aef865c21f5cf72f97c4602aa9f072c0489a04c4482a46802d160c9a812cffc90be0cd6ffc1206c9dd2f5b53111d9098ff207", + // "f86882017384042343e082520894deaddeaddeaddeaddeaddeaddeaddeaddeaddead4b808360306ca087d083fadadeba27f213ebb2b428003aa730035686202547e2260cadfb824ee5a00c93b0f0403fbb78b5810e90b9f5bb63368dd6bcd5c31c56ed1b132167e7d69f", + } + + hashToConstraint := make(HashToConstraintDecoded) + + for i, rawTx := range rawTxs { + rawTxBytes, err := hex.DecodeString(rawTx) + require.NoError(t, err) + tx := new(gethTypes.Transaction) + err = tx.UnmarshalBinary(rawTxBytes) + + require.NoError(t, err) + var index *uint64 + if i < 2 { + index = new(uint64) + *index = uint64(i) + } + hashToConstraint[tx.Hash()] = &ConstraintDecoded{ + Tx: tx, + Index: index, + } + } + + constraintsParsed := ParseConstraintsDecoded(hashToConstraint) + require.Equal(t, uint64(367), constraintsParsed[0].Tx.Nonce()) + require.Equal(t, uint64(368), constraintsParsed[1].Tx.Nonce()) + require.Equal(t, "0x1c8e21622617cc02111389c67f542b5059cf5b024b265c5fdbcac529ae7ab7e0", constraintsParsed[2].Tx.Hash().String()) + require.Equal(t, uint64(369), constraintsParsed[3].Tx.Nonce()) + require.Equal(t, uint64(370), constraintsParsed[4].Tx.Nonce()) +} diff --git a/mev-boost/server/service.go b/mev-boost/server/service.go index 748aa97b..17f77766 100644 --- a/mev-boost/server/service.go +++ b/mev-boost/server/service.go @@ -343,6 +343,7 @@ func (m *BoostService) verifyInclusionProof(responsePayload *BidWithInclusionPro // BOLT: get constraints for the slot inclusionConstraints, exists := m.constraints.Get(slot) + if !exists { log.Warnf("[BOLT]: No constraints found for slot %d", slot) return errMissingConstraint @@ -363,34 +364,39 @@ func (m *BoostService) verifyInclusionProof(responsePayload *BidWithInclusionPro return errInvalidRoot } - leaves := make([][]byte, len(inclusionConstraints)) - i := 0 - + // Decode the constraints, and sort them according to the utility function used + // TODO: this should be done before verification ideally + hashToConstraint := make(HashToConstraintDecoded) for hash, constraint := range inclusionConstraints { - if len(constraint.Tx) == 0 { - log.Warnf("[BOLT]: Raw tx is empty for constraint tx hash %s", hash) - continue - } - - // Compute the hash tree root for the raw preconfirmed transaction - // and use it as "Leaf" in the proof to be verified against - - // TODO: this is pretty inefficient, we should work with the transaction already - // parsed without the blob here to avoid unmarshalling and marshalling again transaction := new(gethTypes.Transaction) err := transaction.UnmarshalBinary(constraint.Tx) if err != nil { log.WithError(err).Error("error unmarshalling transaction while verifying proofs") return err } + hashToConstraint[hash] = &ConstraintDecoded{ + Tx: transaction.WithoutBlobTxSidecar(), + Index: constraint.Index, + } + } + constraints := ParseConstraintsDecoded(hashToConstraint) - withoutBlob, err := transaction.WithoutBlobTxSidecar().MarshalBinary() + leaves := make([][]byte, len(constraints)) + + for i, constraint := range constraints { + // Compute the hash tree root for the raw preconfirmed transaction + // and use it as "Leaf" in the proof to be verified against + + // TODO: this is pretty inefficient, we should work with the transaction already + // parsed without the blob here to avoid unmarshalling and marshalling again + transaction := constraint.Tx + encoded, err := transaction.MarshalBinary() if err != nil { log.WithError(err).Error("error marshalling transaction without blob tx sidecar") return err } - tx := Transaction(withoutBlob) + tx := Transaction(encoded) txHashTreeRoot, err := tx.HashTreeRoot() if err != nil { return errInvalidRoot