From 3d487b3d23003a31a3a355758c218a967a4c49f9 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Tue, 14 May 2024 10:17:50 +0200 Subject: [PATCH] commit before polish --- docker-compose.dev.yml | 2 +- .../internal/api/handlers/postNewSignature.go | 72 +++++++++++-------- listener/internal/api/types/types.go | 3 +- .../api/validation/GetActiveValidators.go | 13 ++-- .../validation/GetActiveValidators_test.go | 23 +++--- .../api/validation/IsValidSignature.go | 34 +++++++-- .../api/validation/IsValidSignature_test.go | 4 +- listener/internal/config/config.go | 2 - web3signer/brain/Dockerfile | 8 +-- web3signer/signer/Dockerfile | 8 --- web3signer/signer/entrypoint.sh | 3 +- 11 files changed, 94 insertions(+), 78 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8b3f074..dbb340d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,7 +46,7 @@ services: networks: dncore_network: aliases: - - web3signer.web3signer.dappnode + - web3signer.web3signer-holesky.dappnode restart: always flyway: diff --git a/listener/internal/api/handlers/postNewSignature.go b/listener/internal/api/handlers/postNewSignature.go index af170e7..da00409 100644 --- a/listener/internal/api/handlers/postNewSignature.go +++ b/listener/internal/api/handlers/postNewSignature.go @@ -16,6 +16,7 @@ import ( type signatureRequest struct { Payload string `json:"payload"` + Pubkey string `json:"pubkey"` Signature string `json:"signature"` Network string `json:"network"` Tag string `json:"tag"` @@ -31,7 +32,7 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded var validRequests []types.SignatureRequestDecoded for _, req := range requests { - if req.Network == "" || req.Tag == "" || req.Signature == "" || req.Payload == "" { + if req.Network == "" || req.Tag == "" || req.Signature == "" || req.Payload == "" || req.Pubkey == "" { logger.Debug("Skipping invalid signature from request, missing fields") continue } @@ -49,10 +50,11 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded logger.Error("Failed to decode JSON payload from request: " + err.Error()) continue } - if decodedPayload.Platform == "dappnode" && decodedPayload.Timestamp != "" && decodedPayload.Pubkey != "" { + if decodedPayload.Platform == "dappnode" && decodedPayload.Timestamp != "" { validRequests = append(validRequests, types.SignatureRequestDecoded{ DecodedPayload: decodedPayload, Payload: req.Payload, + Pubkey: req.Pubkey, Signature: req.Signature, Network: req.Network, Tag: req.Tag, @@ -62,6 +64,17 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded } } + // print req pubkey + for _, req := range validRequests { + logger.Debug("req.Pubkey: " + req.Pubkey) + logger.Debug("req.Signature: " + req.Signature) + logger.Debug("req.Network: " + req.Network) + logger.Debug("req.Tag: " + req.Tag) + logger.Debug("req.DecodedPayload.Type: " + req.DecodedPayload.Type) + logger.Debug("req.DecodedPayload.Platform: " + req.DecodedPayload.Platform) + logger.Debug("req.DecodedPayload.Timestamp: " + req.DecodedPayload.Timestamp) + } + return validRequests, nil } @@ -80,7 +93,7 @@ func validateAndInsertSignature(req types.SignatureRequestDecoded, dbCollection _, err = dbCollection.InsertOne(context.TODO(), bson.M{ "platform": req.DecodedPayload.Platform, "timestamp": req.DecodedPayload.Timestamp, - "pubkey": req.DecodedPayload.Pubkey, + "pubkey": req.Pubkey, "signature": req.Signature, "network": req.Network, "tag": req.Tag, @@ -113,37 +126,34 @@ func PostNewSignature(w http.ResponseWriter, r *http.Request, dbCollection *mong return } - var wg sync.WaitGroup - appendMutex := new(sync.Mutex) // Mutex for appending to the slice - dbMutex := new(sync.Mutex) // Mutex for database operations - allValidatedRequests := []types.SignatureRequestDecoded{} // This will collect all valid requests - - // Iterate over all beacon nodes and get active validators - for _, url := range beaconNodeUrls { - wg.Add(1) - go func(url string) { - defer wg.Done() - activeValidators := validation.GetActiveValidators(validRequests, url, bypassValidatorFiltering) - if len(activeValidators) == 0 { - return - } - - appendMutex.Lock() - allValidatedRequests = append(allValidatedRequests, activeValidators...) // Only one goroutine can append to the slice at a time - appendMutex.Unlock() - - for _, req := range activeValidators { - dbMutex.Lock() - validateAndInsertSignature(req, dbCollection) // Do we really need to lock the db insertions? - dbMutex.Unlock() - } - }(url) // Pass the URL to the goroutine + // Get active validators from the network, get the network from the first item in the array + beaconNodeUrl, ok := beaconNodeUrls[validRequests[0].Network] + if !ok { + respondError(w, http.StatusBadRequest, "Invalid network") + return + } + + // if bypassValidatorFiltering is true, we skip the active validators check + activeValidators := []types.SignatureRequestDecoded{} + if bypassValidatorFiltering { + logger.Debug("Bypassing active validators check") + activeValidators = validRequests + } else { + activeValidators = validation.GetActiveValidators(validRequests, beaconNodeUrl) } - wg.Wait() - if len(allValidatedRequests) == 0 { - respondError(w, http.StatusInternalServerError, "No active validators found in any network") + if len(activeValidators) == 0 { + respondError(w, http.StatusInternalServerError, "No active validators found in network") return } + + // Iterate over all active validators and validate and insert the signature + dbMutex := new(sync.Mutex) // Mutex for database operations + for _, req := range activeValidators { + dbMutex.Lock() + validateAndInsertSignature(req, dbCollection) // Do we really need to lock the db insertions? + dbMutex.Unlock() + } + respondOK(w, "Finished processing signatures") } diff --git a/listener/internal/api/types/types.go b/listener/internal/api/types/types.go index b924e0e..94e3199 100644 --- a/listener/internal/api/types/types.go +++ b/listener/internal/api/types/types.go @@ -3,6 +3,7 @@ package types type SignatureRequestDecoded struct { DecodedPayload DecodedPayload `json:"decodedPayload"` Payload string `json:"payload"` + Pubkey string `json:"pubkey"` Signature string `json:"signature"` Network string `json:"network"` Tag string `json:"tag"` @@ -11,7 +12,7 @@ type SignatureRequestDecoded struct { type DecodedPayload struct { Platform string `json:"platform"` Timestamp string `json:"timestamp"` - Pubkey string `json:"pubkey"` + Type string `json:"type"` } type ActiveValidator struct { diff --git a/listener/internal/api/validation/GetActiveValidators.go b/listener/internal/api/validation/GetActiveValidators.go index c374424..6fe2263 100644 --- a/listener/internal/api/validation/GetActiveValidators.go +++ b/listener/internal/api/validation/GetActiveValidators.go @@ -12,8 +12,7 @@ import ( // GetActiveValidators checks the active status of validators from a specific beacon node. // If bypass is true, it simply returns all decoded requests. -func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beaconNodeUrl string, bypass bool) []types.SignatureRequestDecoded { - +func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beaconNodeUrl string) []types.SignatureRequestDecoded { if len(requestsDecoded) == 0 { fmt.Println("no requests to process") return nil @@ -21,7 +20,7 @@ func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beacon ids := make([]string, 0, len(requestsDecoded)) for _, req := range requestsDecoded { - ids = append(ids, req.DecodedPayload.Pubkey) + ids = append(ids, req.Pubkey) } if len(ids) == 0 { @@ -77,14 +76,10 @@ func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beacon // Filter the list of decoded requests to include only those that are active var activeValidators []types.SignatureRequestDecoded for _, req := range requestsDecoded { - if _, isActive := activeValidatorMap[req.DecodedPayload.Pubkey]; isActive { + if _, isActive := activeValidatorMap[req.Pubkey]; isActive { activeValidators = append(activeValidators, req) } } - if bypass { - return requestsDecoded // do not return the filtered list - } else { - return activeValidators // return the filtered list (default behaviour) - } + return activeValidators } diff --git a/listener/internal/api/validation/GetActiveValidators_test.go b/listener/internal/api/validation/GetActiveValidators_test.go index 9520a96..691763a 100644 --- a/listener/internal/api/validation/GetActiveValidators_test.go +++ b/listener/internal/api/validation/GetActiveValidators_test.go @@ -14,27 +14,24 @@ func TestGetActiveValidators(t *testing.T) { requestsDecoded := []types.SignatureRequestDecoded{ { - Network: "holesky", - DecodedPayload: types.DecodedPayload{ - Pubkey: "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca", - }, + Network: "holesky", + Pubkey: "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca", + DecodedPayload: types.DecodedPayload{}, }, { - Network: "holesky", - DecodedPayload: types.DecodedPayload{ - Pubkey: "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63", - }, + Network: "holesky", + Pubkey: "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63", + DecodedPayload: types.DecodedPayload{}, }, { - Network: "holesky", - DecodedPayload: types.DecodedPayload{ - Pubkey: "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b", - }, + Network: "holesky", + Pubkey: "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b", + DecodedPayload: types.DecodedPayload{}, }, } // Call the function. "bypass" is set to false, so the function will do expected behaviour and filter out inactive validators - result := GetActiveValidators(requestsDecoded, beaconNodeUrls["holesky"], false) + result := GetActiveValidators(requestsDecoded, beaconNodeUrls["holesky"]) // You may need to mock the server's response or adjust the expected values here according to your actual setup expectedNumValidators := 3 // This should match the number of mock validators that are "active" diff --git a/listener/internal/api/validation/IsValidSignature.go b/listener/internal/api/validation/IsValidSignature.go index f0386a3..587a5d7 100644 --- a/listener/internal/api/validation/IsValidSignature.go +++ b/listener/internal/api/validation/IsValidSignature.go @@ -3,45 +3,67 @@ package validation import ( "encoding/hex" "encoding/json" + "strings" "github.com/dappnode/validator-monitoring/listener/internal/api/types" + "github.com/dappnode/validator-monitoring/listener/internal/logger" "github.com/herumi/bls-eth-go-binary/bls" ) func IsValidSignature(req types.SignatureRequestDecoded) (bool, error) { // Initialize the BLS system if err := bls.Init(bls.BLS12_381); err != nil { + logger.Error("Failed to initialize BLS system: " + err.Error()) return false, err } + // Set the BLS mode to ETH draft 07 + if err := bls.SetETHmode(bls.EthModeDraft07); err != nil { + logger.Error("Failed to set BLS mode to ETH draft 07: " + err.Error()) + return false, err + } + + // Decode the public key from hex, remove the 0x prefix ONLY if exists from req.Pubkey + req.Pubkey = strings.TrimPrefix(req.Pubkey, "0x") + req.Pubkey = strings.TrimSpace(req.Pubkey) - // Decode the public key from hex - pubkeyBytes, err := hex.DecodeString(req.DecodedPayload.Pubkey) + pubkeyBytes, err := hex.DecodeString(req.Pubkey) if err != nil { + logger.Error("Failed to decode public key from hex: " + err.Error()) return false, err } - var pubkey bls.PublicKey - if err := pubkey.Deserialize(pubkeyBytes); err != nil { + + var pubkeyDes bls.PublicKey + if err := pubkeyDes.Deserialize(pubkeyBytes); err != nil { + logger.Error("Failed to deserialize public key: " + err.Error()) return false, err } - // Decode the signature from hex + // Decode the signature from hex, remove the 0x prefix ONLY if exists from req.Signature + req.Signature = strings.TrimPrefix(req.Signature, "0x") + req.Signature = strings.TrimSpace(req.Signature) + sigBytes, err := hex.DecodeString(req.Signature) if err != nil { + logger.Error("Failed to decode signature from hex: " + err.Error()) return false, err } + var sig bls.Sign if err := sig.Deserialize(sigBytes); err != nil { + logger.Error("Failed to deserialize signature: " + err.Error()) return false, err } // Serialize payload to string (assuming it's what was signed) payloadBytes, err := json.Marshal(req.DecodedPayload) if err != nil { + logger.Error("Failed to serialize payload to string: " + err.Error()) return false, err } // Verify the signature - if !sig.VerifyByte(&pubkey, payloadBytes) { + if !sig.VerifyByte(&pubkeyDes, payloadBytes) { + logger.Debug("Failed to verify signature") return false, nil } diff --git a/listener/internal/api/validation/IsValidSignature_test.go b/listener/internal/api/validation/IsValidSignature_test.go index b616670..1a1e3a2 100644 --- a/listener/internal/api/validation/IsValidSignature_test.go +++ b/listener/internal/api/validation/IsValidSignature_test.go @@ -24,7 +24,6 @@ func TestIsValidSignature(t *testing.T) { decodedPayload := types.DecodedPayload{ Platform: "dappnode", Timestamp: "2024-04-01T15:00:00Z", - Pubkey: publicKey.SerializeToHexStr(), } // Serialize the message @@ -39,6 +38,7 @@ func TestIsValidSignature(t *testing.T) { // Prepare the request req := types.SignatureRequestDecoded{ DecodedPayload: decodedPayload, + Pubkey: publicKey.SerializeToHexStr(), Payload: base64.StdEncoding.EncodeToString(messageBytes), Signature: signature.SerializeToHexStr(), Network: "mainnet", @@ -69,7 +69,6 @@ func TestIsValidSignatureError(t *testing.T) { decodedPayload := types.DecodedPayload{ Platform: "dappnode", Timestamp: "2024-04-01T15:00:00Z", - Pubkey: badPublicKey, } // Serialize the payload @@ -81,6 +80,7 @@ func TestIsValidSignatureError(t *testing.T) { // Create the SignatureRequestDecoded with a bad signature to ensure it fails req := types.SignatureRequestDecoded{ DecodedPayload: decodedPayload, + Pubkey: badPublicKey, Payload: base64.StdEncoding.EncodeToString(payloadBytes), Signature: "clearlyInvalidSignature", // Intentionally invalid Network: "mainnet", diff --git a/listener/internal/config/config.go b/listener/internal/config/config.go index 2d94333..e2d9ad8 100644 --- a/listener/internal/config/config.go +++ b/listener/internal/config/config.go @@ -21,9 +21,7 @@ type Config struct { BeaconNodeURLs map[string]string } -// TODO: read bypass boolean env func LoadConfig() (*Config, error) { - mongoDBURI := os.Getenv("MONGO_DB_URI") if mongoDBURI == "" { logger.Fatal("MONGO_DB_URI is not set") diff --git a/web3signer/brain/Dockerfile b/web3signer/brain/Dockerfile index c87fad7..6262167 100644 --- a/web3signer/brain/Dockerfile +++ b/web3signer/brain/Dockerfile @@ -1,8 +1,8 @@ -FROM ghcr.io/dappnode/staking-brain:0.1.17 +FROM staking-brain:0.1.18 # Mandatory envs for brain -ENV NETWORK=mainnet -ENV _DAPPNODE_GLOBAL_EXECUTION_CLIENT_MAINNET="erigon.dnp.dappnode.eth" -ENV _DAPPNODE_GLOBAL_CONSENSUS_CLIENT_MAINNET="prysm.dnp.dappnode.eth" +ENV NETWORK=holesky +ENV _DAPPNODE_GLOBAL_EXECUTION_CLIENT_HOLESKY="holesky-geth.dnp.dappnode.eth" +ENV _DAPPNODE_GLOBAL_CONSENSUS_CLIENT_HOLESKY="prysm-holesky.dnp.dappnode.eth" ENV SHARE_DATA_WITH_DAPPNODE="true" ENV VALIDATORS_MONITOR_URL="http://listener.dappnode:8080" ENV SHARE_CRON_INTERVAL="3600" \ No newline at end of file diff --git a/web3signer/signer/Dockerfile b/web3signer/signer/Dockerfile index 0fe8f42..8bf9d97 100644 --- a/web3signer/signer/Dockerfile +++ b/web3signer/signer/Dockerfile @@ -4,14 +4,6 @@ WORKDIR /app RUN apt update && apt install git -y && git clone https://github.com/consensys/web3signer.git WORKDIR /app/web3signer RUN git fetch origin && ./gradlew clean assemble && tar -xzf ./build/distributions/web3signer-develop.tar.gz - -############## -# WEB3SIGNER # -############## - -FROM consensys/web3signer:24.2.0 -# Copy web3signer keymanager binary -COPY --from=jre-build /app/web3signer/web3signer-develop /opt/web3signer COPY entrypoint.sh /usr/bin/entrypoint.sh EXPOSE 9000 diff --git a/web3signer/signer/entrypoint.sh b/web3signer/signer/entrypoint.sh index 516830e..4354989 100755 --- a/web3signer/signer/entrypoint.sh +++ b/web3signer/signer/entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash -exec /opt/web3signer/bin/web3signer --logging=DEBUG --http-listen-port=9000 \ +exec /app/web3signer/web3signer-develop/bin/web3signer --logging=DEBUG --http-listen-port=9000 \ --http-listen-host=0.0.0.0 --http-host-allowlist="*" --http-cors-origins="*" \ eth2 \ + --network=holesky \ --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer.dappnode:5432/web3signer \ --slashing-protection-db-username=postgres \ --slashing-protection-db-password=password \