commit before polish

docker-compose.dev.yml

@@ -46,7 +46,7 @@ services:
     networks:
       dncore_network:
         aliases:
-          - web3signer.web3signer.dappnode
+          - web3signer.web3signer-holesky.dappnode
     restart: always
 
   flyway:

listener/internal/api/handlers/postNewSignature.go

@@ -16,6 +16,7 @@ import (
 
 type signatureRequest struct {
 	Payload   string `json:"payload"`
+	Pubkey    string `json:"pubkey"`
 	Signature string `json:"signature"`
 	Network   string `json:"network"`
 	Tag       string `json:"tag"`
@@ -31,7 +32,7 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded
 
 	var validRequests []types.SignatureRequestDecoded
 	for _, req := range requests {
-		if req.Network == "" || req.Tag == "" || req.Signature == "" || req.Payload == "" {
+		if req.Network == "" || req.Tag == "" || req.Signature == "" || req.Payload == "" || req.Pubkey == "" {
 			logger.Debug("Skipping invalid signature from request, missing fields")
 			continue
 		}
@@ -49,10 +50,11 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded
 			logger.Error("Failed to decode JSON payload from request: " + err.Error())
 			continue
 		}
-		if decodedPayload.Platform == "dappnode" && decodedPayload.Timestamp != "" && decodedPayload.Pubkey != "" {
+		if decodedPayload.Platform == "dappnode" && decodedPayload.Timestamp != "" {
 			validRequests = append(validRequests, types.SignatureRequestDecoded{
 				DecodedPayload: decodedPayload,
 				Payload:        req.Payload,
+				Pubkey:         req.Pubkey,
 				Signature:      req.Signature,
 				Network:        req.Network,
 				Tag:            req.Tag,
@@ -62,6 +64,17 @@ func decodeAndValidateRequests(r *http.Request) ([]types.SignatureRequestDecoded
 		}
 	}
 
+	// print req pubkey
+	for _, req := range validRequests {
+		logger.Debug("req.Pubkey: " + req.Pubkey)
+		logger.Debug("req.Signature: " + req.Signature)
+		logger.Debug("req.Network: " + req.Network)
+		logger.Debug("req.Tag: " + req.Tag)
+		logger.Debug("req.DecodedPayload.Type: " + req.DecodedPayload.Type)
+		logger.Debug("req.DecodedPayload.Platform: " + req.DecodedPayload.Platform)
+		logger.Debug("req.DecodedPayload.Timestamp: " + req.DecodedPayload.Timestamp)
+	}
+
 	return validRequests, nil
 }
 
@@ -80,7 +93,7 @@ func validateAndInsertSignature(req types.SignatureRequestDecoded, dbCollection
 	_, err = dbCollection.InsertOne(context.TODO(), bson.M{
 		"platform":  req.DecodedPayload.Platform,
 		"timestamp": req.DecodedPayload.Timestamp,
-		"pubkey":    req.DecodedPayload.Pubkey,
+		"pubkey":    req.Pubkey,
 		"signature": req.Signature,
 		"network":   req.Network,
 		"tag":       req.Tag,
@@ -113,37 +126,34 @@ func PostNewSignature(w http.ResponseWriter, r *http.Request, dbCollection *mong
 		return
 	}
 
-	var wg sync.WaitGroup
-	appendMutex := new(sync.Mutex)                            // Mutex for appending to the slice
-	dbMutex := new(sync.Mutex)                                // Mutex for database operations
-	allValidatedRequests := []types.SignatureRequestDecoded{} // This will collect all valid requests
-
-	// Iterate over all beacon nodes and get active validators
-	for _, url := range beaconNodeUrls {
-		wg.Add(1)
-		go func(url string) {
-			defer wg.Done()
-			activeValidators := validation.GetActiveValidators(validRequests, url, bypassValidatorFiltering)
-			if len(activeValidators) == 0 {
-				return
-			}
-
-			appendMutex.Lock()
-			allValidatedRequests = append(allValidatedRequests, activeValidators...) // Only one goroutine can append to the slice at a time
-			appendMutex.Unlock()
-
-			for _, req := range activeValidators {
-				dbMutex.Lock()
-				validateAndInsertSignature(req, dbCollection) // Do we really need to lock the db insertions?
-				dbMutex.Unlock()
-			}
-		}(url) // Pass the URL to the goroutine
+	// Get active validators from the network, get the network from the first item in the array
+	beaconNodeUrl, ok := beaconNodeUrls[validRequests[0].Network]
+	if !ok {
+		respondError(w, http.StatusBadRequest, "Invalid network")
+		return
+	}
+
+	// if bypassValidatorFiltering is true, we skip the active validators check
+	activeValidators := []types.SignatureRequestDecoded{}
+	if bypassValidatorFiltering {
+		logger.Debug("Bypassing active validators check")
+		activeValidators = validRequests
+	} else {
+		activeValidators = validation.GetActiveValidators(validRequests, beaconNodeUrl)
 	}
 
-	wg.Wait()
-	if len(allValidatedRequests) == 0 {
-		respondError(w, http.StatusInternalServerError, "No active validators found in any network")
+	if len(activeValidators) == 0 {
+		respondError(w, http.StatusInternalServerError, "No active validators found in network")
 		return
 	}
+
+	// Iterate over all active validators and validate and insert the signature
+	dbMutex := new(sync.Mutex) // Mutex for database operations
+	for _, req := range activeValidators {
+		dbMutex.Lock()
+		validateAndInsertSignature(req, dbCollection) // Do we really need to lock the db insertions?
+		dbMutex.Unlock()
+	}
+
 	respondOK(w, "Finished processing signatures")
 }

listener/internal/api/types/types.go

@@ -3,6 +3,7 @@ package types
 type SignatureRequestDecoded struct {
 	DecodedPayload DecodedPayload `json:"decodedPayload"`
 	Payload        string         `json:"payload"`
+	Pubkey         string         `json:"pubkey"`
 	Signature      string         `json:"signature"`
 	Network        string         `json:"network"`
 	Tag            string         `json:"tag"`
@@ -11,7 +12,7 @@ type SignatureRequestDecoded struct {
 type DecodedPayload struct {
 	Platform  string `json:"platform"`
 	Timestamp string `json:"timestamp"`
-	Pubkey    string `json:"pubkey"`
+	Type      string `json:"type"`
 }
 
 type ActiveValidator struct {

listener/internal/api/validation/GetActiveValidators.go

@@ -12,16 +12,15 @@ import (
 
 // GetActiveValidators checks the active status of validators from a specific beacon node.
 // If bypass is true, it simply returns all decoded requests.
-func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beaconNodeUrl string, bypass bool) []types.SignatureRequestDecoded {
-
+func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beaconNodeUrl string) []types.SignatureRequestDecoded {
 	if len(requestsDecoded) == 0 {
 		fmt.Println("no requests to process")
 		return nil
 	}
 
 	ids := make([]string, 0, len(requestsDecoded))
 	for _, req := range requestsDecoded {
-		ids = append(ids, req.DecodedPayload.Pubkey)
+		ids = append(ids, req.Pubkey)
 	}
 
 	if len(ids) == 0 {
@@ -77,14 +76,10 @@ func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beacon
 	// Filter the list of decoded requests to include only those that are active
 	var activeValidators []types.SignatureRequestDecoded
 	for _, req := range requestsDecoded {
-		if _, isActive := activeValidatorMap[req.DecodedPayload.Pubkey]; isActive {
+		if _, isActive := activeValidatorMap[req.Pubkey]; isActive {
 			activeValidators = append(activeValidators, req)
 		}
 	}
 
-	if bypass {
-		return requestsDecoded // do not return the filtered list
-	} else {
-		return activeValidators // return the filtered list (default behaviour)
-	}
+	return activeValidators
 }

listener/internal/api/validation/GetActiveValidators_test.go

@@ -14,27 +14,24 @@ func TestGetActiveValidators(t *testing.T) {
 
 	requestsDecoded := []types.SignatureRequestDecoded{
 		{
-			Network: "holesky",
-			DecodedPayload: types.DecodedPayload{
-				Pubkey: "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca",
-			},
+			Network:        "holesky",
+			Pubkey:         "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca",
+			DecodedPayload: types.DecodedPayload{},
 		},
 		{
-			Network: "holesky",
-			DecodedPayload: types.DecodedPayload{
-				Pubkey: "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63",
-			},
+			Network:        "holesky",
+			Pubkey:         "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63",
+			DecodedPayload: types.DecodedPayload{},
 		},
 		{
-			Network: "holesky",
-			DecodedPayload: types.DecodedPayload{
-				Pubkey: "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b",
-			},
+			Network:        "holesky",
+			Pubkey:         "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b",
+			DecodedPayload: types.DecodedPayload{},
 		},
 	}
 
 	// Call the function. "bypass" is set to false, so the function will do expected behaviour and filter out inactive validators
-	result := GetActiveValidators(requestsDecoded, beaconNodeUrls["holesky"], false)
+	result := GetActiveValidators(requestsDecoded, beaconNodeUrls["holesky"])
 
 	// You may need to mock the server's response or adjust the expected values here according to your actual setup
 	expectedNumValidators := 3 // This should match the number of mock validators that are "active"

listener/internal/api/validation/IsValidSignature.go

@@ -3,45 +3,67 @@ package validation
 import (
 	"encoding/hex"
 	"encoding/json"
+	"strings"
 
 	"github.com/dappnode/validator-monitoring/listener/internal/api/types"
+	"github.com/dappnode/validator-monitoring/listener/internal/logger"
 	"github.com/herumi/bls-eth-go-binary/bls"
 )
 
 func IsValidSignature(req types.SignatureRequestDecoded) (bool, error) {
 	// Initialize the BLS system
 	if err := bls.Init(bls.BLS12_381); err != nil {
+		logger.Error("Failed to initialize BLS system: " + err.Error())
 		return false, err
 	}
+	// Set the BLS mode to ETH draft 07
+	if err := bls.SetETHmode(bls.EthModeDraft07); err != nil {
+		logger.Error("Failed to set BLS mode to ETH draft 07: " + err.Error())
+		return false, err
+	}
+
+	// Decode the public key from hex, remove the 0x prefix ONLY if exists from req.Pubkey
+	req.Pubkey = strings.TrimPrefix(req.Pubkey, "0x")
+	req.Pubkey = strings.TrimSpace(req.Pubkey)
 
-	// Decode the public key from hex
-	pubkeyBytes, err := hex.DecodeString(req.DecodedPayload.Pubkey)
+	pubkeyBytes, err := hex.DecodeString(req.Pubkey)
 	if err != nil {
+		logger.Error("Failed to decode public key from hex: " + err.Error())
 		return false, err
 	}
-	var pubkey bls.PublicKey
-	if err := pubkey.Deserialize(pubkeyBytes); err != nil {
+
+	var pubkeyDes bls.PublicKey
+	if err := pubkeyDes.Deserialize(pubkeyBytes); err != nil {
+		logger.Error("Failed to deserialize public key: " + err.Error())
 		return false, err
 	}
 
-	// Decode the signature from hex
+	// Decode the signature from hex, remove the 0x prefix ONLY if exists from req.Signature
+	req.Signature = strings.TrimPrefix(req.Signature, "0x")
+	req.Signature = strings.TrimSpace(req.Signature)
+
 	sigBytes, err := hex.DecodeString(req.Signature)
 	if err != nil {
+		logger.Error("Failed to decode signature from hex: " + err.Error())
 		return false, err
 	}
+
 	var sig bls.Sign
 	if err := sig.Deserialize(sigBytes); err != nil {
+		logger.Error("Failed to deserialize signature: " + err.Error())
 		return false, err
 	}
 
 	// Serialize payload to string (assuming it's what was signed)
 	payloadBytes, err := json.Marshal(req.DecodedPayload)
 	if err != nil {
+		logger.Error("Failed to serialize payload to string: " + err.Error())
 		return false, err
 	}
 
 	// Verify the signature
-	if !sig.VerifyByte(&pubkey, payloadBytes) {
+	if !sig.VerifyByte(&pubkeyDes, payloadBytes) {
+		logger.Debug("Failed to verify signature")
 		return false, nil
 	}
 

listener/internal/api/validation/IsValidSignature_test.go

@@ -24,7 +24,6 @@ func TestIsValidSignature(t *testing.T) {
 	decodedPayload := types.DecodedPayload{
 		Platform:  "dappnode",
 		Timestamp: "2024-04-01T15:00:00Z",
-		Pubkey:    publicKey.SerializeToHexStr(),
 	}
 
 	// Serialize the message
@@ -39,6 +38,7 @@ func TestIsValidSignature(t *testing.T) {
 	// Prepare the request
 	req := types.SignatureRequestDecoded{
 		DecodedPayload: decodedPayload,
+		Pubkey:         publicKey.SerializeToHexStr(),
 		Payload:        base64.StdEncoding.EncodeToString(messageBytes),
 		Signature:      signature.SerializeToHexStr(),
 		Network:        "mainnet",
@@ -69,7 +69,6 @@ func TestIsValidSignatureError(t *testing.T) {
 	decodedPayload := types.DecodedPayload{
 		Platform:  "dappnode",
 		Timestamp: "2024-04-01T15:00:00Z",
-		Pubkey:    badPublicKey,
 	}
 
 	// Serialize the payload
@@ -81,6 +80,7 @@ func TestIsValidSignatureError(t *testing.T) {
 	// Create the SignatureRequestDecoded with a bad signature to ensure it fails
 	req := types.SignatureRequestDecoded{
 		DecodedPayload: decodedPayload,
+		Pubkey:         badPublicKey,
 		Payload:        base64.StdEncoding.EncodeToString(payloadBytes),
 		Signature:      "clearlyInvalidSignature", // Intentionally invalid
 		Network:        "mainnet",

listener/internal/config/config.go

@@ -21,9 +21,7 @@ type Config struct {
 	BeaconNodeURLs map[string]string
 }
 
-// TODO: read bypass boolean env
 func LoadConfig() (*Config, error) {
-
 	mongoDBURI := os.Getenv("MONGO_DB_URI")
 	if mongoDBURI == "" {
 		logger.Fatal("MONGO_DB_URI is not set")

web3signer/brain/Dockerfile

@@ -1,8 +1,8 @@
-FROM ghcr.io/dappnode/staking-brain:0.1.17
+FROM staking-brain:0.1.18
 # Mandatory envs for brain
-ENV NETWORK=mainnet
-ENV _DAPPNODE_GLOBAL_EXECUTION_CLIENT_MAINNET="erigon.dnp.dappnode.eth"
-ENV _DAPPNODE_GLOBAL_CONSENSUS_CLIENT_MAINNET="prysm.dnp.dappnode.eth"
+ENV NETWORK=holesky
+ENV _DAPPNODE_GLOBAL_EXECUTION_CLIENT_HOLESKY="holesky-geth.dnp.dappnode.eth"
+ENV _DAPPNODE_GLOBAL_CONSENSUS_CLIENT_HOLESKY="prysm-holesky.dnp.dappnode.eth"
 ENV SHARE_DATA_WITH_DAPPNODE="true"
 ENV VALIDATORS_MONITOR_URL="http://listener.dappnode:8080"
 ENV SHARE_CRON_INTERVAL="3600"

web3signer/signer/Dockerfile

@@ -4,14 +4,6 @@ WORKDIR /app
 RUN apt update && apt install git -y && git clone https://github.com/consensys/web3signer.git
 WORKDIR /app/web3signer 
 RUN git fetch origin && ./gradlew clean assemble && tar -xzf ./build/distributions/web3signer-develop.tar.gz
-
-##############
-# WEB3SIGNER #
-##############
-
-FROM consensys/web3signer:24.2.0
-# Copy web3signer keymanager binary
-COPY --from=jre-build /app/web3signer/web3signer-develop /opt/web3signer
 COPY entrypoint.sh /usr/bin/entrypoint.sh
 
 EXPOSE 9000

web3signer/signer/entrypoint.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 
-exec /opt/web3signer/bin/web3signer --logging=DEBUG --http-listen-port=9000 \
+exec /app/web3signer/web3signer-develop/bin/web3signer --logging=DEBUG --http-listen-port=9000 \
     --http-listen-host=0.0.0.0 --http-host-allowlist="*" --http-cors-origins="*" \
     eth2 \
+    --network=holesky \
     --slashing-protection-db-url=jdbc:postgresql://postgres.web3signer.dappnode:5432/web3signer \
     --slashing-protection-db-username=postgres \
     --slashing-protection-db-password=password \