From 9cb6be48dbfbe83eea1d36c574c095c398fcb5f5 Mon Sep 17 00:00:00 2001 From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com> Date: Fri, 17 May 2024 17:34:11 +0200 Subject: [PATCH] Create cron to remove signatures with status unknown (#49) * create type network with enum * add cron remove unknown * refactor get active validators * return unknown validators * rename to update signatures status * rename to lower letter * do not update status --- listener/cmd/listener/main.go | 4 + listener/internal/api/api.go | 5 +- .../internal/api/handlers/postNewSignature.go | 113 ++++++++------- listener/internal/api/routes/routes.go | 3 +- listener/internal/api/types/types.go | 51 +++++-- .../api/validation/GetActiveValidators.go | 134 ------------------ .../validation/GetActiveValidators_test.go | 44 ------ .../api/validation/getValidatorsStatus.go | 108 ++++++++++++++ .../validation/getValidatorsStatus_test.go | 26 ++++ ...quests.go => validateAndDecodeRequests.go} | 2 +- ...t.go => validateAndDecodeRequests_test.go} | 0 ...{VerifySignature.go => verifySignature.go} | 3 +- ...nature_test.go => verifySignature_test.go} | 4 +- listener/internal/config/config.go | 13 +- .../internal/cron/updateSignaturesStatus.go | 102 +++++++++++++ ...etMongoDbClient.go => getMongoDbClient.go} | 0 16 files changed, 359 insertions(+), 253 deletions(-) delete mode 100644 listener/internal/api/validation/GetActiveValidators.go delete mode 100644 listener/internal/api/validation/GetActiveValidators_test.go create mode 100644 listener/internal/api/validation/getValidatorsStatus.go create mode 100644 listener/internal/api/validation/getValidatorsStatus_test.go rename listener/internal/api/validation/{ValidateAndDecodeRequests.go => validateAndDecodeRequests.go} (97%) rename listener/internal/api/validation/{ValidateAndDecodeRequests_test.go => validateAndDecodeRequests_test.go} (100%) rename listener/internal/api/validation/{VerifySignature.go => verifySignature.go} (91%) rename listener/internal/api/validation/{VerifySignature_test.go => verifySignature_test.go} (96%) create mode 100644 listener/internal/cron/updateSignaturesStatus.go rename listener/internal/mongodb/{GetMongoDbClient.go => getMongoDbClient.go} (100%) diff --git a/listener/cmd/listener/main.go b/listener/cmd/listener/main.go index fadac88..f9d2874 100644 --- a/listener/cmd/listener/main.go +++ b/listener/cmd/listener/main.go @@ -67,6 +67,10 @@ func main() { c.AddFunc("@daily", func() { // there are 24 * 30 = 720 hours in 30 days apiCron.RemoveOldSignatures(dbCollection, 720) + + }) + c.AddFunc("* * * * *", func() { + apiCron.UpdateSignaturesStatus(dbCollection, config.BeaconNodeURLs) }) c.Start() diff --git a/listener/internal/api/api.go b/listener/internal/api/api.go index e30e252..5b8b39c 100644 --- a/listener/internal/api/api.go +++ b/listener/internal/api/api.go @@ -6,6 +6,7 @@ import ( "net/http" "github.com/dappnode/validator-monitoring/listener/internal/api/routes" + "github.com/dappnode/validator-monitoring/listener/internal/api/types" "github.com/dappnode/validator-monitoring/listener/internal/logger" "go.mongodb.org/mongo-driver/mongo" ) @@ -15,11 +16,11 @@ type httpApi struct { port string dbClient *mongo.Client dbCollection *mongo.Collection - beaconNodeUrls map[string]string + beaconNodeUrls map[types.Network]string } // create a new api instance -func NewApi(port string, dbClient *mongo.Client, dbCollection *mongo.Collection, beaconNodeUrls map[string]string) *httpApi { +func NewApi(port string, dbClient *mongo.Client, dbCollection *mongo.Collection, beaconNodeUrls map[types.Network]string) *httpApi { return &httpApi{ port: port, dbClient: dbClient, diff --git a/listener/internal/api/handlers/postNewSignature.go b/listener/internal/api/handlers/postNewSignature.go index 02686b4..e9f8779 100644 --- a/listener/internal/api/handlers/postNewSignature.go +++ b/listener/internal/api/handlers/postNewSignature.go @@ -13,82 +13,100 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -// Posting a new singature consists in the following steps: -// 1. Decode and validate -// 2. Get active validators -// 3. Validate signature and insert into MongoDB -func PostNewSignature(w http.ResponseWriter, r *http.Request, dbCollection *mongo.Collection, beaconNodeUrls map[string]string) { +func PostNewSignature(w http.ResponseWriter, r *http.Request, dbCollection *mongo.Collection, beaconNodeUrls map[types.Network]string) { logger.Debug("Received new POST '/newSignature' request") - - // Parse request body var requests []types.SignatureRequest - err := json.NewDecoder(r.Body).Decode(&requests) - if err != nil { - logger.Error("Failed to decode request body: " + err.Error()) - respondError(w, http.StatusBadRequest, "Invalid request format") - return - } - // Decode and validate incoming requests - validRequests, err := validation.ValidateAndDecodeRequests(requests) - if err != nil { + + // Parse and validate request body + if err := json.NewDecoder(r.Body).Decode(&requests); err != nil { logger.Error("Failed to decode request body: " + err.Error()) respondError(w, http.StatusBadRequest, "Invalid request format") return } - // Respond with an error if no valid requests were found - if len(validRequests) == 0 { + + requestsValidatedAndDecoded, err := validation.ValidateAndDecodeRequests(requests) + if err != nil || len(requestsValidatedAndDecoded) == 0 { + logger.Error("Failed to validate and decode requests: " + err.Error()) respondError(w, http.StatusBadRequest, "No valid requests") return } - // Get active validators from the network, get the network from the first item in the array - beaconNodeUrl, ok := beaconNodeUrls[validRequests[0].Network] + // Check network validity + network := requestsValidatedAndDecoded[0].Network + beaconNodeUrl, ok := beaconNodeUrls[network] if !ok { respondError(w, http.StatusBadRequest, "Invalid network") return } - activeValidators, err := validation.GetActiveValidators(validRequests, beaconNodeUrl) + // Get active validators + pubkeys := getPubkeys(requestsValidatedAndDecoded) + validatorsStatusMap, err := validation.GetValidatorsStatus(pubkeys, beaconNodeUrl) if err != nil { logger.Error("Failed to get active validators: " + err.Error()) - respondError(w, http.StatusInternalServerError, "Failed to get active validators") + respondError(w, http.StatusInternalServerError, "Failed to get active validators: "+err.Error()) return } - if len(activeValidators) == 0 { - respondError(w, http.StatusInternalServerError, "No active validators found in network") + + // Filter and verify signatures + validSignatures := filterAndVerifySignatures(requestsValidatedAndDecoded, validatorsStatusMap) + if len(validSignatures) == 0 { + respondError(w, http.StatusBadRequest, "No valid signatures") + return + } + + // Insert valid signatures into MongoDB + if err := insertSignaturesIntoDB(validSignatures, dbCollection); err != nil { + logger.Error("Failed to insert signatures into MongoDB: " + err.Error()) + respondError(w, http.StatusInternalServerError, "Failed to insert signatures into MongoDB") return } - validSignatures := []types.SignatureRequestDecodedWithActive{} - for _, req := range activeValidators { - isValidSignature, err := validation.VerifySignature(req) - if err != nil { - logger.Error("Failed to validate signature: " + err.Error()) + respondOK(w, "Finished processing signatures") +} + +func getPubkeys(requests []types.SignatureRequestDecoded) []string { + pubkeys := make([]string, len(requests)) + for i, req := range requests { + pubkeys[i] = req.Pubkey + } + return pubkeys +} + +func filterAndVerifySignatures(requests []types.SignatureRequestDecoded, validatorsStatusMap map[string]types.Status) []types.SignatureRequestDecodedWithStatus { + validSignatures := []types.SignatureRequestDecodedWithStatus{} + for _, req := range requests { + status, ok := validatorsStatusMap[req.Pubkey] + if !ok { + logger.Warn("Validator not found: " + req.Pubkey) continue } - if !isValidSignature { - logger.Warn("Invalid signature: " + req.Signature) + if status == types.Inactive { + logger.Warn("Inactive validator: " + req.Pubkey) continue } - validSignatures = append(validSignatures, req) - } - // Respond with an error if no valid signatures were found - if len(validSignatures) == 0 { - respondError(w, http.StatusBadRequest, "No valid signatures") - return + reqWithStatus := types.SignatureRequestDecodedWithStatus{ + SignatureRequestDecoded: req, + Status: status, + } + if isValid, err := validation.VerifySignature(reqWithStatus); err == nil && isValid { + validSignatures = append(validSignatures, reqWithStatus) + } else { + logger.Warn("Invalid signature: " + req.Signature) + } } + return validSignatures +} - // Iterate over all valid signatures and insert the signature - for _, req := range validSignatures { +func insertSignaturesIntoDB(signatures []types.SignatureRequestDecodedWithStatus, dbCollection *mongo.Collection) error { + for _, req := range signatures { filter := bson.M{ "pubkey": req.Pubkey, "tag": req.Tag, "network": req.Network, } update := bson.M{ - "$set": bson.M{ - "status": req.Status, // Only save the last status - }, + "$setOnInsert": bson.M{"status": req.Status}, // do not update status if already exists "$push": bson.M{ "entries": bson.M{ "payload": req.Payload, @@ -96,19 +114,16 @@ func PostNewSignature(w http.ResponseWriter, r *http.Request, dbCollection *mong "decodedPayload": bson.M{ "type": req.DecodedPayload.Type, "platform": req.DecodedPayload.Platform, - "timestamp": req.DecodedPayload.Timestamp, // Needed to filter out old signatures + "timestamp": req.DecodedPayload.Timestamp, }, }, }, } options := options.Update().SetUpsert(true) - _, err := dbCollection.UpdateOne(context.TODO(), filter, update, options) - if err != nil { - logger.Error("Failed to insert signature into MongoDB: " + err.Error()) - continue + if _, err := dbCollection.UpdateOne(context.Background(), filter, update, options); err != nil { + return err } logger.Debug("New Signature " + req.Signature + " inserted into MongoDB") } - - respondOK(w, "Finished processing signatures") + return nil } diff --git a/listener/internal/api/routes/routes.go b/listener/internal/api/routes/routes.go index dda5893..40e1c9a 100644 --- a/listener/internal/api/routes/routes.go +++ b/listener/internal/api/routes/routes.go @@ -4,11 +4,12 @@ import ( "net/http" "github.com/dappnode/validator-monitoring/listener/internal/api/handlers" + "github.com/dappnode/validator-monitoring/listener/internal/api/types" "github.com/gorilla/mux" "go.mongodb.org/mongo-driver/mongo" ) -func SetupRouter(dbCollection *mongo.Collection, beaconNodeUrls map[string]string) *mux.Router { +func SetupRouter(dbCollection *mongo.Collection, beaconNodeUrls map[types.Network]string) *mux.Router { r := mux.NewRouter() // Define routes diff --git a/listener/internal/api/types/types.go b/listener/internal/api/types/types.go index a2d075d..bc665c3 100644 --- a/listener/internal/api/types/types.go +++ b/listener/internal/api/types/types.go @@ -1,11 +1,42 @@ package types +// In sync with brain +type Network string // "mainnet" | "holesky" | "gnosis" | "lukso" + +const ( + Mainnet Network = "mainnet" + Holesky Network = "holesky" + Gnosis Network = "gnosis" + Lukso Network = "lukso" +) + +// In sync with brain +// @see https://github.com/dappnode/StakingBrain/blob/0aaeefa8aec1b21ba2f2882cb444747419a3ff5d/packages/common/src/types/db/types.ts#L27 +type Tag string // "obol" | "diva" | "ssv" | "rocketpool" | "stakewise" | "stakehouse" | "solo" | "stader" + +const ( + Obol Tag = "obol" + Diva Tag = "diva" + Ssv Tag = "ssv" + Rocketpool Tag = "rocketpool" + Stakewise Tag = "stakewise" + Stakehouse Tag = "stakehouse" + Solo Tag = "solo" + Stader Tag = "stader" +) + type SignatureRequest struct { - Payload string `json:"payload"` - Pubkey string `json:"pubkey"` - Signature string `json:"signature"` - Network string `json:"network"` - Tag string `json:"tag"` + Payload string `json:"payload"` + Pubkey string `json:"pubkey"` + Signature string `json:"signature"` + Network Network `json:"network"` + Tag string `json:"tag"` +} + +type DecodedPayload struct { + Type string `json:"type"` + Platform string `json:"platform"` + Timestamp string `json:"timestamp"` } type SignatureRequestDecoded struct { @@ -19,16 +50,10 @@ type Status string const ( Unknown Status = "unknown" Active Status = "active" - Inactive Status = "inactive" + Inactive Status = "inactive" // means any response from beacon node that is not active ) -type SignatureRequestDecodedWithActive struct { +type SignatureRequestDecodedWithStatus struct { SignatureRequestDecoded Status Status `json:"status"` // "unknown" | "active" | "inactive" } - -type DecodedPayload struct { - Type string `json:"type"` - Platform string `json:"platform"` - Timestamp string `json:"timestamp"` -} diff --git a/listener/internal/api/validation/GetActiveValidators.go b/listener/internal/api/validation/GetActiveValidators.go deleted file mode 100644 index 7197c68..0000000 --- a/listener/internal/api/validation/GetActiveValidators.go +++ /dev/null @@ -1,134 +0,0 @@ -package validation - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/dappnode/validator-monitoring/listener/internal/api/types" - "github.com/dappnode/validator-monitoring/listener/internal/logger" -) - -type activeValidator struct { - Pubkey string `json:"pubkey"` - WithdrawalCredentials string `json:"withdrawal_credentials"` - EffectiveBalance string `json:"effective_balance"` - Slashed bool `json:"slashed"` - ActivationEligibilityEpoch string `json:"activation_eligibility_epoch"` - ActivationEpoch string `json:"activation_epoch"` - ExitEpoch string `json:"exit_epoch"` - WithdrawableEpoch string `json:"withdrawable_epoch"` -} - -// https://ethereum.github.io/beacon-APIs/#/Beacon /eth/v1/beacon/states/{state_id}/validators -type activeValidatorsApiResponse struct { - ExecutionOptimistic bool `json:"execution_optimistic"` - Finalized bool `json:"finalized"` - Data []struct { - Index string `json:"index"` - Balance string `json:"balance"` - Status string `json:"status"` - Validator activeValidator `json:"validator"` - } `json:"data"` -} - -// GetActiveValidators checks the active status of validators from a specific beacon node. -func GetActiveValidators(requestsDecoded []types.SignatureRequestDecoded, beaconNodeUrl string) ([]types.SignatureRequestDecodedWithActive, error) { - if len(requestsDecoded) == 0 { - logger.Warn("No requests to process to retrieve active validators") - return nil, fmt.Errorf("No requests to process to retrieve active validators") - } - - ids := make([]string, 0, len(requestsDecoded)) - for _, req := range requestsDecoded { - ids = append(ids, req.Pubkey) - } - if len(ids) == 0 { - logger.Warn("No valid public keys for network " + beaconNodeUrl + " to query") - return nil, fmt.Errorf("No valid public keys for network " + beaconNodeUrl + " to query") - } - - // Serialize the request body to JSON - // See https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidators - // returns only active validators - jsonData, err := json.Marshal(struct { - Ids []string `json:"ids"` - Statuses []string `json:"statuses"` - }{ - Ids: ids, - Statuses: []string{"active_ongoing"}, // Only interested in currently active validators - }) - if err != nil { - logger.Error("Failed to serialize request data: " + err.Error()) - return nil, err - } - - // Create HTTP client with timeout - client := &http.Client{Timeout: 10 * time.Second} - apiUrl := fmt.Sprintf("%s/eth/v1/beacon/states/head/validators", beaconNodeUrl) - - // Make API call - resp, err := client.Post(apiUrl, "application/json", bytes.NewBuffer(jsonData)) - if err != nil { - logger.Error("error making API call to " + apiUrl + ": " + err.Error()) - // if the api call fails return signatures with status unknown - return GetSignatureRequestsDecodedWithUnknown(requestsDecoded), nil - } - defer resp.Body.Close() - - // check if its any server error 5xx - // if its internal server error return unknown since we expect the cron to eventually resolve the status once the server is back up - if resp.StatusCode >= 500 && resp.StatusCode < 600 { - logger.Error("internal server error, returning signatures with status unknown: " + resp.Status) - return GetSignatureRequestsDecodedWithUnknown(requestsDecoded), nil - } - - // Check the HTTP response status before reading the body and return nil if not ok - if resp.StatusCode != http.StatusOK { - logger.Error("unexpected response status from beacon node when retrieving active validators: " + resp.Status) - return nil, fmt.Errorf("unexpected response status from beacon node when retrieving active validators: " + resp.Status) - } - - // Decode the API response directly into the ApiResponse struct - var apiResponse activeValidatorsApiResponse - if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil { - logger.Error("error decoding response data from beacon node: " + err.Error()) - return nil, err - } - - // Use a map to quickly lookup active validators - activeValidatorMap := make(map[string]bool) - for _, validator := range apiResponse.Data { - activeValidatorMap[validator.Validator.Pubkey] = true - } - - // Filter the list of decoded requests to include only those that are active - var activeValidators []types.SignatureRequestDecodedWithActive - for _, req := range requestsDecoded { - if _, isActive := activeValidatorMap[req.Pubkey]; isActive { - activeValidators = append(activeValidators, types.SignatureRequestDecodedWithActive{ - SignatureRequestDecoded: req, - Status: types.Active, - }) - } else { - // do not append inactive validators - logger.Warn("Inactive validator: " + req.Pubkey) - } - } - - return activeValidators, nil -} - -// Append "unknown" status to all requests -func GetSignatureRequestsDecodedWithUnknown(requests []types.SignatureRequestDecoded) []types.SignatureRequestDecodedWithActive { - var signatureRequestsDecodedWithActive []types.SignatureRequestDecodedWithActive - for _, req := range requests { - signatureRequestsDecodedWithActive = append(signatureRequestsDecodedWithActive, types.SignatureRequestDecodedWithActive{ - SignatureRequestDecoded: req, - Status: types.Unknown, - }) - } - return signatureRequestsDecodedWithActive -} diff --git a/listener/internal/api/validation/GetActiveValidators_test.go b/listener/internal/api/validation/GetActiveValidators_test.go deleted file mode 100644 index 1a091b1..0000000 --- a/listener/internal/api/validation/GetActiveValidators_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package validation - -import ( - "testing" - - "github.com/dappnode/validator-monitoring/listener/internal/api/types" -) - -func TestGetActiveValidators(t *testing.T) { - // Setup the input data - beaconNodeUrls := map[string]string{ - "holesky": "https://holeskyvals.53650f79ab75c6ff.dyndns.dappnode.io", - } - - requestsDecoded := []types.SignatureRequestDecoded{ - { - SignatureRequest: types.SignatureRequest{ - Network: "holesky", - Pubkey: "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca", - }, - DecodedPayload: types.DecodedPayload{}, - }, - { - SignatureRequest: types.SignatureRequest{ - Network: "holesky", - Pubkey: "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63"}, - DecodedPayload: types.DecodedPayload{}, - }, - { - SignatureRequest: types.SignatureRequest{ - Network: "holesky", - Pubkey: "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b"}, - DecodedPayload: types.DecodedPayload{}, - }, - } - - result, _ := GetActiveValidators(requestsDecoded, beaconNodeUrls["holesky"]) - - // You may need to mock the server's response or adjust the expected values here according to your actual setup - expectedNumValidators := 3 // This should match the number of mock validators that are "active" - if len(result) != expectedNumValidators { - t.Errorf("Expected %d active validators, got %d", expectedNumValidators, len(result)) - } -} diff --git a/listener/internal/api/validation/getValidatorsStatus.go b/listener/internal/api/validation/getValidatorsStatus.go new file mode 100644 index 0000000..8350de6 --- /dev/null +++ b/listener/internal/api/validation/getValidatorsStatus.go @@ -0,0 +1,108 @@ +package validation + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/dappnode/validator-monitoring/listener/internal/api/types" + "github.com/dappnode/validator-monitoring/listener/internal/logger" +) + +type activeValidator struct { + Pubkey string `json:"pubkey"` + WithdrawalCredentials string `json:"withdrawal_credentials"` + EffectiveBalance string `json:"effective_balance"` + Slashed bool `json:"slashed"` + ActivationEligibilityEpoch string `json:"activation_eligibility_epoch"` + ActivationEpoch string `json:"activation_epoch"` + ExitEpoch string `json:"exit_epoch"` + WithdrawableEpoch string `json:"withdrawable_epoch"` +} + +// https://ethereum.github.io/beacon-APIs/#/Beacon /eth/v1/beacon/states/{state_id}/validators +type activeValidatorsApiResponse struct { + ExecutionOptimistic bool `json:"execution_optimistic"` + Finalized bool `json:"finalized"` + Data []struct { + Index string `json:"index"` + Balance string `json:"balance"` + Status string `json:"status"` + Validator activeValidator `json:"validator"` + } `json:"data"` +} + +// GetValidatorsStatus checks the active status of validators from a specific beacon node. +// @returns validatorStatusMap error +func GetValidatorsStatus(pubkeys []string, beaconNodeUrl string) (map[string]types.Status, error) { + if len(pubkeys) == 0 { + logger.Warn("No public keys provided to retrieve active validators") + return nil, fmt.Errorf("no public keys provided to retrieve active validators from beacon node") + } + + // Use a map to store validator statuses + statusMap := make(map[string]types.Status) + + // Serialize the request body to JSON + // See https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidators + // returns only active validators + jsonData, err := json.Marshal(struct { + Ids []string `json:"ids"` + Statuses []string `json:"statuses"` + }{ + Ids: pubkeys, + Statuses: []string{"active_ongoing"}, // Only interested in currently active validators + }) + if err != nil { + logger.Error("Failed to serialize request data: " + err.Error()) + return nil, err + } + + // Create HTTP client with timeout + client := &http.Client{Timeout: 10 * time.Second} + apiUrl := fmt.Sprintf("%s/eth/v1/beacon/states/head/validators", beaconNodeUrl) + + // Make API call + resp, err := client.Post(apiUrl, "application/json", bytes.NewBuffer(jsonData)) + if err != nil { + logger.Error("Failed to make request to beacon node: " + err.Error()) + return getMapUnknown(pubkeys), nil + } + defer resp.Body.Close() + + // Check if it's any server error 5xx + if resp.StatusCode >= 500 && resp.StatusCode < 600 { + logger.Error("internal server error due to server issue, keeping signatures to be stored with status unknown: " + resp.Status) + return getMapUnknown(pubkeys), nil + } + + // Check the HTTP response status before reading the body and return nil if not ok + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected response status from beacon node when retrieving active validators: " + resp.Status) + } + + // Decode the API response directly into the ApiResponse struct + var apiResponse activeValidatorsApiResponse + if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil { + return nil, fmt.Errorf("error decoding response data from beacon node: " + err.Error()) + } + + for _, pubkey := range pubkeys { + statusMap[pubkey] = types.Inactive + } + for _, validator := range apiResponse.Data { + statusMap[validator.Validator.Pubkey] = types.Active + } + + return statusMap, nil +} + +func getMapUnknown(pubkeys []string) map[string]types.Status { + statusMap := make(map[string]types.Status) + for _, pubkey := range pubkeys { + statusMap[pubkey] = types.Unknown + } + return statusMap +} diff --git a/listener/internal/api/validation/getValidatorsStatus_test.go b/listener/internal/api/validation/getValidatorsStatus_test.go new file mode 100644 index 0000000..f5c3783 --- /dev/null +++ b/listener/internal/api/validation/getValidatorsStatus_test.go @@ -0,0 +1,26 @@ +package validation + +import ( + "testing" +) + +func TestGetActiveValidators(t *testing.T) { + // Setup the input data + beaconNodeUrls := map[string]string{ + "holesky": "https://holeskyvals.53650f79ab75c6ff.dyndns.dappnode.io", + } + + requestsDecoded := []string{ + "0xa685beb5a1f317f5a01ecd6dade42113aad945b2ab53fb1b356334ab441323e538feadd2889894b17f8fa2babe1989ca", + "0xab31efdd97f32087e96d3262f6fb84a4480411d391689be0dfc931fd8a5c16c3f51f10b127040b1cb65eb955f2b78a63", + "0xa24a030d7d8ca3c5e1f5824760d0f4157a7a89bcca6414377cca97e6e63445bef0e1b63761ee35a0fc46bb317e31b34b", + } + + validatorsStatusMap, _ := GetValidatorsStatus(requestsDecoded, beaconNodeUrls["holesky"]) + + // You may need to mock the server's response or adjust the expected values here according to your actual setup + expectedNumValidators := 3 // This should match the number of mock validators that are "active" + if len(validatorsStatusMap) != expectedNumValidators { + t.Errorf("Expected %d active validators, got %d", expectedNumValidators, len(validatorsStatusMap)) + } +} diff --git a/listener/internal/api/validation/ValidateAndDecodeRequests.go b/listener/internal/api/validation/validateAndDecodeRequests.go similarity index 97% rename from listener/internal/api/validation/ValidateAndDecodeRequests.go rename to listener/internal/api/validation/validateAndDecodeRequests.go index 19dbc3c..6de991e 100644 --- a/listener/internal/api/validation/ValidateAndDecodeRequests.go +++ b/listener/internal/api/validation/validateAndDecodeRequests.go @@ -40,7 +40,7 @@ func ValidateAndDecodeRequests(requests []types.SignatureRequest) ([]types.Signa } // isValidCodedRequest checks if the request has all the required fields, the correct signature format, and a valid BLS pubkey -// TODO: we should consider having an enum for Network and Tag fields and validate them as well. +// TODO: validate network and tag against enums func isValidCodedRequest(req *types.SignatureRequest) bool { // Check for any empty required fields if req.Network == "" || req.Tag == "" || req.Signature == "" || req.Payload == "" || req.Pubkey == "" { diff --git a/listener/internal/api/validation/ValidateAndDecodeRequests_test.go b/listener/internal/api/validation/validateAndDecodeRequests_test.go similarity index 100% rename from listener/internal/api/validation/ValidateAndDecodeRequests_test.go rename to listener/internal/api/validation/validateAndDecodeRequests_test.go diff --git a/listener/internal/api/validation/VerifySignature.go b/listener/internal/api/validation/verifySignature.go similarity index 91% rename from listener/internal/api/validation/VerifySignature.go rename to listener/internal/api/validation/verifySignature.go index c67c086..1ba5fa0 100644 --- a/listener/internal/api/validation/VerifySignature.go +++ b/listener/internal/api/validation/verifySignature.go @@ -10,7 +10,8 @@ import ( "github.com/herumi/bls-eth-go-binary/bls" ) -func VerifySignature(req types.SignatureRequestDecodedWithActive) (bool, error) { +// TODO: this function shoul take as arg only the required inputs and not the full request +func VerifySignature(req types.SignatureRequestDecodedWithStatus) (bool, error) { // Decode the public key from hex, remove the 0x prefix ONLY if exists from req.Pubkey req.Pubkey = strings.TrimPrefix(req.Pubkey, "0x") req.Pubkey = strings.TrimSpace(req.Pubkey) diff --git a/listener/internal/api/validation/VerifySignature_test.go b/listener/internal/api/validation/verifySignature_test.go similarity index 96% rename from listener/internal/api/validation/VerifySignature_test.go rename to listener/internal/api/validation/verifySignature_test.go index 7416d3b..efcc120 100644 --- a/listener/internal/api/validation/VerifySignature_test.go +++ b/listener/internal/api/validation/verifySignature_test.go @@ -36,7 +36,7 @@ func TestVerifySignature(t *testing.T) { signature := secretKey.SignByte(messageBytes) // Prepare the request - req := types.SignatureRequestDecodedWithActive{ + req := types.SignatureRequestDecodedWithStatus{ SignatureRequestDecoded: types.SignatureRequestDecoded{ DecodedPayload: decodedPayload, SignatureRequest: types.SignatureRequest{ @@ -82,7 +82,7 @@ func TestVerifySignatureError(t *testing.T) { } // Create the SignatureRequestDecoded with a bad signature to ensure it fails - req := types.SignatureRequestDecodedWithActive{ + req := types.SignatureRequestDecodedWithStatus{ SignatureRequestDecoded: types.SignatureRequestDecoded{ DecodedPayload: decodedPayload, SignatureRequest: types.SignatureRequest{ diff --git a/listener/internal/config/config.go b/listener/internal/config/config.go index a5fb764..f0d68e2 100644 --- a/listener/internal/config/config.go +++ b/listener/internal/config/config.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/dappnode/validator-monitoring/listener/internal/api/types" "github.com/dappnode/validator-monitoring/listener/internal/logger" ) @@ -16,7 +17,7 @@ type Config struct { // LogLevel is the level of logging LogLevel string // BeaconNodeURLs is the URLs of the beacon nodes for different networks - BeaconNodeURLs map[string]string + BeaconNodeURLs map[types.Network]string } func LoadConfig() (*Config, error) { @@ -58,11 +59,11 @@ func LoadConfig() (*Config, error) { // print all envs in a single line logger.Info("Loaded config: LOG_LEVEL=" + logLevel + " API_PORT=" + apiPort + " MONGO_DB_URI=" + mongoDBURI + " BEACON_NODE_URL_MAINNET=" + beaconMainnet + " BEACON_NODE_URL_HOLESKY=" + beaconHolesky + " BEACON_NODE_URL_GNOSIS=" + beaconGnosis + " BEACON_NODE_URL_LUKSO=" + beaconLukso) - beaconNodeURLs := map[string]string{ - "mainnet": beaconMainnet, - "holesky": beaconHolesky, - "gnosis": beaconGnosis, - "lukso": beaconLukso, + beaconNodeURLs := map[types.Network]string{ + types.Mainnet: beaconMainnet, + types.Holesky: beaconHolesky, + types.Gnosis: beaconGnosis, + types.Lukso: beaconLukso, } return &Config{ diff --git a/listener/internal/cron/updateSignaturesStatus.go b/listener/internal/cron/updateSignaturesStatus.go new file mode 100644 index 0000000..d580d1b --- /dev/null +++ b/listener/internal/cron/updateSignaturesStatus.go @@ -0,0 +1,102 @@ +package cron + +import ( + "context" + + "github.com/dappnode/validator-monitoring/listener/internal/api/types" + "github.com/dappnode/validator-monitoring/listener/internal/api/validation" + "github.com/dappnode/validator-monitoring/listener/internal/logger" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func UpdateSignaturesStatus(collection *mongo.Collection, beaconNodeUrls map[types.Network]string) { + logger.Debug("Updating statuses and removing inactive signatures") + + // Step 1: Query the MongoDB collection to retrieve all documents with status "unknown" + filter := bson.M{"status": types.Unknown} + projection := bson.M{ + "pubkey": 1, + "tag": 1, + "network": 1, + } + cursor, err := collection.Find(context.Background(), filter, options.Find().SetProjection(projection)) + if err != nil { + logger.Error("Failed to query MongoDB collection: " + err.Error()) + return + } + defer cursor.Close(context.Background()) + + // Step 2: Extract pubkeys, tags, and networks from the documents + type PubkeyTagNetwork struct { + Pubkey string + Tag types.Tag + Network types.Network + } + var pubkeyTagNetworkPairs []PubkeyTagNetwork + for cursor.Next(context.Background()) { + var signature PubkeyTagNetwork + if err := cursor.Decode(&signature); err != nil { + logger.Error("Failed to decode MongoDB document: " + err.Error()) + continue + + } + pubkeyTagNetworkPairs = append(pubkeyTagNetworkPairs, signature) + } + + if err := cursor.Err(); err != nil { + logger.Error("Failed to iterate over MongoDB cursor: " + err.Error()) + return + } + + // Step 3: Query GetValidatorsStatus using these pubkeys and the corresponding beacon node URL + pubkeyStatusMap := make(map[string]types.Status) + for network, url := range beaconNodeUrls { + var networkPubkeys []string + for _, pair := range pubkeyTagNetworkPairs { + if pair.Network == network { + networkPubkeys = append(networkPubkeys, pair.Pubkey) + } + } + if len(networkPubkeys) > 0 { + statusMap, err := validation.GetValidatorsStatus(networkPubkeys, url) + if err != nil { + logger.Error("Failed to get active validators: " + err.Error()) + continue + } + for pubkey, status := range statusMap { + pubkeyStatusMap[pubkey] = status + } + } + } + + // Step 4: Update or remove documents based on the validator status + for _, pair := range pubkeyTagNetworkPairs { + status, exists := pubkeyStatusMap[pair.Pubkey] + if !exists { + continue + } + + if status == types.Active { + // Update the status to "active" + update := bson.M{ + "$set": bson.M{"status": types.Active}, + } + _, err := collection.UpdateOne(context.Background(), bson.M{"pubkey": pair.Pubkey, "tag": pair.Tag, "network": pair.Network, "status": types.Unknown}, update) + if err != nil { + logger.Error("Failed to update signature: " + err.Error()) + continue + } + logger.Debug("Updated signature with pubkey " + pair.Pubkey + " to active") + } else if status == types.Inactive { + // Remove the signature + _, err := collection.DeleteOne(context.Background(), bson.M{"pubkey": pair.Pubkey, "tag": pair.Tag, "network": pair.Network, "status": types.Unknown}) + if err != nil { + logger.Error("Failed to remove signature: " + err.Error()) + continue + } + logger.Debug("Removed signature with pubkey " + pair.Pubkey + " due to inactive validator status") + } + } +} diff --git a/listener/internal/mongodb/GetMongoDbClient.go b/listener/internal/mongodb/getMongoDbClient.go similarity index 100% rename from listener/internal/mongodb/GetMongoDbClient.go rename to listener/internal/mongodb/getMongoDbClient.go