statesync: populate parameters dynamically

 * flags vochainStateSyncTrustHash and vochainStateSyncTrustHeight take precedence
 * new flag vochainStateSyncFetchParamsFromAPI allows passing a URL to fetch params dinamically
 * if vochainStateSyncFetchParamsFromAPI is empty, parameters will be fetch from default URL
 * to skip this feature, set --vochainStateSyncFetchParamsFromAPI=disabled
 * as a fallback, added some hardcoded height+hash on genesis.go, with least precedence

apiclient:
 * added var DefaultAPIUrls
 * added funcs FetchChainHeightAndHash and ...FromDefaultAPI
 * added method Block()

testsuite:
 * replace brittle hack to populate HASH and HEIGHT with new vochainStateSyncFetchParamsFromAPI
 * hotfix: make the test actually fail when "Timed out waiting!"
 * allow passing `BUILD=0 start_test.sh` to skip `docker compose build`

apiclient/blockchain.go

@@ -35,6 +35,23 @@ func (c *HTTPclient) ChainInfo() (*api.ChainInfo, error) {
 	return chainInfo, nil
 }
 
+// Block returns information about a block, given a height.
+func (c *HTTPclient) Block(height uint32) (*api.Block, error) {
+	resp, code, err := c.Request(HTTPGET, nil, "chain", "blocks", fmt.Sprintf("%d", height))
+	if err != nil {
+		return nil, err
+	}
+	if code != apirest.HTTPstatusOK {
+		return nil, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp)
+	}
+	block := &api.Block{}
+	err = json.Unmarshal(resp, block)
+	if err != nil {
+		return nil, err
+	}
+	return block, nil
+}
+
 // TransactionsCost returns a map with the current cost for all transactions
 func (c *HTTPclient) TransactionsCost() (map[string]uint64, error) {
 	resp, code, err := c.Request(HTTPGET, nil, "chain", "transactions", "cost")

apiclient/client.go

@@ -36,6 +36,16 @@ const (
 	DefaultTimeout = 10 * time.Second
 )
 
+// DefaultAPIUrls is a map of default API URLs for each network.
+var DefaultAPIUrls = map[string]string{
+	"dev":     "https://api-dev.vocdoni.net/v2/",
+	"develop": "https://api-dev.vocdoni.net/v2/",
+	"stg":     "https://api-stg.vocdoni.net/v2/",
+	"stage":   "https://api-stg.vocdoni.net/v2/",
+	"lts":     "https://api.vocdoni.io/v2/",
+	"prod":    "https://api.vocdoni.io/v2/",
+}
+
 // HTTPclient is the Vocdoni API HTTP client.
 type HTTPclient struct {
 	c       *http.Client

apiclient/helpers.go

@@ -337,3 +337,35 @@ func UnmarshalFaucetPackage(data []byte) (*models.FaucetPackage, error) {
 		Signature: fpackage.Signature,
 	}, nil
 }
+
+// FetchChainHeightAndHashFromDefaultAPI returns the current chain height and block hash
+// from the default API url for the given network.
+func FetchChainHeightAndHashFromDefaultAPI(network string) (int64, types.HexBytes, error) {
+	apiURL, ok := DefaultAPIUrls[network]
+	if !ok {
+		return 0, nil, fmt.Errorf("no default API URL for network %s", network)
+	}
+	height, hash, err := FetchChainHeightAndHash(apiURL)
+	if err != nil {
+		return 0, nil, fmt.Errorf("couldn't fetch info: %w", err)
+	}
+	return height, hash, nil
+}
+
+// FetchChainHeightAndHash returns current chain height and block hash from an API endpoint.
+func FetchChainHeightAndHash(apiURL string) (int64, types.HexBytes, error) {
+	log.Infow("requesting chain height and hash", "url", apiURL)
+	c, err := New(apiURL)
+	if err != nil {
+		return 0, nil, fmt.Errorf("couldn't init apiclient: %w", err)
+	}
+	chain, err := c.ChainInfo()
+	if err != nil {
+		return 0, nil, fmt.Errorf("couldn't fetch chain info: %w", err)
+	}
+	block, err := c.Block(chain.Height)
+	if err != nil {
+		return 0, nil, fmt.Errorf("couldn't fetch block: %w", err)
+	}
+	return int64(chain.Height), block.Hash, nil
+}

cmd/node/main.go

@@ -24,6 +24,7 @@ import (
 	urlapi "go.vocdoni.io/dvote/api"
 	"go.vocdoni.io/dvote/api/censusdb"
 	"go.vocdoni.io/dvote/api/faucet"
+	"go.vocdoni.io/dvote/apiclient"
 	"go.vocdoni.io/dvote/config"
 	"go.vocdoni.io/dvote/crypto/ethereum"
 	"go.vocdoni.io/dvote/crypto/zk/circuit"
@@ -194,11 +195,13 @@ func loadConfig() *config.Config {
 	flag.StringSlice("vochainStateSyncRPCServers", []string{},
 		"list of RPC servers to bootstrap the StateSync (optional, defaults to using seeds)")
 	flag.String("vochainStateSyncTrustHash", "",
-		"hash of the trusted block (required if vochainStateSyncEnabled)")
+		"hash of the trusted block (takes precedence over API URL and hardcoded defaults)")
 	flag.Int64("vochainStateSyncTrustHeight", 0,
-		"height of the trusted block (required if vochainStateSyncEnabled)")
+		"height of the trusted block (takes precedence over API URL and hardcoded defaults)")
 	flag.Int64("vochainStateSyncChunkSize", 10*(1<<20), // 10 MB
 		"cometBFT chunk size in bytes")
+	flag.String("vochainStateSyncFetchParamsFromAPI", "",
+		"API URL to fetch needed params from (by default, it will use hardcoded URLs, set to 'disabled' to skip this feature)")
 
 	flag.Int("vochainMinerTargetBlockTimeSeconds", 10,
 		"vochain consensus block time target (in seconds)")
@@ -438,6 +441,42 @@ func main() {
 		}
 	}
 
+	// If StateSync is enabled but parameters are empty, try our best to populate them
+	// (cmdline flags take precedence if defined, of course)
+	if conf.Vochain.StateSyncEnabled &&
+		conf.Vochain.StateSyncTrustHeight == 0 && conf.Vochain.StateSyncTrustHash == "" {
+		conf.Vochain.StateSyncTrustHeight, conf.Vochain.StateSyncTrustHash = func() (int64, string) {
+			// first try to fetch params from remote API endpoint
+			switch strings.ToLower(conf.Vochain.StateSyncFetchParamsFromAPI) {
+			case "disabled":
+				// magic keyword to skip this feature, do nothing
+			case "":
+				height, hash, err := apiclient.FetchChainHeightAndHashFromDefaultAPI(conf.Vochain.Network)
+				if err != nil {
+					log.Warnw("couldn't fetch current state sync params", "err", err)
+				} else {
+					return height, hash.String()
+				}
+			default:
+				height, hash, err := apiclient.FetchChainHeightAndHash(conf.Vochain.StateSyncFetchParamsFromAPI)
+				if err != nil {
+					log.Warnw("couldn't fetch current state sync params", "err", err)
+				} else {
+					return height, hash.String()
+				}
+			}
+			// else, fallback to hardcoded params, if defined for the current network & chainID
+			if g, ok := genesis.Genesis[conf.Vochain.Network]; ok {
+				if statesync, ok := g.StateSync[g.Genesis.ChainID]; ok {
+					return statesync.TrustHeight, statesync.TrustHash.String()
+				}
+			}
+			return 0, ""
+		}()
+		log.Infow("automatically determined statesync params",
+			"height", conf.Vochain.StateSyncTrustHeight, "hash", conf.Vochain.StateSyncTrustHash)
+	}
+
 	//
 	// Vochain and Indexer
 	//

config/config.go

@@ -140,6 +140,9 @@ type VochainCfg struct {
 	StateSyncTrustHash string
 	// StateSyncChunkSize defines the size of the chunks when splitting a Snapshot for sending via StateSync
 	StateSyncChunkSize int64
+	// StateSyncFetchParamsFromAPI defines an API URL to fetch the params from.
+	// If empty it will use default API urls, special keyword "disable" means to skip this feature altogether
+	StateSyncFetchParamsFromAPI string
 }
 
 // IndexerCfg handles the configuration options of the indexer

dockerfiles/testsuite/docker-compose.yml

@@ -174,10 +174,9 @@ services:
       - GOCOVERDIR=/app/run/gocoverage
       - LOG_PANIC_ON_INVALIDCHARS
       - VOCDONI_LOGLEVEL=debug
-      - VOCDONI_VOCHAIN_STATESYNCENABLED=True
+      - VOCDONI_VOCHAIN_STATESYNCENABLED=true
       - VOCDONI_VOCHAIN_STATESYNCRPCSERVERS=miner0:26657,miner0:26657
-      - VOCDONI_VOCHAIN_STATESYNCTRUSTHEIGHT
-      - VOCDONI_VOCHAIN_STATESYNCTRUSTHASH
+      - VOCDONI_VOCHAIN_STATESYNCFETCHPARAMSFROMAPI
     profiles:
       - statesync
 

dockerfiles/testsuite/start_test.sh

@@ -23,6 +23,7 @@ COMPOSE_CMD_RUN="$COMPOSE_CMD run"
 
 ELECTION_SIZE=${TESTSUITE_ELECTION_SIZE:-30}
 ELECTION_SIZE_ANON=${TESTSUITE_ELECTION_SIZE_ANON:-8}
+BUILD=${BUILD:-1}
 CLEAN=${CLEAN:-1}
 LOGLEVEL=${LOGLEVEL:-debug}
 CONCURRENT=${CONCURRENT:-1}
@@ -159,18 +160,7 @@ e2etest_ballotelection() {
 }
 
 test_statesync() {
-	HEIGHT=3
-	HASH=
-
-	log "### Waiting for height $HEIGHT ###"
-	for i in {1..20}; do
-		# very brittle hack to extract the hash without using jq, to avoid dependencies
-		HASH=$($COMPOSE_CMD_RUN test curl -s --fail $APIHOST/chain/blocks/$HEIGHT 2>/dev/null | grep -oP '"hash":"\K[^"]+' | head -1)
-		[ -n "$HASH" ] && break || sleep 2
-	done
-
-	export VOCDONI_VOCHAIN_STATESYNCTRUSTHEIGHT=$HEIGHT
-	export VOCDONI_VOCHAIN_STATESYNCTRUSTHASH=$HASH
+	export VOCDONI_VOCHAIN_STATESYNCFETCHPARAMSFROMAPI=$APIHOST
 	$COMPOSE_CMD --profile statesync up gatewaySync -d
 	# watch logs for 2 minutes, until catching 'startup complete'. in case of timeout, or panic, or whatever, test will fail
 	timeout 120 sh -c "($COMPOSE_CMD logs gatewaySync -f | grep -m 1 'startup complete')"
@@ -179,8 +169,10 @@ test_statesync() {
 ### end tests definition
 
 log "### Starting test suite ###"
-$COMPOSE_CMD build
-$COMPOSE_CMD build test
+[ $BUILD -eq 1 ] && {
+	$COMPOSE_CMD build
+	$COMPOSE_CMD build test
+}
 $COMPOSE_CMD up -d seed # start the seed first so the nodes can properly bootstrap
 sleep 10
 $COMPOSE_CMD up -d
@@ -204,6 +196,7 @@ if [ $i -eq 30 ] ; then
 	log "### Timed out waiting! Abort, don't even try running tests ###"
 	tests_to_run=()
 	GOCOVERDIR=
+	RET=30
 else
 	log "### Test suite ready ###"
 fi

vochain/genesis/genesis.go

@@ -14,6 +14,12 @@ var Genesis = map[string]Vochain{
 		SeedNodes: []string{
 			"[email protected]:26656",
 		},
+		StateSync: map[string]StateSyncParams{
+			"vocdoni/DEV/32": {
+				TrustHeight: 10000,
+				TrustHash:   types.HexStringToHexBytes("0x2b430478c7867dc078c0380b81838d75358db7c8b65bfaf84ade85448a0abd54"),
+			},
+		},
 		Genesis: &devGenesis,
 	},
 
@@ -23,6 +29,12 @@ var Genesis = map[string]Vochain{
 		SeedNodes: []string{
 			"[email protected]:26656",
 		},
+		StateSync: map[string]StateSyncParams{
+			"Vocdoni/STAGE/11": {
+				TrustHeight: 150000,
+				TrustHash:   types.HexStringToHexBytes("0xd964cd5ec4704d3b3e1864c174edd1331044926bb2e6d3fe0b239b1c59329ff2"),
+			},
+		},
 		Genesis: &stageGenesis,
 	},
 
@@ -33,6 +45,12 @@ var Genesis = map[string]Vochain{
 			"[email protected]:26656",
 			"[email protected]:26656",
 		},
+		StateSync: map[string]StateSyncParams{
+			"Vocdoni/LTS/1.2": {
+				TrustHeight: 1000000,
+				TrustHash:   types.HexStringToHexBytes("0xd782c4a8e889a12fb326dd7f098336756f4238169a603501ae4a2b2f88c19db9"),
+			},
+		},
 		Genesis: &ltsGenesis,
 	},
 }

vochain/genesis/types.go

@@ -15,6 +15,7 @@ import (
 type Vochain struct {
 	AutoUpdateGenesis bool
 	SeedNodes         []string
+	StateSync         map[string]StateSyncParams
 	Genesis           *Doc
 }
 
@@ -132,6 +133,12 @@ type AppStateValidators struct {
 	KeyIndex uint8          `json:"key_index"`
 }
 
+// StateSyncParams define the parameters used by StateSync
+type StateSyncParams struct {
+	TrustHeight int64
+	TrustHash   types.HexBytes
+}
+
 // StringifiedInt64 is a wrapper around int64 that marshals/unmarshals as a string.
 // This is a dirty non-sense workaround. Blame Tendermint not me.
 // For some (unknown) reason Tendermint requires the integer values to be strings in