e2etest: general refactor

e2etest: refactor operations structure (VochainTest interface)

* implement VochainTest interface to separate Setup, Run and Teardown
* populate ops var (now a map[string]operation) in init() of each file
* rename operations
 * anonvoting -> anonelection
 * tokentransactions -> tokentxs
 * vtest -> plaintextelection
* new test encryptedelection

* apiclient: add SecretUntilTheEnd support to apiclient.Vote()
  * New methods WaitUntilElectionKeys, ElectionKeys,
    prepareVoteEnvelope, prepareVotePackageBytes

* ci: update tests_to_run
 * add e2etest_plaintextelection
 * add e2etest_encryptedelection
 * run new end2endtests concurrently
 * drop legacy vochaintests

* fix: tokentxs WaitUntilNextBlock before checking resulting state

.github/workflows/main.yml

@@ -141,6 +141,7 @@ jobs:
           COMPOSE_HOST_PATH: ${{ github.workspace }}/dockerfiles/testsuite
           LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch)
           GOCOVERDIR: "./gocoverage/" # collect code coverage when running binaries
+          CONCURRENT: 1 # run all the start_test.sh tests concurrently
         run: |
           cd dockerfiles/testsuite && ./start_test.sh
       - name: Send integration test coverage to coveralls.io

apiclient/election.go

@@ -396,3 +396,20 @@ func (c *HTTPclient) ElectionFilterPaginated(organizationId types.HexBytes,
 	}
 	return &elections, nil
 }
+
+// ElectionKeys fetches the encryption keys for an election.
+// Note that only elections that are SecretUntilTheEnd will return keys
+func (c *HTTPclient) ElectionKeys(electionID types.HexBytes) (*api.ElectionKeys, error) {
+	resp, code, err := c.Request("GET", nil, "elections", electionID.String(), "keys")
+	if err != nil {
+		return nil, err
+	}
+	if code != 200 {
+		return nil, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp)
+	}
+	electionKeys := &api.ElectionKeys{}
+	if err = json.Unmarshal(resp, &electionKeys); err != nil {
+		return nil, fmt.Errorf("could not unmarshal response: %w", err)
+	}
+	return electionKeys, nil
+}

apiclient/helpers.go

@@ -18,7 +18,11 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-const pollInterval = 4 * time.Second
+const (
+	TimeBetweenBlocks = 6 * time.Second
+	WaitTimeout       = 3 * TimeBetweenBlocks
+	PollInterval      = TimeBetweenBlocks / 6
+)
 
 func (c *HTTPclient) DateToHeight(date time.Time) (uint32, error) {
 	resp, code, err := c.Request(HTTPGET, nil, "chain", "dateToBlock", fmt.Sprintf("%d", date.Unix()))
@@ -67,7 +71,7 @@ func (c *HTTPclient) WaitUntilNBlocks(ctx context.Context, n uint32) {
 		info, err := c.ChainInfo()
 		if err != nil {
 			log.Error(err)
-			time.Sleep(pollInterval)
+			time.Sleep(PollInterval)
 			continue
 		}
 		c.WaitUntilHeight(ctx, info.Height+n)
@@ -92,7 +96,7 @@ func (c *HTTPclient) WaitUntilHeight(ctx context.Context, height uint32) error {
 			return nil
 		}
 		select {
-		case <-time.After(pollInterval):
+		case <-time.After(PollInterval):
 			continue
 		case <-ctx.Done():
 			return ctx.Err()
@@ -137,7 +141,7 @@ func (c *HTTPclient) WaitUntilElectionStatus(ctx context.Context,
 			return election, nil
 		}
 		select {
-		case <-time.After(pollInterval):
+		case <-time.After(PollInterval):
 			continue
 		case <-ctx.Done():
 			return nil, fmt.Errorf("election %v never reached status %s: %w", electionID, status, ctx.Err())
@@ -154,14 +158,33 @@ func (c *HTTPclient) WaitUntilTxIsMined(ctx context.Context,
 			return tr, nil
 		}
 		select {
-		case <-time.After(pollInterval):
+		case <-time.After(PollInterval):
 			continue
 		case <-ctx.Done():
 			return nil, ctx.Err()
 		}
 	}
 }
 
+// WaitUntilElectionKeys waits until the election has published its encryption keys,
+// and returns them.
+func (c *HTTPclient) WaitUntilElectionKeys(ctx context.Context, electionID types.HexBytes) (
+	*api.ElectionKeys, error) {
+	log.Infof("waiting for election %s to publish keys...", electionID)
+	for {
+		ek, err := c.ElectionKeys(electionID)
+		if err == nil {
+			return ek, nil
+		}
+		select {
+		case <-time.After(PollInterval):
+			continue
+		case <-ctx.Done():
+			return nil, fmt.Errorf("election %s keys not yet published: %w", electionID, ctx.Err())
+		}
+	}
+}
+
 // GetFaucetPackageFromDevService returns a faucet package.
 // Needs just the destination wallet address, the URL and bearer token are hardcoded
 func GetFaucetPackageFromDevService(account string) (*models.FaucetPackage, error) {

apiclient/vote.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 
 	"go.vocdoni.io/dvote/api"
+	"go.vocdoni.io/dvote/crypto/nacl"
 	"go.vocdoni.io/dvote/crypto/zk"
 	"go.vocdoni.io/dvote/crypto/zk/circuit"
 	"go.vocdoni.io/dvote/crypto/zk/prover"
@@ -45,21 +46,12 @@ type VoteData struct {
 // which contains the electionID, the choices and the proof. The
 // return value is the voteID (nullifier).
 func (c *HTTPclient) Vote(v *VoteData) (types.HexBytes, error) {
-	votePackage := &vochain.VotePackage{
-		Votes: v.Choices,
-	}
-	votePackageBytes, err := json.Marshal(votePackage)
+	election, err := c.Election(v.ElectionID)
 	if err != nil {
 		return nil, err
 	}
-	vote := &models.VoteEnvelope{
-		Nonce:       util.RandomBytes(16),
-		ProcessId:   v.ElectionID,
-		VotePackage: votePackageBytes,
-	}
 
-	// Get de election metadata
-	election, err := c.Election(v.ElectionID)
+	vote, err := c.prepareVoteEnvelope(v.Choices, election)
 	if err != nil {
 		return nil, err
 	}
@@ -188,6 +180,80 @@ func (c *HTTPclient) Verify(electionID, voteID types.HexBytes) (bool, error) {
 	return false, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp)
 }
 
+// prepareVoteEnvelope returns a models.VoteEnvelope struct with
+// * a random Nonce
+// * ProcessID set to the passed election
+// * VotePackage with a plaintext or encrypted vochain.VotePackage
+// * EncryptionKeyIndexes filled in, for encrypted votes
+func (c *HTTPclient) prepareVoteEnvelope(choices []int, election *api.Election) (*models.VoteEnvelope, error) {
+	var err error
+	var keys []types.HexBytes
+	var keyIndexes []uint32
+	if election.VoteMode.EncryptedVotes { // Get encryption keys
+		ctx, cancel := context.WithTimeout(context.Background(), WaitTimeout)
+		defer cancel()
+		ek, err := c.WaitUntilElectionKeys(ctx, election.ElectionID)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, k := range ek.PublicKeys {
+			if len(k.Key) > 0 {
+				keys = append(keys, k.Key)
+				keyIndexes = append(keyIndexes, uint32(k.Index))
+			}
+		}
+
+		if len(keys) == 0 {
+			return nil, fmt.Errorf("no keys for election %s", election.ElectionID)
+		}
+	}
+	// if EncryptedVotes is false, keys will be nil and prepareVotePackageBytes returns plaintext
+	vpb, err := c.prepareVotePackageBytes(&vochain.VotePackage{Votes: choices}, keys)
+	if err != nil {
+		return nil, err
+	}
+
+	return &models.VoteEnvelope{
+		Nonce:                util.RandomBytes(32),
+		ProcessId:            election.ElectionID,
+		VotePackage:          vpb,
+		EncryptionKeyIndexes: keyIndexes,
+	}, nil
+
+}
+
+// prepareVotePackageBytes returns a plaintext json.Marshal(vp) if keys is nil,
+// else assigns a random hex string to vp.Nonce
+// and encrypts the vp bytes for each given key as recipient
+func (c *HTTPclient) prepareVotePackageBytes(vp *vochain.VotePackage, keys []types.HexBytes) ([]byte, error) {
+	if len(keys) > 0 {
+		vp.Nonce = fmt.Sprintf("%x", util.RandomHex(32))
+	}
+
+	vpb, err := json.Marshal(vp)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, k := range keys { // skipped if len(keys) == 0
+		if len(k) == 0 {
+			continue
+		}
+		log.Debugw("encrypting vote", "nonce", vp.Nonce, "key", k)
+		pub, err := nacl.DecodePublic(k.String())
+		if err != nil {
+			return nil, fmt.Errorf("cannot decode encryption key with index %d: (%s)", i, err)
+		}
+		if vpb, err = nacl.Anonymous.Encrypt(vpb, pub); err != nil {
+			return nil, fmt.Errorf("cannot encrypt: (%s)", err)
+		}
+
+	}
+
+	return vpb, nil
+}
+
 // prepareVoteTx prepare an api.Vote struct with the inner transactions encoded
 // based on the vote provided and if it is signed or not.
 func (c *HTTPclient) prepareVoteTx(vote *models.VoteEnvelope, signed bool) (*api.Vote, error) {

cmd/end2endtest/account.go

@@ -4,13 +4,12 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"net/url"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/uuid"
 	apipkg "go.vocdoni.io/dvote/api"
 	"go.vocdoni.io/dvote/apiclient"
 	"go.vocdoni.io/dvote/crypto/ethereum"
@@ -20,51 +19,74 @@ import (
 	"go.vocdoni.io/proto/build/go/models"
 )
 
-func testTokenTransactions(c config) {
+func init() {
+	ops["tokentxs"] = operation{
+		test:        &E2ETokenTxs{},
+		description: "Tests all token related transactions",
+		example: os.Args[0] + " --operation=tokentxs " +
+			"--host http://127.0.0.1:9090/v2",
+	}
+}
+
+var _ VochainTest = (*E2ETokenTxs)(nil)
+
+type E2ETokenTxs struct {
+	api    *apiclient.HTTPclient
+	config *config
+
+	alice, bob *ethereum.SignKeys
+	aliceFP    *models.FaucetPackage
+}
+
+func (t *E2ETokenTxs) Setup(api *apiclient.HTTPclient, config *config) error {
+	t.api = api
+	t.config = config
 
 	// create alice signer
-	alice := &ethereum.SignKeys{}
-	if err := alice.Generate(); err != nil {
-		log.Fatal(err)
+	t.alice = ethereum.NewSignKeys()
+	err := t.alice.Generate()
+	if err != nil {
+		return err
 	}
 
 	// create bob signer
-	bob := &ethereum.SignKeys{}
-	if err := bob.Generate(); err != nil {
-		log.Fatal(err)
-	}
-
-	// Connect to the API host
-	hostURL, err := url.Parse(c.host)
+	t.bob = ethereum.NewSignKeys()
+	err = t.bob.Generate()
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
-	log.Debugw("connecting to API", "host", hostURL.String())
 
-	token := uuid.New()
-	api, err := apiclient.NewHTTPclient(hostURL, &token)
+	// get faucet package for alice
+	t.aliceFP, err = getFaucetPackage(t.config, t.alice.Address().Hex())
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	// check transaction cost
-	if err := testGetTxCost(api); err != nil {
+	if err := testGetTxCost(t.api); err != nil {
 		log.Fatal(err)
 	}
 
-	fp, err := getFaucetPackage(c, alice.Address().Hex())
-	if err != nil {
-		log.Fatal(err)
-	}
 	// check create and set account
-	if err := testCreateAndSetAccount(api, fp, alice, bob); err != nil {
+	if err := testCreateAndSetAccount(t.api, t.aliceFP, t.alice, t.bob); err != nil {
 		log.Fatal(err)
 	}
 
+	return nil
+}
+
+func (t *E2ETokenTxs) Teardown() error {
+	// nothing to do here
+	return nil
+}
+
+func (t *E2ETokenTxs) Run() error {
 	// check send tokens
-	if err := testSendTokens(api, alice, bob); err != nil {
+	if err := testSendTokens(t.api, t.alice, t.bob); err != nil {
 		log.Fatal(err)
 	}
+
+	return nil
 }
 
 func testGetTxCost(api *apiclient.HTTPclient) error {
@@ -190,6 +212,13 @@ func testSendTokens(api *apiclient.HTTPclient, aliceKeys, bobKeys *ethereum.Sign
 	}
 	log.Debugf("mined, tx refs are %+v and %+v", txrefa, txrefb)
 
+	// after a tx is mined in a block, the indexer takes some time to update the balances
+	// (i.e. seconds, if there are votes to be indexed)
+	// give it one more block time
+	ctx, cancel = context.WithTimeout(context.Background(), apiclient.WaitTimeout)
+	defer cancel()
+	api.WaitUntilNextBlock(ctx)
+
 	// now check the resulting state
 	err = checkAccountNonceAndBalance(alice, aliceAcc.Nonce+1,
 		(aliceAcc.Balance - amountAtoB - txCost + amountBtoA))