vochain/indexer: add support for backups

A new SaveBackup method which creates a backup to a file on disk,
and a new BackupPath constructor option which can restore them.

This only implements backing up and restoring from a file on disk.
Future changes will integrate this with the rest of the codebase,
such as taking regular backups or listing the stored backups.

Updates vocdoni#1062.

vochain/indexer/indexer.go

@@ -6,6 +6,7 @@ import (
 	"embed"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -97,6 +98,9 @@ type Indexer struct {
 type Options struct {
 	DataDir string
 
+	// BackupPath can be set to restore the database from a backup created via SaveBackup.
+	BackupPath string
+
 	IgnoreLiveResults bool
 }
 
@@ -123,6 +127,12 @@ func New(app *vochain.BaseApplication, opts Options) (*Indexer, error) {
 		return nil, err
 	}
 	dbPath := filepath.Join(opts.DataDir, "db.sqlite")
+	if bpath := opts.BackupPath; bpath != "" {
+		if err := copyFile(dbPath, bpath); err != nil {
+			return nil, fmt.Errorf("could not restore indexer backup: %w", err)
+		}
+	}
+
 	var err error
 
 	// sqlite doesn't support multiple concurrent writers.
@@ -174,6 +184,25 @@ func New(app *vochain.BaseApplication, opts Options) (*Indexer, error) {
 	return idx, nil
 }
 
+func copyFile(dst, src string) error {
+	srcf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcf.Close()
+
+	// For now, we don't care about permissions
+	dstf, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(dstf, srcf)
+	if err2 := dstf.Close(); err == nil {
+		err = err2
+	}
+	return err
+}
+
 func (idx *Indexer) Close() error {
 	if err := idx.readOnlyDB.Close(); err != nil {
 		return err
@@ -184,6 +213,16 @@ func (idx *Indexer) Close() error {
 	return nil
 }
 
+// SaveBackup backs up the database to a file on disk.
+// Note that writes to the database may be blocked until the backup finishes,
+// and an error may occur if a file at path already exists.
+//
+// For sqlite, this is done via "VACUUM INTO", so the resulting file is also a database.
+func (idx *Indexer) SaveBackup(ctx context.Context, path string) error {
+	_, err := idx.readOnlyDB.ExecContext(ctx, `VACUUM INTO ?`, path)
+	return err
+}
+
 // blockTxQueries assumes that lockPool is locked.
 func (idx *Indexer) blockTxQueries() *indexerdb.Queries {
 	if idx.blockMu.TryLock() {

vochain/indexer/indexer_test.go

@@ -2,11 +2,13 @@ package indexer
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
 	stdlog "log"
 	"math/big"
+	"path/filepath"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -43,6 +45,77 @@ func newTestIndexer(tb testing.TB, app *vochain.BaseApplication) *Indexer {
 	return idx
 }
 
+func TestBackup(t *testing.T) {
+	app := vochain.TestBaseApplication(t)
+
+	idx, err := New(app, Options{DataDir: t.TempDir()})
+	qt.Assert(t, err, qt.IsNil)
+
+	wantTotalVotes := func(want uint64) {
+		got, err := idx.CountTotalVotes()
+		qt.Assert(t, err, qt.IsNil)
+		qt.Assert(t, got, qt.Equals, want)
+	}
+
+	vp, err := state.NewVotePackage([]int{1, 1, 1}).Encode()
+	qt.Assert(t, err, qt.IsNil)
+
+	// A new indexer has no votes.
+	wantTotalVotes(0)
+
+	// Add 10 votes and check they are counted.
+	pid := util.RandomBytes(32)
+	err = app.State.AddProcess(&models.Process{
+		ProcessId:     pid,
+		EnvelopeType:  &models.EnvelopeType{EncryptedVotes: false},
+		Status:        models.ProcessStatus_READY,
+		Mode:          &models.ProcessMode{AutoStart: true},
+		BlockCount:    10,
+		MaxCensusSize: 1000,
+		VoteOptions: &models.ProcessVoteOptions{
+			MaxCount:     5,
+			MaxValue:     1,
+			MaxTotalCost: 3,
+			CostExponent: 1,
+		},
+	})
+	qt.Assert(t, err, qt.IsNil)
+	for i := 0; i < 10; i++ {
+		v := &state.Vote{ProcessID: pid, VotePackage: vp, Nullifier: util.RandomBytes(32)}
+		qt.Assert(t, app.State.AddVote(v), qt.IsNil)
+	}
+	app.AdvanceTestBlock()
+	wantTotalVotes(10)
+
+	// Back up the database.
+	backupPath := filepath.Join(t.TempDir(), "backup")
+	err = idx.SaveBackup(context.TODO(), backupPath)
+	qt.Assert(t, err, qt.IsNil)
+
+	// Add another 5 votes which aren't in the backup.
+	for i := 0; i < 5; i++ {
+		v := &state.Vote{ProcessID: pid, VotePackage: vp, Nullifier: util.RandomBytes(32)}
+		qt.Assert(t, app.State.AddVote(v), qt.IsNil)
+	}
+	app.AdvanceTestBlock()
+	wantTotalVotes(15)
+
+	// Starting a new database without the backup should see zero votes.
+	idx.Close()
+	idx, err = New(app, Options{DataDir: t.TempDir()})
+	qt.Assert(t, err, qt.IsNil)
+	wantTotalVotes(0)
+
+	// Starting a new database with the backup should see the votes from the backup.
+	idx.Close()
+	idx, err = New(app, Options{DataDir: t.TempDir(), BackupPath: backupPath})
+	qt.Assert(t, err, qt.IsNil)
+	wantTotalVotes(10)
+
+	// Close the last indexer.
+	idx.Close()
+}
+
 func TestEntityList(t *testing.T) {
 	for _, count := range []int{2, 100, 155} {
 		t.Run(fmt.Sprintf("count=%03d", count), func(t *testing.T) {