Merge pull request #16132 from geetasg/pr5

Add a method to export membership info to v2 store from RaftCluster

etcdutl/etcdutl/backup_command.go

@@ -28,9 +28,9 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
+	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
 	"go.etcd.io/etcd/server/v3/storage/schema"
@@ -167,32 +167,20 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster)
 		walsnap.Index, walsnap.Term, walsnap.ConfState = snapshot.Metadata.Index, snapshot.Metadata.Term, &desired.confState
 		newss := snap.New(lg, destSnap)
 		snapshot.Metadata.ConfState = desired.confState
-		snapshot.Data = mustTranslateV2store(lg, snapshot.Data, desired)
+		snapshot.Data = mustTranslateV2store(lg, desired)
 		if err = newss.SaveSnap(*snapshot); err != nil {
 			lg.Fatal("saveSnap(Snapshoter.SaveSnap) failed", zap.Error(err))
 		}
 	}
 	return walsnap
 }
 
-// mustTranslateV2store processes storeData such that they match 'desiredCluster'.
-// In particular the method overrides membership information.
-func mustTranslateV2store(lg *zap.Logger, storeData []byte, desired *desiredCluster) []byte {
-	st := v2store.New()
-	if err := st.Recovery(storeData); err != nil {
-		lg.Panic("cannot translate v2store", zap.Error(err))
-	}
-
+// mustTranslateV2store returns membership info matching 'desiredCluster' in v2 format.
+func mustTranslateV2store(lg *zap.Logger, desired *desiredCluster) []byte {
 	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
 	raftCluster.SetID(desired.nodeId, desired.clusterId)
-	raftCluster.SetStore(st)
-	raftCluster.PushMembershipToStorage()
-
-	outputData, err := st.Save()
-	if err != nil {
-		lg.Panic("cannot save v2store", zap.Error(err))
-	}
-	return outputData
+	d := etcdserver.GetMembershipInfoInV2Format(lg, raftCluster)
+	return d
 }
 
 func translateWAL(lg *zap.Logger, srcWAL string, walsnap walpb.Snapshot) (etcdserverpb.Metadata, raftpb.HardState, []raftpb.Entry) {

server/etcdserver/api/membership/cluster.go

@@ -854,3 +854,30 @@ func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearner
 
 	return nil
 }
+
+func (c *RaftCluster) Store(store v2store.Store) {
+	c.Lock()
+	defer c.Unlock()
+
+	verifyNoMembersInStore(c.lg, store)
+
+	for _, m := range c.members {
+		mustSaveMemberToStore(c.lg, store, m)
+		if m.ClientURLs != nil {
+			mustUpdateMemberAttrInStore(c.lg, store, m)
+		}
+		c.lg.Info(
+			"snapshot storing member",
+			zap.String("id", m.ID.String()),
+			zap.Strings("peer-urls", m.PeerURLs),
+			zap.Bool("is-learner", m.IsLearner),
+		)
+	}
+	for id, _ := range c.removed {
+		//We do not need to delete the member since the store is empty.
+		mustAddToRemovedMembersInStore(c.lg, store, id)
+	}
+	if c.version != nil {
+		mustSaveClusterVersionToStore(c.lg, store, c.version)
+	}
+}

server/etcdserver/api/membership/cluster_test.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
@@ -975,3 +976,69 @@ func TestIsReadyToPromoteMember(t *testing.T) {
 		}
 	}
 }
+
+func TestClusterStore(t *testing.T) {
+	name := "etcd"
+	clientURLs := []string{"http://127.0.0.1:4001"}
+
+	tests := []struct {
+		name    string
+		mems    []*Member
+		removed map[types.ID]bool
+	}{
+		{
+			name: "Single member, no removed members",
+			mems: []*Member{
+				newTestMember(1, nil, name, clientURLs),
+			},
+			removed: map[types.ID]bool{},
+		},
+		{
+			name: "Multiple members, no removed members",
+			mems: []*Member{
+				newTestMember(1, nil, name, clientURLs),
+				newTestMember(2, nil, name, clientURLs),
+				newTestMember(3, nil, name, clientURLs),
+			},
+			removed: map[types.ID]bool{},
+		},
+		{
+			name: "Single member, one removed member",
+			mems: []*Member{
+				newTestMember(1, nil, name, clientURLs),
+			},
+			removed: map[types.ID]bool{types.ID(2): true},
+		},
+		{
+			name: "Multiple members, some removed members",
+			mems: []*Member{
+				newTestMember(1, nil, name, clientURLs),
+				newTestMember(2, nil, name, clientURLs),
+				newTestMember(3, nil, name, clientURLs),
+			},
+			removed: map[types.ID]bool{
+				types.ID(4): true,
+				types.ID(5): true,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newTestCluster(t, tt.mems)
+			c.removed = tt.removed
+
+			st := v2store.New("/0", "/1")
+			c.Store(st)
+
+			// Verify that the members are properly stored
+			mst, rst := membersFromStore(c.lg, st)
+			for _, mem := range tt.mems {
+				assert.Equal(t, mem, mst[mem.ID])
+			}
+
+			// Verify that removed members are correctly stored
+			assert.Equal(t, tt.removed, rst)
+		})
+	}
+}

server/etcdserver/api/membership/storev2.go

@@ -78,6 +78,13 @@ func TrimMembershipFromV2Store(lg *zap.Logger, s v2store.Store) error {
 	return nil
 }
 
+func verifyNoMembersInStore(lg *zap.Logger, s v2store.Store) {
+	members, removed := membersFromStore(lg, s)
+	if len(members) != 0 || len(removed) != 0 {
+		lg.Panic("store has membership info")
+	}
+}
+
 func mustSaveMemberToStore(lg *zap.Logger, s v2store.Store, m *Member) {
 	b, err := json.Marshal(m.RaftAttributes)
 	if err != nil {
@@ -101,6 +108,12 @@ func mustDeleteMemberFromStore(lg *zap.Logger, s v2store.Store, id types.ID) {
 			zap.Error(err),
 		)
 	}
+
+	mustAddToRemovedMembersInStore(lg, s, id)
+}
+
+func mustAddToRemovedMembersInStore(lg *zap.Logger, s v2store.Store, id types.ID) {
+
 	if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil {
 		lg.Panic(
 			"failed to create removedMember",

server/etcdserver/cluster_util.go

@@ -28,6 +28,7 @@ import (
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/errors"
 
 	"github.com/coreos/go-semver/semver"
@@ -416,3 +417,14 @@ func convertToClusterVersion(v string) (*semver.Version, error) {
 	ver = &semver.Version{Major: ver.Major, Minor: ver.Minor}
 	return ver, nil
 }
+
+func GetMembershipInfoInV2Format(lg *zap.Logger, cl *membership.RaftCluster) []byte {
+	var st v2store.Store
+	st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
+	cl.Store(st)
+	d, err := st.SaveNoCopy()
+	if err != nil {
+		lg.Panic("failed to save v2 store", zap.Error(err))
+	}
+	return d
+}