diff --git a/etcdutl/etcdutl/backup_command.go b/etcdutl/etcdutl/backup_command.go index 89121a37e95..155e655069e 100644 --- a/etcdutl/etcdutl/backup_command.go +++ b/etcdutl/etcdutl/backup_command.go @@ -28,9 +28,9 @@ import ( "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/idutil" "go.etcd.io/etcd/pkg/v3/pbutil" + "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" - "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/schema" @@ -167,7 +167,7 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster) walsnap.Index, walsnap.Term, walsnap.ConfState = snapshot.Metadata.Index, snapshot.Metadata.Term, &desired.confState newss := snap.New(lg, destSnap) snapshot.Metadata.ConfState = desired.confState - snapshot.Data = mustTranslateV2store(lg, snapshot.Data, desired) + snapshot.Data = mustTranslateV2store(lg, desired) if err = newss.SaveSnap(*snapshot); err != nil { lg.Fatal("saveSnap(Snapshoter.SaveSnap) failed", zap.Error(err)) } @@ -175,24 +175,12 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster) return walsnap } -// mustTranslateV2store processes storeData such that they match 'desiredCluster'. -// In particular the method overrides membership information. -func mustTranslateV2store(lg *zap.Logger, storeData []byte, desired *desiredCluster) []byte { - st := v2store.New() - if err := st.Recovery(storeData); err != nil { - lg.Panic("cannot translate v2store", zap.Error(err)) - } - +// mustTranslateV2store returns membership info matching 'desiredCluster' in v2 format. +func mustTranslateV2store(lg *zap.Logger, desired *desiredCluster) []byte { raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members) raftCluster.SetID(desired.nodeId, desired.clusterId) - raftCluster.SetStore(st) - raftCluster.PushMembershipToStorage() - - outputData, err := st.Save() - if err != nil { - lg.Panic("cannot save v2store", zap.Error(err)) - } - return outputData + d := etcdserver.GetMembershipInfoInV2Format(lg, raftCluster) + return d } func translateWAL(lg *zap.Logger, srcWAL string, walsnap walpb.Snapshot) (etcdserverpb.Metadata, raftpb.HardState, []raftpb.Entry) { diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go index 44ea53ea83b..c8f04c9dd01 100644 --- a/server/etcdserver/api/membership/cluster.go +++ b/server/etcdserver/api/membership/cluster.go @@ -854,3 +854,30 @@ func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearner return nil } + +func (c *RaftCluster) Store(store v2store.Store) { + c.Lock() + defer c.Unlock() + + verifyNoMembersInStore(c.lg, store) + + for _, m := range c.members { + mustSaveMemberToStore(c.lg, store, m) + if m.ClientURLs != nil { + mustUpdateMemberAttrInStore(c.lg, store, m) + } + c.lg.Info( + "snapshot storing member", + zap.String("id", m.ID.String()), + zap.Strings("peer-urls", m.PeerURLs), + zap.Bool("is-learner", m.IsLearner), + ) + } + for id, _ := range c.removed { + //We do not need to delete the member since the store is empty. + mustAddToRemovedMembersInStore(c.lg, store, id) + } + if c.version != nil { + mustSaveClusterVersionToStore(c.lg, store, c.version) + } +} diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go index ce98472df7b..6fb65ac1f82 100644 --- a/server/etcdserver/api/membership/cluster_test.go +++ b/server/etcdserver/api/membership/cluster_test.go @@ -21,6 +21,7 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" @@ -975,3 +976,69 @@ func TestIsReadyToPromoteMember(t *testing.T) { } } } + +func TestClusterStore(t *testing.T) { + name := "etcd" + clientURLs := []string{"http://127.0.0.1:4001"} + + tests := []struct { + name string + mems []*Member + removed map[types.ID]bool + }{ + { + name: "Single member, no removed members", + mems: []*Member{ + newTestMember(1, nil, name, clientURLs), + }, + removed: map[types.ID]bool{}, + }, + { + name: "Multiple members, no removed members", + mems: []*Member{ + newTestMember(1, nil, name, clientURLs), + newTestMember(2, nil, name, clientURLs), + newTestMember(3, nil, name, clientURLs), + }, + removed: map[types.ID]bool{}, + }, + { + name: "Single member, one removed member", + mems: []*Member{ + newTestMember(1, nil, name, clientURLs), + }, + removed: map[types.ID]bool{types.ID(2): true}, + }, + { + name: "Multiple members, some removed members", + mems: []*Member{ + newTestMember(1, nil, name, clientURLs), + newTestMember(2, nil, name, clientURLs), + newTestMember(3, nil, name, clientURLs), + }, + removed: map[types.ID]bool{ + types.ID(4): true, + types.ID(5): true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newTestCluster(t, tt.mems) + c.removed = tt.removed + + st := v2store.New("/0", "/1") + c.Store(st) + + // Verify that the members are properly stored + mst, rst := membersFromStore(c.lg, st) + for _, mem := range tt.mems { + assert.Equal(t, mem, mst[mem.ID]) + } + + // Verify that removed members are correctly stored + assert.Equal(t, tt.removed, rst) + }) + } +} diff --git a/server/etcdserver/api/membership/storev2.go b/server/etcdserver/api/membership/storev2.go index d428cb66e22..faf38e2be7c 100644 --- a/server/etcdserver/api/membership/storev2.go +++ b/server/etcdserver/api/membership/storev2.go @@ -78,6 +78,13 @@ func TrimMembershipFromV2Store(lg *zap.Logger, s v2store.Store) error { return nil } +func verifyNoMembersInStore(lg *zap.Logger, s v2store.Store) { + members, removed := membersFromStore(lg, s) + if len(members) != 0 || len(removed) != 0 { + lg.Panic("store has membership info") + } +} + func mustSaveMemberToStore(lg *zap.Logger, s v2store.Store, m *Member) { b, err := json.Marshal(m.RaftAttributes) if err != nil { @@ -101,6 +108,12 @@ func mustDeleteMemberFromStore(lg *zap.Logger, s v2store.Store, id types.ID) { zap.Error(err), ) } + + mustAddToRemovedMembersInStore(lg, s, id) +} + +func mustAddToRemovedMembersInStore(lg *zap.Logger, s v2store.Store, id types.ID) { + if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil { lg.Panic( "failed to create removedMember", diff --git a/server/etcdserver/cluster_util.go b/server/etcdserver/cluster_util.go index 065283a5855..220045c0590 100644 --- a/server/etcdserver/cluster_util.go +++ b/server/etcdserver/cluster_util.go @@ -28,6 +28,7 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/etcdserver/errors" "github.com/coreos/go-semver/semver" @@ -416,3 +417,14 @@ func convertToClusterVersion(v string) (*semver.Version, error) { ver = &semver.Version{Major: ver.Major, Minor: ver.Minor} return ver, nil } + +func GetMembershipInfoInV2Format(lg *zap.Logger, cl *membership.RaftCluster) []byte { + var st v2store.Store + st = v2store.New(StoreClusterPrefix, StoreKeysPrefix) + cl.Store(st) + d, err := st.SaveNoCopy() + if err != nil { + lg.Panic("failed to save v2 store", zap.Error(err)) + } + return d +}