Add test to cover the etcdutl migrate command

Signed-off-by: Benjamin Wang <[email protected]>

tests/e2e/utl_migrate_test.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cluster_proxy
+
 package e2e
 
 import (
@@ -42,62 +44,91 @@ func TestEtctlutlMigrate(t *testing.T) {
 		name           string
 		targetVersion  string
 		clusterVersion e2e.ClusterVersion
+		clusterSize    int
 		force          bool
 
 		expectLogsSubString  string
 		expectStorageVersion *semver.Version
+		expectTargetBinary   string
 	}{
 		{
 			name:                 "Invalid target version string",
 			targetVersion:        "abc",
+			clusterSize:          1,
 			expectLogsSubString:  `Error: wrong target version format, expected "X.Y", got "abc"`,
 			expectStorageVersion: &version.V3_6,
 		},
 		{
 			name:                 "Invalid target version",
 			targetVersion:        "3.a",
+			clusterSize:          1,
 			expectLogsSubString:  `Error: failed to parse target version: strconv.ParseInt: parsing "a": invalid syntax`,
 			expectStorageVersion: &version.V3_6,
 		},
 		{
 			name:                 "Target with only major version is invalid",
 			targetVersion:        "3",
+			clusterSize:          1,
 			expectLogsSubString:  `Error: wrong target version format, expected "X.Y", got "3"`,
 			expectStorageVersion: &version.V3_6,
 		},
 		{
 			name:                 "Target with patch version is invalid",
 			targetVersion:        "3.6.0",
+			clusterSize:          1,
 			expectLogsSubString:  `Error: wrong target version format, expected "X.Y", got "3.6.0"`,
 			expectStorageVersion: &version.V3_6,
 		},
 		{
 			name:                "Migrate v3.5 to v3.5 is no-op",
 			clusterVersion:      e2e.LastVersion,
 			targetVersion:       "3.5",
+			clusterSize:         1,
 			expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.5"}`,
 		},
 		{
-			name:                 "Upgrade v3.5 to v3.6 should work",
+			name:                 "Upgrade 1 member cluster from v3.5 to v3.6 should work",
 			clusterVersion:       e2e.LastVersion,
 			targetVersion:        "3.6",
+			clusterSize:          1,
 			expectStorageVersion: &version.V3_6,
+			expectTargetBinary:   e2e.BinPath.Etcd,
+		},
+		{
+			name:                 "Upgrade 3 member cluster from v3.5 to v3.6 should work",
+			clusterVersion:       e2e.LastVersion,
+			targetVersion:        "3.6",
+			clusterSize:          3,
+			expectStorageVersion: &version.V3_6,
+			expectTargetBinary:   e2e.BinPath.Etcd,
 		},
 		{
 			name:                 "Migrate v3.6 to v3.6 is no-op",
 			targetVersion:        "3.6",
+			clusterSize:          1,
 			expectLogsSubString:  "storage version up-to-date\t" + `{"storage-version": "3.6"}`,
 			expectStorageVersion: &version.V3_6,
 		},
 		{
-			name:                 "Downgrade v3.6 to v3.5 should work",
+			name:                 "Downgrade 1 member cluster from v3.6 to v3.5 should work",
 			targetVersion:        "3.5",
+			clusterSize:          1,
 			expectLogsSubString:  "updated storage version",
 			expectStorageVersion: nil, // 3.5 doesn't have the field `storageVersion`, so it returns nil.
+			expectTargetBinary:   e2e.BinPath.EtcdLastRelease,
+		},
+		{
+			name:                 "Downgrade 3 member cluster from v3.6 to v3.5 should work",
+			targetVersion:        "3.5",
+			clusterSize:          3,
+			expectLogsSubString:  "updated storage version",
+			expectStorageVersion: nil, // 3.5 doesn't have the field `storageVersion`, so it returns nil.
+			expectTargetBinary:   e2e.BinPath.EtcdLastRelease,
 		},
 		{
 			name:                 "Upgrade v3.6 to v3.7 with force should work",
 			targetVersion:        "3.7",
+			clusterSize:          1,
 			force:                true,
 			expectLogsSubString:  "forcefully set storage version\t" + `{"storage-version": "3.7"}`,
 			expectStorageVersion: &semver.Version{Major: 3, Minor: 7},
@@ -115,7 +146,7 @@ func TestEtctlutlMigrate(t *testing.T) {
 			epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t,
 				e2e.WithVersion(tc.clusterVersion),
 				e2e.WithDataDirPath(dataDirPath),
-				e2e.WithClusterSize(1),
+				e2e.WithClusterSize(tc.clusterSize),
 				e2e.WithKeepDataDir(true),
 				// Set low SnapshotCount to ensure wal snapshot is done
 				e2e.WithSnapshotCount(1),
@@ -137,29 +168,73 @@ func TestEtctlutlMigrate(t *testing.T) {
 				require.NoError(t, e2e.SpawnWithExpect(append(prefixArgs, "put", fmt.Sprintf("%d", i), "value"), expect.ExpectedResponse{Value: "OK"}))
 			}
 
-			t.Log("Stopping the server...")
-			err = epc.Procs[0].Stop()
-			require.NoError(t, err)
+			t.Log("Stopping all the servers")
+			for i := 0; i < len(epc.Procs); i++ {
+				t.Logf("Stopping server %d: %v", i, epc.Procs[i].EndpointsGRPC())
+				err = epc.Procs[i].Stop()
+				require.NoError(t, err)
+			}
+
+			t.Log("etcdutl migrate all members")
+			for i := 0; i < len(epc.Procs); i++ {
+				t.Logf("etcdutl migrate member %d: %v", i, epc.Procs[i].EndpointsGRPC())
+				memberDataDir := epc.Procs[i].Config().DataDirPath
+				args := []string{e2e.BinPath.Etcdutl, "migrate", "--data-dir", memberDataDir, "--target-version", tc.targetVersion}
+				if tc.force {
+					args = append(args, "--force")
+				}
+				err = e2e.SpawnWithExpect(args, expect.ExpectedResponse{Value: tc.expectLogsSubString})
+				if err != nil && tc.expectLogsSubString != "" {
+					require.ErrorContains(t, err, tc.expectLogsSubString)
+				} else {
+					require.NoError(t, err)
+				}
 
-			t.Log("etcdutl migrate...")
-			memberDataDir := epc.Procs[0].Config().DataDirPath
-			args := []string{e2e.BinPath.Etcdutl, "migrate", "--data-dir", memberDataDir, "--target-version", tc.targetVersion}
-			if tc.force {
-				args = append(args, "--force")
+				be := backend.NewDefaultBackend(lg, filepath.Join(memberDataDir, "member/snap/db"))
+				ver := schema.ReadStorageVersion(be.ReadTx())
+				assert.Equal(t, tc.expectStorageVersion, ver)
+				be.Close()
 			}
-			err = e2e.SpawnWithExpect(args, expect.ExpectedResponse{Value: tc.expectLogsSubString})
-			if err != nil && tc.expectLogsSubString != "" {
-				require.ErrorContains(t, err, tc.expectLogsSubString)
-			} else {
-				require.NoError(t, err)
+
+			if len(tc.expectTargetBinary) == 0 {
+				return
 			}
 
-			t.Log("etcdutl migrate...")
-			be := backend.NewDefaultBackend(lg, filepath.Join(memberDataDir, "member/snap/db"))
-			defer be.Close()
+			t.Log("Start all members with new binary")
+			for i := 0; i < len(epc.Procs); i++ {
+				t.Logf("Replace binary for member %d: %v", i, epc.Procs[i].EndpointsGRPC())
+				member := epc.Procs[i]
+				member.Config().ExecPath = tc.expectTargetBinary
+			}
+			require.NoError(t, epc.Start(context.TODO()))
 
-			ver := schema.ReadStorageVersion(be.ReadTx())
-			assert.Equal(t, tc.expectStorageVersion, ver)
+			t.Log("Verify the versions of all members")
+			for i := 0; i < len(epc.Procs); i++ {
+				t.Logf("Verify the version of member %d: %v", i, epc.Procs[i].EndpointsGRPC())
+				expectedVersion := tc.expectStorageVersion
+				if expectedVersion == nil {
+					expectedVersion = &version.V3_5
+				}
+
+				verifyVersion(t, epc, epc.Procs[i], expectedVersion, expectedVersion)
+			}
 		})
 	}
 }
+
+func verifyVersion(t *testing.T, clus *e2e.EtcdProcessCluster, member e2e.EtcdProcess, expectedServerVersion, expectedClusterVersion *semver.Version) error {
+	var err error
+	expected := fmt.Sprintf(`"etcdserver":"%d.%d\..*"etcdcluster":"%d\.%d\.`, expectedServerVersion.Major, expectedServerVersion.Minor, expectedClusterVersion.Major, expectedClusterVersion.Minor)
+	for i := 0; i < 35; i++ {
+		if err = e2e.CURLGetFromMember(clus, member, e2e.CURLReq{Endpoint: "/version", Expected: expect.ExpectedResponse{Value: expected, IsRegularExpr: true}}); err != nil {
+			t.Logf("#%d: v3 is not ready yet (%v)", i, err)
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		break
+	}
+	if err != nil {
+		return fmt.Errorf("failed to verify version, expected %v got (%w)", expected, err)
+	}
+	return nil
+}