diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 8a1f2d40e7..8312d4df76 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -127,7 +127,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 874cf53c78..834ab5ca5b 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -104,7 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { } By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 6b10ba9259..0a35e00cb1 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -145,6 +145,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots + version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + OriginVeleroCfg.VeleroVersion = version + // self represents v1.12 if veleroCLI2Version.VeleroVersion == "self" { if OriginVeleroCfg.SnapshotMoveData { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 58b8922554..f5102b79a3 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -119,6 +119,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC veleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", veleroCLI2Version.VeleroVersion), func() { + //Set VeleroImage and RestoreHelperImage to blank //VeleroImage and RestoreHelperImage should be the default value in originalCli tmpCfgForOldVeleroInstall := veleroCfg @@ -128,6 +129,10 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.RestoreHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" tmpCfgForOldVeleroInstall.UploaderType = "" + version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + tmpCfgForOldVeleroInstall.VeleroVersion = version + if supportUploaderType { tmpCfgForOldVeleroInstall.UseRestic = false tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go index 677de41279..56af1ef58f 100644 --- a/test/e2e/util/csi/common.go +++ b/test/e2e/util/csi/common.go @@ -91,6 +91,43 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error } return snapshotHandleList, nil } +func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { + _, snapshotClient, err := GetClients() + if err != nil { + return nil, err + } + vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err1 != nil { + return nil, err + } + + var snapshotHandleList []string + for _, i := range vscList.Items { + if i.Status == nil { + fmt.Println("SnapshotHandle Status s nil") + continue + } + if i.Status.SnapshotHandle == nil { + fmt.Println("SnapshotHandle is nil") + continue + } + + if i.Labels == nil { + fmt.Println("VolumeSnapshotContents label is nil") + continue + } + + if i.Labels["velero.io/backup-name"] == backupName { + tmp := strings.Split(*i.Status.SnapshotHandle, "/") + snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) + } + } + + if len(snapshotHandleList) == 0 { + fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + } + return snapshotHandleList, nil +} func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) { pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) if err != nil { @@ -128,11 +165,19 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { var err error var snapshotContentNameList []string - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + if apiVersion == "v1beta1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else if apiVersion == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else { + return nil, errors.New("API version is invalid") } if len(snapshotContentNameList) != expectedCount { return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 1830b15d3a..be8727b549 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -111,7 +111,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent + if !veleroCfg.UseRestic { + veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent + } + veleroInstallOptions.UseRestic = veleroCfg.UseRestic veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace veleroInstallOptions.UploaderType = veleroCfg.UploaderType @@ -198,6 +201,9 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Image) > 0 { args = append(args, "--image", options.Image) } + if options.UseRestic { + args = append(args, "--use-restic") + } if options.UseNodeAgent { args = append(args, "--use-node-agent") } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index c5d1834c16..ad5359260d 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -56,6 +56,27 @@ const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" var pluginsMatrix = map[string]map[string][]string{ + "v1.7": { + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, + }, + "v1.8": { + "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, + }, + "v1.9": { + "aws": {"velero/velero-plugin-for-aws:v1.5.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.5.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.5.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.3.0"}, + }, "v1.10": { "aws": {"velero/velero-plugin-for-aws:v1.6.0"}, "azure": {"velero/velero-plugin-for-microsoft-azure:v1.6.0"}, @@ -558,7 +579,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return []string{}, errors.New("CloudProvider should be provided") } - version, err := getVeleroVersion(ctx, veleroCLI, true) + version, err := GetVeleroVersion(ctx, veleroCLI, true) if err != nil { return nil, errors.WithMessage(err, "failed to get velero version") } @@ -593,11 +614,17 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if objectStoreProvider == "" { objectStoreProvider = cloudProvider } - version, err := getVeleroVersion(ctx, veleroCLI, true) - if err != nil { - return nil, errors.WithMessage(err, "failed to get velero version") - } + var version string + var err error + if veleroCfg.VeleroVersion != "" { + version = veleroCfg.VeleroVersion + } else { + version, err = GetVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + } if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" { needDataMoverPlugin = true } @@ -611,8 +638,14 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider) +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, plugin string) error { + var err error + var plugins []string + if plugin == "" { + plugins, err = getProviderPlugins(ctx, veleroCLI, provider) + } else { + plugins = append(plugins, plugin) + } fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins) if err != nil { @@ -743,7 +776,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace return result, nil } -func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { +func GetVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { args := []string{"version", "--timeout", "60s"} if clientOnly { args = append(args, "--client-only") @@ -778,7 +811,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { tag := expectedVer - tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false) + tagInstalled, err := GetVeleroVersion(ctx, veleroCLI, false) if err != nil { return errors.WithMessagef(err, "failed to get Velero version") } @@ -1107,13 +1140,23 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { var snapshotCheckPoint SnapshotCheckPoint - var err error + snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { + resourceName := "snapshot.storage.k8s.io" + + srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName) + + if err != nil { + return snapshotCheckPoint, err + } + if len(srcVersions) == 0 { + return snapshotCheckPoint, errors.New("Fail to get APIVersion") + } + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } }