From 9695340c129c2eb92262bfa4d7fa893be8702754 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 21 Oct 2022 14:11:09 +0800 Subject: [PATCH] repo config for s3 compatible store Signed-off-by: Lyndon-Li --- changelogs/unreleased/5478-lyndon | 1 + pkg/repository/config/config.go | 18 ++++++++++++++---- pkg/repository/config/config_test.go | 21 ++++++++++++++++++++- pkg/repository/provider/unified_repo.go | 6 +++--- pkg/restic/common.go | 2 +- 5 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 changelogs/unreleased/5478-lyndon diff --git a/changelogs/unreleased/5478-lyndon b/changelogs/unreleased/5478-lyndon new file mode 100644 index 0000000000..d068694491 --- /dev/null +++ b/changelogs/unreleased/5478-lyndon @@ -0,0 +1 @@ +Issue fix 5477: create the common way to support S3 compatible object storages that work for both Restic and Kopia; Keep the resticRepoPrefix parameter for compatibility \ No newline at end of file diff --git a/pkg/repository/config/config.go b/pkg/repository/config/config.go index d7ed99b69e..c1ef8b906c 100644 --- a/pkg/repository/config/config.go +++ b/pkg/repository/config/config.go @@ -56,7 +56,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) prefix = layout.GetResticDir() } - backendType := GetBackendType(location.Spec.Provider) + backendType := GetBackendType(location.Spec.Provider, location.Spec.Config) if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" { return repoPrefix, nil @@ -87,15 +87,25 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) return fmt.Sprintf("gs:%s:/%s", bucket, prefix), nil } - return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config") + return "", errors.Errorf("invalid backend type %s, provider %s", backendType, location.Spec.Provider) } -func GetBackendType(provider string) BackendType { +// GetBackendType returns a backend type that is known by Velero. +// If the provider doesn't indicate a known backend type, but the endpoint is +// specified, Velero regards it as a S3 compatible object store and return AWSBackend as the type. +func GetBackendType(provider string, config map[string]string) BackendType { if !strings.Contains(provider, "/") { provider = "velero.io/" + provider } - return BackendType(provider) + bt := BackendType(provider) + if IsBackendTypeValid(bt) { + return bt + } else if config != nil && config["s3Url"] != "" { + return AWSBackend + } else { + return bt + } } func IsBackendTypeValid(backendType BackendType) bool { diff --git a/pkg/repository/config/config_test.go b/pkg/repository/config/config_test.go index 2fa26a1936..4f18d6faea 100644 --- a/pkg/repository/config/config_test.go +++ b/pkg/repository/config/config_test.go @@ -48,7 +48,7 @@ func TestGetRepoIdentifier(t *testing.T) { }, }, repoName: "repo-1", - expectedErr: "restic repository prefix (resticRepoPrefix) not specified in backup storage location's config", + expectedErr: "invalid backend type velero.io/unsupported-provider, provider unsupported-provider", }, { name: "resticRepoPrefix in BSL config is used if set", @@ -69,6 +69,25 @@ func TestGetRepoIdentifier(t *testing.T) { repoName: "repo-1", expected: "custom:prefix:/restic/repo-1", }, + { + name: "s3Url in BSL config is used", + bsl: &velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "custom-repo-identifier", + Config: map[string]string{ + "s3Url": "s3Url", + }, + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "bucket", + Prefix: "prefix", + }, + }, + }, + }, + repoName: "repo-1", + expected: "s3:s3Url/bucket/prefix/restic/repo-1", + }, { name: "s3.amazonaws.com URL format is used if region cannot be determined for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 30d1c97aaa..991ffc6980 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -344,7 +344,7 @@ func getRepoPassword(secretStore credentials.SecretStore) (string, error) { } func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string { - backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) switch backendType { case repoconfig.AWSBackend: @@ -368,7 +368,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr return map[string]string{}, errors.New("invalid credentials interface") } - backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) if !repoconfig.IsBackendTypeValid(backendType) { return map[string]string{}, errors.New("invalid storage provider") } @@ -414,7 +414,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string) (map[string]string, error) { result := make(map[string]string) - backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) if !repoconfig.IsBackendTypeValid(backendType) { return map[string]string{}, errors.New("invalid storage provider") } diff --git a/pkg/restic/common.go b/pkg/restic/common.go index f1ecb9a718..26eb2ef27a 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -92,7 +92,7 @@ func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileSto config[repoconfig.CredentialsFileKey] = credsFile } - backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) switch backendType { case repoconfig.AWSBackend: