From 561739691a2ce8f411148f6dede962d3c2257c4b Mon Sep 17 00:00:00 2001 From: jackyalbo Date: Sun, 9 Jul 2023 16:28:59 +0300 Subject: [PATCH] Adding basic support for working with external postgres DB Signed-off-by: jackyalbo --- deploy/crds/noobaa.io_noobaas_crd.yaml | 13 ++++ deploy/internal/deployment-endpoint.yaml | 1 + deploy/internal/statefulset-core.yaml | 1 + pkg/apis/noobaa/v1alpha1/noobaa_types.go | 4 ++ .../noobaa/v1alpha1/zz_generated.deepcopy.go | 5 ++ pkg/bundle/deploy.go | 21 +++++- pkg/options/options.go | 8 +++ pkg/system/phase2_creating.go | 36 +++++++++- pkg/system/reconciler.go | 68 ++++++++++++++----- pkg/system/system.go | 52 ++++++++++++-- 10 files changed, 183 insertions(+), 26 deletions(-) diff --git a/deploy/crds/noobaa.io_noobaas_crd.yaml b/deploy/crds/noobaa.io_noobaas_crd.yaml index d10dcad813..266c3010c7 100644 --- a/deploy/crds/noobaa.io_noobaas_crd.yaml +++ b/deploy/crds/noobaa.io_noobaas_crd.yaml @@ -1393,6 +1393,19 @@ spec: type: object type: object type: object + externalPgSecret: + description: ExternalPgSecret (optional) holds an optional secret + with a url to an extrenal Postgres DB to be used + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object image: description: Image (optional) overrides the default image for the server container diff --git a/deploy/internal/deployment-endpoint.yaml b/deploy/internal/deployment-endpoint.yaml index b0927a277f..98a290f4ca 100644 --- a/deploy/internal/deployment-endpoint.yaml +++ b/deploy/internal/deployment-endpoint.yaml @@ -88,6 +88,7 @@ spec: - name: DB_TYPE - name: MONGODB_URL - name: POSTGRES_HOST + - name: POSTGRES_PORT - name: POSTGRES_DBNAME value: nbcore - name: POSTGRES_USER diff --git a/deploy/internal/statefulset-core.yaml b/deploy/internal/statefulset-core.yaml index 6c667aac94..435152dcb7 100644 --- a/deploy/internal/statefulset-core.yaml +++ b/deploy/internal/statefulset-core.yaml @@ -99,6 +99,7 @@ spec: value: "mongodb://noobaa-db-0.noobaa-db/nbcore" - name: POSTGRES_HOST value: "noobaa-db-pg-0.noobaa-db-pg" + - name: POSTGRES_PORT - name: POSTGRES_DBNAME value: nbcore - name: POSTGRES_USER diff --git a/pkg/apis/noobaa/v1alpha1/noobaa_types.go b/pkg/apis/noobaa/v1alpha1/noobaa_types.go index 5f3bc95fcc..8e1c761f8a 100644 --- a/pkg/apis/noobaa/v1alpha1/noobaa_types.go +++ b/pkg/apis/noobaa/v1alpha1/noobaa_types.go @@ -117,6 +117,10 @@ type NooBaaSpec struct { // +optional MongoDbURL string `json:"mongoDbURL,omitempty"` + // ExternalPgSecret (optional) holds an optional secret with a url to an extrenal Postgres DB to be used + // +optional + ExternalPgSecret *corev1.SecretReference `json:"externalPgSecret,omitempty"` + // DebugLevel (optional) sets the debug level // +optional // +kubebuilder:validation:Enum=all;nsfs;warn;default_level diff --git a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go index dc0a39ee6e..cf9e35f8f6 100644 --- a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go @@ -1090,6 +1090,11 @@ func (in *NooBaaSpec) DeepCopyInto(out *NooBaaSpec) { *out = new(string) **out = **in } + if in.ExternalPgSecret != nil { + in, out := &in.ExternalPgSecret, &out.ExternalPgSecret + *out = new(corev1.SecretReference) + **out = **in + } if in.PVPoolDefaultStorageClass != nil { in, out := &in.PVPoolDefaultStorageClass, &out.PVPoolDefaultStorageClass *out = new(string) diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go index 043d5a0b13..590a1087ed 100644 --- a/pkg/bundle/deploy.go +++ b/pkg/bundle/deploy.go @@ -1437,7 +1437,7 @@ spec: status: {} ` -const Sha256_deploy_crds_noobaa_io_noobaas_crd_yaml = "5d42c4e8e815c9fed4705d6bf312848202aa4b8f7733d971151fb1cac8eea279" +const Sha256_deploy_crds_noobaa_io_noobaas_crd_yaml = "ada3ba6a3a6aecc2a957947d822baaad68c87f4d04eda5df31e883354dcbff71" const File_deploy_crds_noobaa_io_noobaas_crd_yaml = `apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -2834,6 +2834,19 @@ spec: type: object type: object type: object + externalPgSecret: + description: ExternalPgSecret (optional) holds an optional secret + with a url to an extrenal Postgres DB to be used + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object image: description: Image (optional) overrides the default image for the server container @@ -3585,7 +3598,7 @@ data: su postgres -c "bash -x /usr/bin/run-postgresql" ` -const Sha256_deploy_internal_deployment_endpoint_yaml = "591b74cab691e57295c490d0963131b4eefb345aef08c383ef01a9ed4e19ca2b" +const Sha256_deploy_internal_deployment_endpoint_yaml = "b87bb78e630d9e007b71b5aa7745f5d6b6f1771cdd949735652ddc6ebb6ff9d5" const File_deploy_internal_deployment_endpoint_yaml = `apiVersion: apps/v1 kind: Deployment @@ -3677,6 +3690,7 @@ spec: - name: DB_TYPE - name: MONGODB_URL - name: POSTGRES_HOST + - name: POSTGRES_PORT - name: POSTGRES_DBNAME value: nbcore - name: POSTGRES_USER @@ -4600,7 +4614,7 @@ spec: noobaa-s3-svc: "true" ` -const Sha256_deploy_internal_statefulset_core_yaml = "acb0b7199dfc55c7e4ceaff49d2ab754c527ad6bc9be7af539153e10b07294d3" +const Sha256_deploy_internal_statefulset_core_yaml = "71a1afa6000a2ad334ec234951f0cd245d44ceea36fe57c444869accce9c75b7" const File_deploy_internal_statefulset_core_yaml = `apiVersion: apps/v1 kind: StatefulSet @@ -4703,6 +4717,7 @@ spec: value: "mongodb://noobaa-db-0.noobaa-db/nbcore" - name: POSTGRES_HOST value: "noobaa-db-pg-0.noobaa-db-pg" + - name: POSTGRES_PORT - name: POSTGRES_DBNAME value: nbcore - name: POSTGRES_USER diff --git a/pkg/options/options.go b/pkg/options/options.go index a8aed0af65..d8809db40a 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -97,6 +97,10 @@ var DBStorageClass = "" // it can be overridden for testing or different url. var MongoDbURL = "" +// PostgresDbURL is used for providing postgres url +// it can be overridden for testing or different url. +var PostgresDbURL = "" + // DebugLevel can be used to override the default debug level var DebugLevel = "default_level" @@ -224,6 +228,10 @@ func init() { &MongoDbURL, "mongodb-url", MongoDbURL, "url for mongodb", ) + FlagSet.StringVar( + &PostgresDbURL, "postgres-url", + PostgresDbURL, "url for postgresql", + ) FlagSet.StringVar( &DebugLevel, "debug-level", DebugLevel, "The type of debug sets that the system prints (all, nsfs, warn, default_level)", diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index 9a520a38b8..f215a43fdc 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -122,8 +122,8 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { if err := r.UpgradeSplitDB(); err != nil { return err } - // create the mongo db only if mongo db url is not given. - if r.NooBaa.Spec.MongoDbURL == "" { + // create the mongo db only if mongo db url is not given, and postgres secret as well + if r.NooBaa.Spec.MongoDbURL == "" && r.NooBaa.Spec.ExternalPgSecret == nil { if err := r.UpgradeSplitDB(); err != nil { return err } @@ -398,7 +398,33 @@ func (r *Reconciler) setDesiredCoreEnv(c *corev1.Container) { } case "POSTGRES_HOST": - c.Env[j].Value = r.NooBaaPostgresDB.Name + "-0." + r.NooBaaPostgresDB.Spec.ServiceName + if r.PgExternalHost != "" { + c.Env[j].Value = r.PgExternalHost + } else { + c.Env[j].Value = r.NooBaaPostgresDB.Name + "-0." + r.NooBaaPostgresDB.Spec.ServiceName + } + + case "POSTGRES_PORT": + if r.PgExternalPort != "" { + c.Env[j].Value = r.PgExternalPort + } else { + c.Env[j].Value = "5432" + } + + case "POSTGRES_DBNAME": + if r.NooBaa.Spec.DBType == "postgres" { + if c.Env[j].Value != "" { + c.Env[j].Value = "" + } + c.Env[j].ValueFrom = &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.SecretDB.Name, + }, + Key: "dbname", + }, + } + } case "DB_TYPE": if r.NooBaa.Spec.DBType == "postgres" { @@ -1038,6 +1064,10 @@ func (r *Reconciler) reconcileDBRBAC() error { func (r *Reconciler) ReconcileDB() error { var err error + if r.NooBaa.Spec.ExternalPgSecret != nil { + return nil + } + if err = r.reconcileDBRBAC(); err != nil { return err } diff --git a/pkg/system/reconciler.go b/pkg/system/reconciler.go index 87ef708ddf..b4dee1a52b 100644 --- a/pkg/system/reconciler.go +++ b/pkg/system/reconciler.go @@ -3,6 +3,8 @@ package system import ( "context" "fmt" + "net" + "net/url" "os" goruntime "runtime" "strings" @@ -56,18 +58,19 @@ var ( // Reconciler is the context for loading or reconciling a noobaa system type Reconciler struct { - Request types.NamespacedName - Client client.Client - Scheme *runtime.Scheme - Ctx context.Context - Logger *logrus.Entry - Recorder record.EventRecorder - NBClient nb.Client - CoreVersion string - OperatorVersion string - OAuthEndpoints *util.OAuth2Endpoints - MongoConnectionString string - ApplyCAsToPods string + Request types.NamespacedName + Client client.Client + Scheme *runtime.Scheme + Ctx context.Context + Logger *logrus.Entry + Recorder record.EventRecorder + NBClient nb.Client + CoreVersion string + OperatorVersion string + OAuthEndpoints *util.OAuth2Endpoints + MongoConnectionString string + PostgresConnectionString string + ApplyCAsToPods string NooBaa *nbv1.NooBaa ServiceAccount *corev1.ServiceAccount @@ -118,6 +121,8 @@ type Reconciler struct { KedaTriggerAuthentication *kedav1alpha1.TriggerAuthentication KedaScaled *kedav1alpha1.ScaledObject AdapterHPA *autoscalingv2.HorizontalPodAutoscaler + PgExternalHost string + PgExternalPort string } // NewReconciler initializes a reconciler to be used for loading or reconciling a noobaa system @@ -281,6 +286,7 @@ func NewReconciler( r.SecretDB.StringData["user"] = "noobaa" r.SecretDB.StringData["password"] = util.RandomBase64(10) + r.SecretDB.StringData["db_name"] = "nbcore" // Set STS default backing store session name r.AWSSTSRoleSessionName = "noobaa-sts-default-backing-store-session" @@ -306,10 +312,12 @@ func (r *Reconciler) CheckAll() { if r.NooBaa.Spec.MongoDbURL == "" { if r.NooBaa.Spec.DBType == "postgres" { util.KubeCheck(r.SecretDB) - util.KubeCheck(r.PostgresDBConf) - util.KubeCheck(r.PostgresDBInitDb) - util.KubeCheck(r.NooBaaPostgresDB) - util.KubeCheck(r.ServiceDbPg) + if r.NooBaa.Spec.ExternalPgSecret == nil { + util.KubeCheck(r.PostgresDBConf) + util.KubeCheck(r.PostgresDBInitDb) + util.KubeCheck(r.NooBaaPostgresDB) + util.KubeCheck(r.ServiceDbPg) + } } else { util.KubeCheck(r.NooBaaMongoDB) util.KubeCheck(r.ServiceDb) @@ -386,6 +394,34 @@ func (r *Reconciler) Reconcile() (reconcile.Result, error) { return res, nil } + if r.NooBaa.Spec.ExternalPgSecret != nil { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.NooBaa.Spec.ExternalPgSecret.Namespace, + Name: r.NooBaa.Spec.ExternalPgSecret.Name, + }, + } + if !util.KubeCheck(secret) { + log.Errorf("❌ External DB secret %q was not found or deleted", r.NooBaa.Spec.ExternalPgSecret.Name) + return res, nil + } + u, err := url.Parse(secret.StringData["db_url"]) + if err != nil { + log.Errorf("❌ Failed pasting external DB url in secret: %q", r.NooBaa.Spec.ExternalPgSecret.Name) + return res, nil + } + r.SecretDB.StringData["user"] = u.User.Username() + r.SecretDB.StringData["password"], _ = u.User.Password() + r.SecretDB.StringData["dbname"] = u.Path[1:] + r.PgExternalHost = u.Host + r.PgExternalPort = "5432" + host, port, err := net.SplitHostPort(u.Host) + if err == nil { + r.PgExternalHost = host + r.PgExternalPort = port + } + } + err = r.ReconcilePhases() if err != nil { diff --git a/pkg/system/system.go b/pkg/system/system.go index f2399e4b70..cd9470956c 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -297,6 +297,29 @@ func RunCreate(cmd *cobra.Command, args []string) { log.Fatalf(`❌ %s`, err) } + if options.PostgresDbURL != "" { + if sys.Spec.DBType != "postgres" { + log.Fatalf("❌ expecting the DBType to be postgres when using external PostgresDbURL, got %s", sys.Spec.DBType) + } + err = CheckPostgresURL(options.PostgresDbURL) + if err != nil { + log.Fatalf(`❌ %s`, err) + } + o := util.KubeObject(bundle.File_deploy_internal_secret_empty_yaml) + secret := o.(*corev1.Secret) + secret.Namespace = options.Namespace + secret.Name = "noobaa-external-db" + secret.StringData = map[string]string{ + "db_url": options.PostgresDbURL, + } + secret.Data = nil + util.KubeCreateSkipExisting(secret) + sys.Spec.ExternalPgSecret = &corev1.SecretReference{ + Name: secret.Name, + Namespace: secret.Namespace, + } + } + // TODO check PVC if exist and the system does not exist - // fail and suggest to delete them first with cli system delete. util.KubeCreateSkipExisting(ns) @@ -353,9 +376,21 @@ func RunDelete(cmd *cobra.Command, args []string) { isMongoDbURL := sys.Spec.MongoDbURL util.KubeDelete(sys) - if isMongoDbURL == "" { + if sys.Spec.ExternalPgSecret != nil { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "noobaa-external-db", + Namespace: options.Namespace, + }, + } + + util.KubeDelete(secret) + } else if isMongoDbURL == "" { // NoobaaDB noobaaDB := util.KubeObject(bundle.File_deploy_internal_statefulset_db_yaml).(*appsv1.StatefulSet) + if sys.Spec.DBType == "postgres" { + noobaaDB = util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet) + } for i := range noobaaDB.Spec.VolumeClaimTemplates { t := &noobaaDB.Spec.VolumeClaimTemplates[i] pvc := &corev1.PersistentVolumeClaim{ @@ -600,7 +635,7 @@ func RunStatus(cmd *cobra.Command, args []string) { NooBaaDB = r.NooBaaMongoDB } // create the mongo db only if mongo db url is not given. - if r.NooBaa.Spec.MongoDbURL == "" { + if r.NooBaa.Spec.MongoDbURL == "" && r.NooBaa.Spec.ExternalPgSecret == nil { // NoobaaDB for i := range NooBaaDB.Spec.VolumeClaimTemplates { t := &NooBaaDB.Spec.VolumeClaimTemplates[i] @@ -1055,16 +1090,25 @@ func CheckSystem(sys *nbv1.NooBaa) bool { func CheckMongoURL(sys *nbv1.NooBaa) error { if sys.Spec.MongoDbURL != "" { if sys.Spec.DBType != "mongodb" { - return fmt.Errorf("Expecting the DBType to be mongodb when using external MongoDbURL, got %s", sys.Spec.DBType) + return fmt.Errorf("expecting the DBType to be mongodb when using external MongoDbURL, got %s", sys.Spec.DBType) } if !strings.Contains(sys.Spec.MongoDbURL, "mongodb://") && !strings.Contains(sys.Spec.MongoDbURL, "mongodb+srv://") { - return fmt.Errorf("Invalid mongo db url %s, expecting the url to start with mongodb:// or mongodb+srv://", sys.Spec.MongoDbURL) + return fmt.Errorf("invalid mongo db url %s, expecting the url to start with mongodb:// or mongodb+srv://", sys.Spec.MongoDbURL) } } return nil } +// CheckPostgresURL checks if the postgresurl structure is valid and if we use postgres as db +func CheckPostgresURL(postgresDbURL string) error { + if !strings.Contains(postgresDbURL, "postgres://") && + !strings.Contains(postgresDbURL, "postgresql://") { + return fmt.Errorf("invalid postgres db url %s, expecting the url to start with postgres:// or postgresql://", postgresDbURL) + } + return nil +} + // LoadConfigMapFromFlags loads a config-map with values from the cli flags, if provided. func LoadConfigMapFromFlags() { if options.DebugLevel != "default_level" {