db upgrage: upgrade postgres 15 to postgres 16

The Postgres upgrade from 15 to 16 or any later versions can be done by adding the env variable named POSTGRES_UPGRADE = copy in the pg db pod. More about upgrade process is here: https://www.postgresql.org/docs/current/pgupgrade.html In our case, we achieve the upgrade by setting this env var in noobaa-db-pg STS. Once after upgrade is done, we have to remove the POSTGRES_UPGRADE env from the STS. Otherwise pg db pod fails to come up because the db pod check PG directory version against upgrade image version and exits if there is no upgrade required and POSTGRES_UPGRADE is still provided. To remove this env, we store the upgrade status in noobaa-core CR after the PG upgrade and refer the same status during db reconcile. If the upgrade is already done, we remove this env from desired sts status. This will help to upgrade the db smoothly Signed-off-by: Vinayak Hariharmath <[email protected]>
noobaa · Nov 5, 2024 · 1db13e5 · 1db13e5
1 parent 1fc5447
commit 1db13e5
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 28 deletions.
diff --git a/deploy/internal/statefulset-postgres-db.yaml b/deploy/internal/statefulset-postgres-db.yaml
@@ -26,6 +26,8 @@ spec:
         - name: db
           image: NOOBAA_DB_IMAGE
           env:
+            - name: POSTGRESQL_UPGRADE
+              value: copy
             - name: POSTGRESQL_DATABASE
               value: nbcore
             - name: LC_COLLATE

diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go
@@ -5070,7 +5070,7 @@ spec:
                   resource: limits.memory
 `
 
-const Sha256_deploy_internal_statefulset_postgres_db_yaml = "37a6c36928ba426ca04fd89e1eb2685e10d1a5f65c63ebb40c68a4f5c37645de"
+const Sha256_deploy_internal_statefulset_postgres_db_yaml = "46460879b565c80900a09017d145e835ca0491a0c9e465ff0c8b9172535843fe"
 
 const File_deploy_internal_statefulset_postgres_db_yaml = `apiVersion: apps/v1
 kind: StatefulSet
@@ -5100,6 +5100,8 @@ spec:
         - name: db
           image: NOOBAA_DB_IMAGE
           env:
+            - name: POSTGRESQL_UPGRADE
+              value: copy
             - name: POSTGRESQL_DATABASE
               value: nbcore
             - name: LC_COLLATE

diff --git a/pkg/options/options.go b/pkg/options/options.go
@@ -72,7 +72,7 @@ var NooBaaImage = ContainerImage
 
 // DBImage is the default db image url
 // it can be overridden for testing or different registry locations.
-var DBImage = "quay.io/sclorg/postgresql-15-c9s"
+var DBImage = "quay.io/sclorg/postgresql-16-c9s"
 
 // Psql12Image is the default postgres12 db image url
 // currently it can not be overridden.

diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go
@@ -223,6 +223,80 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error {
 	return nil
 }
 
+// Check db update status and add or remove the POSTGRES_UPDATE env from STS
+func (r *Reconciler) addOrRemovePGUpgradeEnvInSTS() {
+	if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseFinished {
+		for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+			if container.Name == "db" {
+				envVars := container.Env
+				newEnvVar := corev1.EnvVar{
+					Name:  "POSTGRESQL_UPGRADE",
+					Value: "copy",
+				}
+				envVars = append(envVars, newEnvVar)
+				r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars
+
+				break
+			}
+		}
+
+		return
+	}
+
+	if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseFinished {
+		for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+			if container.Name == "db" {
+				newEnvVars := []corev1.EnvVar{}
+				for _, env := range container.Env {
+					if env.Name != "POSTGRESQL_UPGRADE" {
+						newEnvVars = append(newEnvVars, env)
+					}
+				}
+				r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars
+
+				break
+			}
+		}
+	}
+}
+
+func (r *Reconciler) isPostgresDBUpgradeReqd() bool {
+	if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseFinished {
+		for _, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+			if container.Name == "db" {
+				for _, env := range container.Env {
+					if env.Name == "POSTGRESQL_UPGRADE" && env.Value == "copy" {
+						return true
+					}
+				}
+			}
+		}
+	}
+	return false
+}
+
+// GetDesiredDBImage returns the desired DB image according to spec or env or default (in options)
+func (r *Reconciler) GetDesiredDBImage(currentImage string) string {
+	// Postgres upgrade failure workaround
+	// if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage
+	if IsPostgresql12Image(currentImage) {
+		psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE")
+		util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image)
+		if !ok {
+			psql12Image = currentImage
+			util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage)
+		}
+		return psql12Image
+	}
+
+	if r.isPostgresDBUpgradeReqd() {
+		r.NooBaa.Status.BeforeUpgradeDbImage = r.NooBaa.Spec.DBImage
+		r.NooBaa.Spec.DBImage = &options.DBImage
+	}
+
+	return options.DBImage
+}
+
 // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling
 func (r *Reconciler) SetDesiredNooBaaDB() error {
 	var NooBaaDBTemplate *appsv1.StatefulSet = nil
@@ -239,6 +313,9 @@ func (r *Reconciler) SetDesiredNooBaaDB() error {
 	NooBaaDB.Spec.ServiceName = r.ServiceDbPg.Name
 	NooBaaDBTemplate = util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet)
 
+	// Check db update status and add or remove the POSTGRES_UPDATE env to db STS
+	r.addOrRemovePGUpgradeEnvInSTS()
+
 	podSpec := &NooBaaDB.Spec.Template.Spec
 	podSpec.ServiceAccountName = "noobaa-db"
 	defaultUID := int64(10001)
@@ -256,7 +333,7 @@ func (r *Reconciler) SetDesiredNooBaaDB() error {
 	for i := range podSpec.Containers {
 		c := &podSpec.Containers[i]
 		if c.Name == "db" {
-			c.Image = GetDesiredDBImage(r.NooBaa, c.Image)
+			c.Image = r.GetDesiredDBImage(c.Image)
 			if r.NooBaa.Spec.DBResources != nil {
 				c.Resources = *r.NooBaa.Spec.DBResources
 			}
@@ -1133,6 +1210,7 @@ func (r *Reconciler) ReconcileDB() error {
 
 	result, reconcilePostgresError := r.reconcileObjectAndGetResult(r.NooBaaPostgresDB, r.SetDesiredNooBaaDB, false)
 	if reconcilePostgresError != nil {
+
 		return reconcilePostgresError
 	}
 	if !r.isObjectUpdated(result) && (isDBConfUpdated) {
@@ -1143,6 +1221,11 @@ func (r *Reconciler) ReconcileDB() error {
 		}
 
 	}
+
+	if r.isObjectUpdated(result) {
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished
+	}
+
 	return nil
 }
 

diff --git a/pkg/system/system.go b/pkg/system/system.go
@@ -1115,31 +1115,6 @@ func Connect(isExternal bool) (*Client, error) {
 	}, nil
 }
 
-// GetDesiredDBImage returns the desired DB image according to spec or env or default (in options)
-func GetDesiredDBImage(sys *nbv1.NooBaa, currentImage string) string {
-	// Postgres upgrade failure workaround
-	// if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage
-	if IsPostgresql12Image(currentImage) {
-		psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE")
-		util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image)
-		if !ok {
-			psql12Image = currentImage
-			util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage)
-		}
-		return psql12Image
-	}
-
-	if sys.Spec.DBImage != nil {
-		return *sys.Spec.DBImage
-	}
-
-	if os.Getenv("NOOBAA_DB_IMAGE") != "" {
-		return os.Getenv("NOOBAA_DB_IMAGE")
-	}
-
-	return options.DBImage
-}
-
 // IsPostgresql12Image checks if the image is a postgresql-12 image
 func IsPostgresql12Image(image string) bool {
 	return strings.Contains(image, "postgresql-12")