diff --git a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml index 8c0958d2ea..7982ea4dcf 100644 --- a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml +++ b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml @@ -8171,8 +8171,6 @@ spec: - azure type: string type: object - required: - - image type: object image: description: The image name to use for PostgreSQL containers. @@ -17511,6 +17509,10 @@ spec: properties: host: type: string + installedCustomExtensions: + items: + type: string + type: array pgbouncer: properties: ready: diff --git a/build/postgres-operator/Dockerfile b/build/postgres-operator/Dockerfile index d7776c2133..7c97c7dcb0 100644 --- a/build/postgres-operator/Dockerfile +++ b/build/postgres-operator/Dockerfile @@ -11,7 +11,7 @@ ARG GO_LDFLAGS ARG GOOS=linux ARG TARGETARCH ARG OPERATOR_CGO_ENABLED=1 -ARG EXTENSION_INSTALLER_CGO_ENABLED=0 +ARG EXTENSION_INSTALLER_CGO_ENABLED=1 ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml index aa4cfdaa03..4e2719326b 100644 --- a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml +++ b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml @@ -8577,8 +8577,6 @@ spec: - azure type: string type: object - required: - - image type: object image: description: The image name to use for PostgreSQL containers. @@ -17917,6 +17915,10 @@ spec: properties: host: type: string + installedCustomExtensions: + items: + type: string + type: array pgbouncer: properties: ready: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 15eccf143d..2df0ccf93a 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -8870,8 +8870,6 @@ spec: - azure type: string type: object - required: - - image type: object image: description: The image name to use for PostgreSQL containers. @@ -18210,6 +18208,10 @@ spec: properties: host: type: string + installedCustomExtensions: + items: + type: string + type: array pgbouncer: properties: ready: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index ed3e3b1bfb..8f6116adf4 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -8870,8 +8870,6 @@ spec: - azure type: string type: object - required: - - image type: object image: description: The image name to use for PostgreSQL containers. @@ -18210,6 +18208,10 @@ spec: properties: host: type: string + installedCustomExtensions: + items: + type: string + type: array pgbouncer: properties: ready: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 74e1f91779..e5fb4fc4a8 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -8870,8 +8870,6 @@ spec: - azure type: string type: object - required: - - image type: object image: description: The image name to use for PostgreSQL containers. @@ -18210,6 +18208,10 @@ spec: properties: host: type: string + installedCustomExtensions: + items: + type: string + type: array pgbouncer: properties: ready: diff --git a/e2e-tests/tests/custom-extensions/09-assert.yaml b/e2e-tests/tests/custom-extensions/09-assert.yaml new file mode 100644 index 0000000000..1217de376b --- /dev/null +++ b/e2e-tests/tests/custom-extensions/09-assert.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 11-check-extensions +data: + data: |2- + pg_stat_monitor + pgaudit + plpgsql diff --git a/e2e-tests/tests/custom-extensions/09-check-installed-extensions.yaml b/e2e-tests/tests/custom-extensions/09-check-installed-extensions.yaml new file mode 100644 index 0000000000..fa5607a324 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/09-check-installed-extensions.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + data=$(kubectl -n ${NAMESPACE} exec $(get_client_pod) -- psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres) -c "\c postgres" -c "select extname from pg_extension order by extname") + + kubectl create configmap -n "${NAMESPACE}" 11-check-extensions --from-literal=data="${data}" diff --git a/go.mod b/go.mod index 8ce277bbf8..34d986f8ab 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 - github.com/onsi/ginkgo/v2 v2.22.1 + github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 github.com/pganalyze/pg_query_go/v5 v5.1.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 14750da510..e71933329e 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.15.2 h1:l77YT15o814C2qVL47NOyjV/6RbaP7kKdrvZnxQ3Org= github.com/onsi/ginkgo v1.15.2/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o= -github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= -github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/percona/controller/pgcluster/controller.go b/percona/controller/pgcluster/controller.go index a09cc39b82..1bfa8837e6 100644 --- a/percona/controller/pgcluster/controller.go +++ b/percona/controller/pgcluster/controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" "fmt" + "io" "reflect" "strings" "time" @@ -30,13 +31,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/percona/percona-postgresql-operator/internal/controller/runtime" "github.com/percona/percona-postgresql-operator/internal/logging" "github.com/percona/percona-postgresql-operator/internal/naming" + "github.com/percona/percona-postgresql-operator/internal/postgres" perconaController "github.com/percona/percona-postgresql-operator/percona/controller" "github.com/percona/percona-postgresql-operator/percona/extensions" "github.com/percona/percona-postgresql-operator/percona/k8s" pNaming "github.com/percona/percona-postgresql-operator/percona/naming" "github.com/percona/percona-postgresql-operator/percona/pmm" + perconaPG "github.com/percona/percona-postgresql-operator/percona/postgres" "github.com/percona/percona-postgresql-operator/percona/utils/registry" "github.com/percona/percona-postgresql-operator/percona/watcher" v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" @@ -50,8 +54,12 @@ const ( // Reconciler holds resources for the PerconaPGCluster reconciler type PGClusterReconciler struct { - Client client.Client - Owner client.FieldOwner + Client client.Client + Owner client.FieldOwner + PodExec func( + ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error Recorder record.EventRecorder Tracer trace.Tracer Platform string @@ -66,6 +74,13 @@ type PGClusterReconciler struct { // SetupWithManager adds the PerconaPGCluster controller to the provided runtime manager func (r *PGClusterReconciler) SetupWithManager(mgr manager.Manager) error { + if r.PodExec == nil { + var err error + r.PodExec, err = runtime.NewPodExecutor(mgr.GetConfig()) + if err != nil { + return err + } + } if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}, r.watchSecrets())); err != nil { return errors.Wrap(err, "unable to watch secrets") } @@ -242,7 +257,9 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, errors.Wrap(err, "failed to handle monitor user password change") } - r.reconcileCustomExtensions(cr) + if err := r.reconcileCustomExtensions(ctx, cr); err != nil { + return reconcile.Result{}, errors.Wrap(err, "reconcile custom extensions") + } if err := r.reconcileScheduledBackups(ctx, cr); err != nil { return reconcile.Result{}, errors.Wrap(err, "reconcile scheduled backups") @@ -535,15 +552,57 @@ func (r *PGClusterReconciler) handleMonitorUserPassChange(ctx context.Context, c return nil } -func (r *PGClusterReconciler) reconcileCustomExtensions(cr *v2.PerconaPGCluster) { +func (r *PGClusterReconciler) reconcileCustomExtensions(ctx context.Context, cr *v2.PerconaPGCluster) error { if cr.Spec.Extensions.Storage.Secret == nil { - return + return nil + } + + if len(cr.Spec.Extensions.Image) == 0 && len(cr.Spec.Extensions.Custom) > 0 { + return errors.New("you need to set spec.extensions.image to install custom extensions") } extensionKeys := make([]string, 0) + extensionNames := make([]string, 0) + for _, extension := range cr.Spec.Extensions.Custom { key := extensions.GetExtensionKey(cr.Spec.PostgresVersion, extension.Name, extension.Version) extensionKeys = append(extensionKeys, key) + extensionNames = append(extensionNames, extension.Name) + } + + if cr.CompareVersion("2.6.0") >= 0 { + var removedExtension []string + installedExtensions := cr.Status.InstalledCustomExtensions + crExtensions := make(map[string]struct{}) + for _, ext := range extensionNames { + crExtensions[ext] = struct{}{} + } + + // Check for missing entries in crExtensions + for _, ext := range installedExtensions { + // If an object exists in installedExtensions but not in crExtensions, the extension should be deleted. + if _, ok := crExtensions[ext]; !ok { + removedExtension = append(removedExtension, ext) + } + } + + if len(removedExtension) > 0 { + disable := func(ctx context.Context, exec postgres.Executor) error { + return errors.WithStack(disableCustomExtensionsInDB(ctx, exec, removedExtension)) + } + + primary, err := perconaPG.GetPrimaryPod(ctx, r.Client, cr) + if err != nil { + return errors.New("primary pod not found") + } + + err = disable(ctx, func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error { + return r.PodExec(ctx, primary.Namespace, primary.Name, naming.ContainerDatabase, stdin, stdout, stderr, command...) + }) + if err != nil { + return errors.Wrap(err, "deletion extension from installed") + } + } } for i := 0; i < len(cr.Spec.InstanceSets); i++ { @@ -560,6 +619,31 @@ func (r *PGClusterReconciler) reconcileCustomExtensions(cr *v2.PerconaPGCluster) )) set.VolumeMounts = append(set.VolumeMounts, extensions.ExtensionVolumeMounts(cr.Spec.PostgresVersion)...) } + return nil +} + +func disableCustomExtensionsInDB(ctx context.Context, exec postgres.Executor, customExtensionsForDeletion []string) error { + log := logging.FromContext(ctx) + + for _, extensionName := range customExtensionsForDeletion { + sqlCommand := fmt.Sprintf( + `SET client_min_messages = WARNING; DROP EXTENSION IF EXISTS %s;`, + extensionName, + ) + _, _, err := exec.ExecInAllDatabases(ctx, + sqlCommand, + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one command fails. + "QUIET": "on", // Do not print successful commands to stdout. + }, + ) + + log.V(1).Info("extension was disabled ", "extensionName", extensionName) + + return errors.Wrap(err, "custom extension deletion") + } + + return nil } func isBackupRunning(ctx context.Context, cl client.Reader, cr *v2.PerconaPGCluster) (bool, error) { diff --git a/percona/controller/pgcluster/status.go b/percona/controller/pgcluster/status.go index 477fe2a604..30123abd6a 100644 --- a/percona/controller/pgcluster/status.go +++ b/percona/controller/pgcluster/status.go @@ -2,7 +2,6 @@ package pgcluster import ( "context" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -78,6 +77,11 @@ func (r *PGClusterReconciler) updateStatus(ctx context.Context, cr *v2.PerconaPG return errors.Wrap(err, "get app host") } + installedCustomExtensions := make([]string, 0) + for _, extension := range cr.Spec.Extensions.Custom { + installedCustomExtensions = append(installedCustomExtensions, extension.Name) + } + var size, ready int32 ss := make([]v2.PostgresInstanceSetStatus, 0, len(status.InstanceSets)) for _, is := range status.InstanceSets { @@ -110,7 +114,8 @@ func (r *PGClusterReconciler) updateStatus(ctx context.Context, cr *v2.PerconaPG Size: status.Proxy.PGBouncer.Replicas, Ready: status.Proxy.PGBouncer.ReadyReplicas, }, - Host: host, + Host: host, + InstalledCustomExtensions: installedCustomExtensions, } cluster.Status.State = r.getState(cr, &cluster.Status, status) diff --git a/percona/postgres/common.go b/percona/postgres/common.go new file mode 100644 index 0000000000..9997e05bdd --- /dev/null +++ b/percona/postgres/common.go @@ -0,0 +1,36 @@ +package perconaPG + +import ( + "context" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" +) + +func GetPrimaryPod(ctx context.Context, cli client.Client, cr *v2.PerconaPGCluster) (*corev1.Pod, error) { + podList := &corev1.PodList{} + err := cli.List(ctx, podList, &client.ListOptions{ + Namespace: cr.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + "app.kubernetes.io/instance": cr.Name, + "postgres-operator.crunchydata.com/role": "master", + }), + }) + if err != nil { + return nil, err + } + + if len(podList.Items) == 0 { + return nil, errors.New("no primary pod found") + } + + if len(podList.Items) > 1 { + return nil, errors.New("multiple primary pods found") + } + + return &podList.Items[0], nil +} diff --git a/percona/watcher/wal.go b/percona/watcher/wal.go index 65787d1b3b..797172a5fc 100644 --- a/percona/watcher/wal.go +++ b/percona/watcher/wal.go @@ -7,16 +7,15 @@ import ( "time" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "github.com/percona/percona-postgresql-operator/internal/logging" "github.com/percona/percona-postgresql-operator/percona/clientcmd" "github.com/percona/percona-postgresql-operator/percona/pgbackrest" + perconaPG "github.com/percona/percona-postgresql-operator/percona/postgres" pgv2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" ) @@ -153,7 +152,7 @@ func getLatestBackup(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGC func GetLatestCommitTimestamp(ctx context.Context, cli client.Client, execCli *clientcmd.Client, cr *pgv2.PerconaPGCluster, backup *pgv2.PerconaPGBackup) (*metav1.Time, error) { log := logging.FromContext(ctx) - primary, err := getPrimaryPod(ctx, cli, cr) + primary, err := perconaPG.GetPrimaryPod(ctx, cli, cr) if err != nil { return nil, PrimaryPodNotFound } @@ -188,32 +187,8 @@ func GetLatestCommitTimestamp(ctx context.Context, cli client.Client, execCli *c return &commitTsMeta, nil } -func getPrimaryPod(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGCluster) (*corev1.Pod, error) { - podList := &corev1.PodList{} - err := cli.List(ctx, podList, &client.ListOptions{ - Namespace: cr.Namespace, - LabelSelector: labels.SelectorFromSet(map[string]string{ - "app.kubernetes.io/instance": cr.Name, - "postgres-operator.crunchydata.com/role": "master", - }), - }) - if err != nil { - return nil, err - } - - if len(podList.Items) == 0 { - return nil, errors.New("no primary pod found") - } - - if len(podList.Items) > 1 { - return nil, errors.New("multiple primary pods found") - } - - return &podList.Items[0], nil -} - func getBackupStartTimestamp(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGCluster, backup *pgv2.PerconaPGBackup) (time.Time, error) { - primary, err := getPrimaryPod(ctx, cli, cr) + primary, err := perconaPG.GetPrimaryPod(ctx, cli, cr) if err != nil { return time.Time{}, PrimaryPodNotFound } diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 0c49e631fc..50e18ed1ea 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -417,6 +417,10 @@ type PerconaPGClusterStatus struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=status Host string `json:"host"` + + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=status + InstalledCustomExtensions []string `json:"installedCustomExtensions"` } type Backups struct { @@ -580,8 +584,7 @@ type BuiltInExtensionsSpec struct { } type ExtensionsSpec struct { - // +kubebuilder:validation:Required - Image string `json:"image"` + Image string `json:"image,omitempty"` ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Storage CustomExtensionsStorageSpec `json:"storage,omitempty"` BuiltIn BuiltInExtensionsSpec `json:"builtin,omitempty"` diff --git a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go index d8bc5d3b7f..1c482ad48d 100644 --- a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go +++ b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go @@ -757,6 +757,11 @@ func (in *PerconaPGClusterStatus) DeepCopyInto(out *PerconaPGClusterStatus) { *out = *in in.Postgres.DeepCopyInto(&out.Postgres) out.PGBouncer = in.PGBouncer + if in.InstalledCustomExtensions != nil { + in, out := &in.InstalledCustomExtensions, &out.InstalledCustomExtensions + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerconaPGClusterStatus.