K8SPG-594 delete custom extensions from installed

build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml

@@ -17494,6 +17494,10 @@ spec:
             properties:
               host:
                 type: string
+              installedCustomExtensions:
+                items:
+                  type: string
+                type: array
               pgbouncer:
                 properties:
                   ready:

build/postgres-operator/Dockerfile

@@ -11,7 +11,7 @@ ARG GO_LDFLAGS
 ARG GOOS=linux
 ARG TARGETARCH
 ARG OPERATOR_CGO_ENABLED=1
-ARG EXTENSION_INSTALLER_CGO_ENABLED=0
+ARG EXTENSION_INSTALLER_CGO_ENABLED=1
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 

config/crd/bases/pgv2.percona.com_perconapgclusters.yaml

@@ -17900,6 +17900,10 @@ spec:
             properties:
               host:
                 type: string
+              installedCustomExtensions:
+                items:
+                  type: string
+                type: array
               pgbouncer:
                 properties:
                   ready:

deploy/bundle.yaml

@@ -18193,6 +18193,10 @@ spec:
             properties:
               host:
                 type: string
+              installedCustomExtensions:
+                items:
+                  type: string
+                type: array
               pgbouncer:
                 properties:
                   ready:

deploy/crd.yaml

@@ -18193,6 +18193,10 @@ spec:
             properties:
               host:
                 type: string
+              installedCustomExtensions:
+                items:
+                  type: string
+                type: array
               pgbouncer:
                 properties:
                   ready:

deploy/cw-bundle.yaml

@@ -18193,6 +18193,10 @@ spec:
             properties:
               host:
                 type: string
+              installedCustomExtensions:
+                items:
+                  type: string
+                type: array
               pgbouncer:
                 properties:
                   ready:

e2e-tests/tests/custom-extensions/09-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 11-check-extensions
+data:
+  data: |2-
+     pg_stat_monitor
+     pgaudit

e2e-tests/tests/custom-extensions/09-check-installed-extensions.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      data=$(kubectl -n ${NAMESPACE} exec $(get_client_pod) -- psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres) -c "\c postgres" -c "select extname from pg_extension order by extname")
+
+      kubectl create configmap -n "${NAMESPACE}" 11-check-extensions --from-literal=data="${data}"

go.mod

@@ -111,7 +111,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect

go.sum

@@ -295,6 +295,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
 golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
 golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=

percona/controller/pgcluster/controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"io"
 	"reflect"
 	"strings"
 	"time"
@@ -29,13 +30,16 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	"github.com/percona/percona-postgresql-operator/internal/controller/runtime"
 	"github.com/percona/percona-postgresql-operator/internal/logging"
 	"github.com/percona/percona-postgresql-operator/internal/naming"
+	"github.com/percona/percona-postgresql-operator/internal/postgres"
 	perconaController "github.com/percona/percona-postgresql-operator/percona/controller"
 	"github.com/percona/percona-postgresql-operator/percona/extensions"
 	"github.com/percona/percona-postgresql-operator/percona/k8s"
 	pNaming "github.com/percona/percona-postgresql-operator/percona/naming"
 	"github.com/percona/percona-postgresql-operator/percona/pmm"
+	common "github.com/percona/percona-postgresql-operator/percona/postgres"
 	"github.com/percona/percona-postgresql-operator/percona/utils/registry"
 	"github.com/percona/percona-postgresql-operator/percona/watcher"
 	v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2"
@@ -49,8 +53,12 @@ const (
 
 // Reconciler holds resources for the PerconaPGCluster reconciler
 type PGClusterReconciler struct {
-	Client               client.Client
-	Owner                client.FieldOwner
+	Client  client.Client
+	Owner   client.FieldOwner
+	PodExec func(
+		ctx context.Context, namespace, pod, container string,
+		stdin io.Reader, stdout, stderr io.Writer, command ...string,
+	) error
 	Recorder             record.EventRecorder
 	Tracer               trace.Tracer
 	Platform             string
@@ -65,6 +73,13 @@ type PGClusterReconciler struct {
 
 // SetupWithManager adds the PerconaPGCluster controller to the provided runtime manager
 func (r *PGClusterReconciler) SetupWithManager(mgr manager.Manager) error {
+	if r.PodExec == nil {
+		var err error
+		r.PodExec, err = runtime.NewPodExecutor(mgr.GetConfig())
+		if err != nil {
+			return err
+		}
+	}
 	if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}, r.watchSecrets())); err != nil {
 		return errors.Wrap(err, "unable to watch secrets")
 	}
@@ -241,7 +256,9 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 		return reconcile.Result{}, errors.Wrap(err, "failed to handle monitor user password change")
 	}
 
-	r.reconcileCustomExtensions(cr)
+	if err := r.reconcileCustomExtensions(ctx, cr); err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "reconcile custom extensions")
+	}
 
 	if err := r.reconcileScheduledBackups(ctx, cr); err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "reconcile scheduled backups")
@@ -524,15 +541,54 @@ func (r *PGClusterReconciler) handleMonitorUserPassChange(ctx context.Context, c
 	return nil
 }
 
-func (r *PGClusterReconciler) reconcileCustomExtensions(cr *v2.PerconaPGCluster) {
+func (r *PGClusterReconciler) reconcileCustomExtensions(ctx context.Context, cr *v2.PerconaPGCluster) error {
 	if cr.Spec.Extensions.Storage.Secret == nil {
-		return
+		return nil
 	}
 
 	extensionKeys := make([]string, 0)
+	extensionNames := make([]string, 0)
+
 	for _, extension := range cr.Spec.Extensions.Custom {
 		key := extensions.GetExtensionKey(cr.Spec.PostgresVersion, extension.Name, extension.Version)
 		extensionKeys = append(extensionKeys, key)
+		extensionNames = append(extensionNames, extension.Name)
+	}
+
+	if cr.CompareVersion("2.6.0") >= 0 {
+		var removedExtension []string
+		installedExtensions := cr.Status.InstalledCustomExtensions
+		crExtensions := make(map[string]struct{})
+		for _, ext := range extensionNames {
+			crExtensions[ext] = struct{}{}
+		}
+
+		// Check for missing entries in crExtensions
+		for _, ext := range installedExtensions {
+			// If an object exists in installedExtensions but not in crExtensions, the extension should be deleted.
+			if _, ok := crExtensions[ext]; !ok {
+				removedExtension = append(removedExtension, ext)
+			}
+		}
+
+		if len(removedExtension) > 0 {
+
+			disabled := func(ctx context.Context, exec postgres.Executor) error {
+				return errors.WithStack(disableCustomExtensionsInDB(ctx, exec, removedExtension))
+			}
+
+			primary, err := common.GetPrimaryPod(ctx, r.Client, cr)
+			if err != nil {
+				return errors.New("primary pod not found")
+			}
+
+			err = disabled(ctx, func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error {
+				return r.PodExec(ctx, primary.Namespace, primary.Name, naming.ContainerDatabase, stdin, stdout, stderr, command...)
+			})
+			if err != nil {
+				return errors.Wrap(err, "deletion extension from installed")
+			}
+		}
 	}
 
 	for i := 0; i < len(cr.Spec.InstanceSets); i++ {
@@ -549,6 +605,31 @@ func (r *PGClusterReconciler) reconcileCustomExtensions(cr *v2.PerconaPGCluster)
 		))
 		set.VolumeMounts = append(set.VolumeMounts, extensions.ExtensionVolumeMounts(cr.Spec.PostgresVersion)...)
 	}
+	return nil
+}
+
+func disableCustomExtensionsInDB(ctx context.Context, exec postgres.Executor, customExtensionsForDeletion []string) error {
+	log := logging.FromContext(ctx)
+
+	for _, extensionName := range customExtensionsForDeletion {
+		sqlCommand := fmt.Sprintf(
+			`SET client_min_messages = WARNING; DROP EXTENSION IF EXISTS %s;`,
+			extensionName,
+		)
+		_, _, err := exec.ExecInAllDatabases(ctx,
+			sqlCommand,
+			map[string]string{
+				"ON_ERROR_STOP": "on", // Abort when any one command fails.
+				"QUIET":         "on", // Do not print successful commands to stdout.
+			},
+		)
+
+		log.V(1).Info("extension was disabled ", "extensionName", extensionName)
+
+		return errors.Wrap(err, "custom extension deletion")
+	}
+
+	return nil
 }
 
 func isBackupRunning(ctx context.Context, cl client.Reader, cr *v2.PerconaPGCluster) (bool, error) {

percona/controller/pgcluster/status.go

@@ -2,7 +2,6 @@ package pgcluster
 
 import (
 	"context"
-
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -78,6 +77,11 @@ func (r *PGClusterReconciler) updateStatus(ctx context.Context, cr *v2.PerconaPG
 		return errors.Wrap(err, "get app host")
 	}
 
+	installedCustomExtensions := make([]string, 0)
+	for _, extension := range cr.Spec.Extensions.Custom {
+		installedCustomExtensions = append(installedCustomExtensions, extension.Name)
+	}
+
 	var size, ready int32
 	ss := make([]v2.PostgresInstanceSetStatus, 0, len(status.InstanceSets))
 	for _, is := range status.InstanceSets {
@@ -110,7 +114,8 @@ func (r *PGClusterReconciler) updateStatus(ctx context.Context, cr *v2.PerconaPG
 				Size:  status.Proxy.PGBouncer.Replicas,
 				Ready: status.Proxy.PGBouncer.ReadyReplicas,
 			},
-			Host: host,
+			Host:                      host,
+			InstalledCustomExtensions: installedCustomExtensions,
 		}
 
 		cluster.Status.State = r.getState(cr, &cluster.Status, status)

percona/postgres/common.go

@@ -0,0 +1,36 @@
+package postgres
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2"
+)
+
+func GetPrimaryPod(ctx context.Context, cli client.Client, cr *v2.PerconaPGCluster) (*corev1.Pod, error) {
+	podList := &corev1.PodList{}
+	err := cli.List(ctx, podList, &client.ListOptions{
+		Namespace: cr.Namespace,
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			"app.kubernetes.io/instance":             cr.Name,
+			"postgres-operator.crunchydata.com/role": "master",
+		}),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(podList.Items) == 0 {
+		return nil, errors.New("no primary pod found")
+	}
+
+	if len(podList.Items) > 1 {
+		return nil, errors.New("multiple primary pods found")
+	}
+
+	return &podList.Items[0], nil
+}

percona/watcher/wal.go

@@ -7,16 +7,15 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 
 	"github.com/percona/percona-postgresql-operator/internal/logging"
 	"github.com/percona/percona-postgresql-operator/percona/clientcmd"
 	"github.com/percona/percona-postgresql-operator/percona/pgbackrest"
+	common "github.com/percona/percona-postgresql-operator/percona/postgres"
 	pgv2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2"
 )
 
@@ -153,7 +152,7 @@ func getLatestBackup(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGC
 func GetLatestCommitTimestamp(ctx context.Context, cli client.Client, execCli *clientcmd.Client, cr *pgv2.PerconaPGCluster, backup *pgv2.PerconaPGBackup) (*metav1.Time, error) {
 	log := logging.FromContext(ctx)
 
-	primary, err := getPrimaryPod(ctx, cli, cr)
+	primary, err := common.GetPrimaryPod(ctx, cli, cr)
 	if err != nil {
 		return nil, PrimaryPodNotFound
 	}
@@ -188,32 +187,8 @@ func GetLatestCommitTimestamp(ctx context.Context, cli client.Client, execCli *c
 	return &commitTsMeta, nil
 }
 
-func getPrimaryPod(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGCluster) (*corev1.Pod, error) {
-	podList := &corev1.PodList{}
-	err := cli.List(ctx, podList, &client.ListOptions{
-		Namespace: cr.Namespace,
-		LabelSelector: labels.SelectorFromSet(map[string]string{
-			"app.kubernetes.io/instance":             cr.Name,
-			"postgres-operator.crunchydata.com/role": "master",
-		}),
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if len(podList.Items) == 0 {
-		return nil, errors.New("no primary pod found")
-	}
-
-	if len(podList.Items) > 1 {
-		return nil, errors.New("multiple primary pods found")
-	}
-
-	return &podList.Items[0], nil
-}
-
 func getBackupStartTimestamp(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGCluster, backup *pgv2.PerconaPGBackup) (time.Time, error) {
-	primary, err := getPrimaryPod(ctx, cli, cr)
+	primary, err := common.GetPrimaryPod(ctx, cli, cr)
 	if err != nil {
 		return time.Time{}, PrimaryPodNotFound
 	}