Enhance support for control-plane split

Signed-off-by: Nicolas Bigler <[email protected]>

cmd/controller.go

@@ -23,6 +23,7 @@ type controller struct {
 	leaderElect             bool
 	enableWebhooks          bool
 	enableAppcatWebhooks    bool
+	enableProviderWebhooks  bool
 	enableQuotas            bool
 	enableEventForwarding   bool
 	certDir                 string
@@ -46,6 +47,7 @@ func init() {
 		"Enabling this will ensure there is only one active controller manager.")
 	ControllerCMD.Flags().BoolVar(&c.enableWebhooks, "webhooks", true, "Disable the validation webhooks.")
 	ControllerCMD.Flags().BoolVar(&c.enableAppcatWebhooks, "appcat-webhooks", true, "Disable the appcat validation webhooks")
+	ControllerCMD.Flags().BoolVar(&c.enableProviderWebhooks, "provider-webhooks", true, "Disable the provider validation webhooks")
 	ControllerCMD.Flags().StringVar(&c.certDir, "certdir", "/etc/webhook/certs", "Set the webhook certificate directory")
 	ControllerCMD.Flags().BoolVar(&c.enableQuotas, "quotas", false, "Enable the quota webhooks, is only active if webhooks is also true")
 	ControllerCMD.Flags().BoolVar(&c.enableEventForwarding, "event-forwarding", true, "Disable event-forwarding")
@@ -96,7 +98,7 @@ func (c *controller) executeController(cmd *cobra.Command, _ []string) error {
 			return fmt.Errorf("PLANS_NAMEPSACE env variable needs to be set for quota support")
 		}
 
-		err := setupWebhooks(mgr, c.enableQuotas, c.enableAppcatWebhooks)
+		err := setupWebhooks(mgr, c.enableQuotas, c.enableAppcatWebhooks, c.enableProviderWebhooks)
 		if err != nil {
 			return err
 		}
@@ -112,7 +114,7 @@ func (c *controller) executeController(cmd *cobra.Command, _ []string) error {
 	return mgr.Start(ctrl.SetupSignalHandler())
 }
 
-func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool) error {
+func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool, withProviderWebhooks bool) error {
 	if withAppcatWebhooks {
 		err := webhooks.SetupPostgreSQLWebhookHandlerWithManager(mgr, withQuota)
 		if err != nil {
@@ -149,31 +151,33 @@ func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool)
 		}
 	}
 
-	err := webhooks.SetupNamespaceDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
-	}
-	err = webhooks.SetupReleaseDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
-	}
-	err = webhooks.SetupMysqlDatabaseDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
-	}
-	err = webhooks.SetupMysqlGrantDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
-	}
-	err = webhooks.SetupMysqlUserDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
-	}
-	err = webhooks.SetupObjectDeletionProtectionHandlerWithManager(mgr)
-	if err != nil {
-		return err
+	if withProviderWebhooks {
+		err := webhooks.SetupReleaseDeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
+		err = webhooks.SetupMysqlDatabaseDeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
+		err = webhooks.SetupMysqlGrantDeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
+		err = webhooks.SetupMysqlUserDeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
+		err = webhooks.SetupObjectDeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
+		err = webhooks.SetupObjectv1alpha1DeletionProtectionHandlerWithManager(mgr)
+		if err != nil {
+			return err
+		}
 	}
-	err = webhooks.SetupObjectv1alpha1DeletionProtectionHandlerWithManager(mgr)
+	err := webhooks.SetupNamespaceDeletionProtectionHandlerWithManager(mgr)
 	if err != nil {
 		return err
 	}

pkg/comp-functions/functions/common/instance_namespace.go

@@ -89,8 +89,11 @@ func createNamespaceObserver(claimNs string, instance string, svc *runtime.Servi
 			Name: claimNs,
 		},
 	}
+	labels := map[string]string{
+		"appcat.vshn.io/ignore-provider-config": "true",
+	}
 
-	return svc.SetDesiredKubeObserveObject(ns, instance+claimNsObserverSuffix)
+	return svc.SetDesiredKubeObserveObjectWithLabels(ns, instance+claimNsObserverSuffix, labels)
 }
 
 // Create the namespace for the service instance

pkg/comp-functions/functions/vshnpostgres/objectbucket.go

@@ -19,12 +19,17 @@ func EnsureObjectBucketLabels(ctx context.Context, comp *vshnv1.VSHNPostgreSQL,
 		return runtime.NewFatalResult(fmt.Errorf("Cannot get composite from function io: %w", err))
 	}
 
+	labels := map[string]string{
+		"appcat.vshn.io/ignore-provider-config": "true",
+	}
+
 	bucket := &appcatv1.XObjectBucket{}
 
 	err = svc.GetDesiredComposedResourceByName(bucket, "pg-bucket")
 	if err != nil {
 		return runtime.NewWarningResult("cannot get xobjectbucket")
 	}
+	bucket.SetLabels(labels)
 
 	err = svc.SetDesiredComposedResourceWithName(bucket, "pg-bucket")
 	if err != nil {

pkg/comp-functions/runtime/function_mgr.go

@@ -560,6 +560,18 @@ func (s *ServiceRuntime) SetDesiredKubeObserveObject(obj client.Object, objectNa
 	return s.SetDesiredComposedResourceWithName(kobj, objectName)
 }
 
+// SetDesiredKubeObserveObjectWithLabels takes any `runtime.Object`, puts it into a provider-kubernetes Object and then
+// adds it to the desired composed resources.
+func (s *ServiceRuntime) SetDesiredKubeObserveObjectWithLabels(obj client.Object, objectName string, labels map[string]string, refs ...xkube.Reference) error {
+
+	kobj, err := s.putIntoObjectWithLabels(true, obj, objectName, objectName, labels, refs...)
+	if err != nil {
+		return err
+	}
+
+	return s.SetDesiredComposedResourceWithName(kobj, objectName)
+}
+
 // putIntoObject adds or updates the desired resource into its kube object
 // It will inject the same labels as any managed resource gets.
 func (s *ServiceRuntime) putIntoObject(observeOnly bool, o client.Object, kon, resourceName string, refs ...xkube.Reference) (*xkube.Object, error) {
@@ -631,6 +643,17 @@ func (s *ServiceRuntime) putIntoObject(observeOnly bool, o client.Object, kon, r
 	return ko, nil
 }
 
+// putIntoObjectWithLabels adds or updates the desired resource into its kube object
+// It will inject the same labels as any managed resource gets as well as add any additional labels specified.
+func (s *ServiceRuntime) putIntoObjectWithLabels(observeOnly bool, o client.Object, kon, resourceName string, labels map[string]string, refs ...xkube.Reference) (*xkube.Object, error) {
+	ko, err := s.putIntoObject(observeOnly, o, kon, resourceName, refs...)
+	if err != nil {
+		return nil, err
+	}
+	ko.SetLabels(labels)
+	return ko, nil
+}
+
 // GetObservedComposite returns the observed composite and unmarshals it into the given object.
 func (s *ServiceRuntime) GetObservedComposite(obj client.Object) error {
 	comp, err := request.GetObservedCompositeResource(s.req)

pkg/controller/webhooks/deletionprotection.go

@@ -38,7 +38,7 @@ type DeletionProtectionInfo interface {
 // checkManagedObject will find the highest composite for any object that is deployed via Crossplane or provider-kubernetes.
 // For example: XObjectBucket, Release, Namespace, etc.
 // Anything that either contains an owner reference to a composite or an owner annotation.
-func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, l logr.Logger) (compositeInfo, error) {
+func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, cpClient client.Client, l logr.Logger) (compositeInfo, error) {
 
 	ownerKind, ok := obj.GetLabels()[runtime.OwnerKindAnnotation]
 	if !ok || ownerKind == "" {
@@ -86,7 +86,7 @@ func checkManagedObject(ctx context.Context, obj client.Object, c client.Client,
 		return compositeInfo{Exists: false, Name: ownerName}, fmt.Errorf("object is not a valid client object: %s", ownerName)
 	}
 
-	err = c.Get(ctx, client.ObjectKey{Name: ownerName}, comp)
+	err = cpClient.Get(ctx, client.ObjectKey{Name: ownerName}, comp)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			return compositeInfo{Exists: false, Name: ownerName}, nil
@@ -109,18 +109,18 @@ func checkManagedObject(ctx context.Context, obj client.Object, c client.Client,
 // It checks if the namespace it belongs to is managed by a composite, if that's the case it uses the same logic to
 // determine the state of the deletion protection.
 // Such objects would be: any helm generated object, pvcs for sts and any other 3rd party managed objects.
-func checkUnmanagedObject(ctx context.Context, obj client.Object, c client.Client, l logr.Logger) (compositeInfo, error) {
+func checkUnmanagedObject(ctx context.Context, obj client.Object, c client.Client, cpClient client.Client, l logr.Logger) (compositeInfo, error) {
 	namespace := &corev1.Namespace{}
 
-	err := c.Get(ctx, client.ObjectKey{Name: obj.GetNamespace()}, namespace)
+	err := cpClient.Get(ctx, client.ObjectKey{Name: obj.GetNamespace()}, namespace)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			return compositeInfo{Exists: false}, nil
 		}
 		return compositeInfo{Exists: false}, err
 	}
 
-	compInfo, err := checkManagedObject(ctx, namespace, c, l)
+	compInfo, err := checkManagedObject(ctx, namespace, c, cpClient, l)
 	if err != nil {
 		return compositeInfo{}, err
 	}

pkg/controller/webhooks/deletionprotection_test.go

@@ -86,7 +86,7 @@ func Test_checkManagedObject(t *testing.T) {
 		Build()
 
 	// Then expect parent
-	compInfo, err := checkManagedObject(context.TODO(), obj, c, logr.Discard())
+	compInfo, err := checkManagedObject(context.TODO(), obj, c, c, logr.Discard())
 	assert.NoError(t, err)
 	assert.Equal(t, compositeInfo{Exists: true, Name: "redis"}, compInfo)
 
@@ -96,7 +96,7 @@ func Test_checkManagedObject(t *testing.T) {
 	obj.SetLabels(labels)
 
 	// Then don't expect parent
-	compInfo, err = checkManagedObject(context.TODO(), obj, c, logr.Discard())
+	compInfo, err = checkManagedObject(context.TODO(), obj, c, c, logr.Discard())
 	assert.NoError(t, err)
 	assert.Equal(t, compositeInfo{Exists: false, Name: "redis"}, compInfo)
 
@@ -127,7 +127,7 @@ func Test_checkManagedObject(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Then expect parent
-	compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, logr.Discard())
+	compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, c, logr.Discard())
 	assert.NoError(t, err)
 	assert.Equal(t, compositeInfo{Exists: true, Name: "redis"}, compInfo)
 
@@ -137,7 +137,7 @@ func Test_checkManagedObject(t *testing.T) {
 	pvc.SetLabels(labels)
 
 	// Then expect no parent
-	compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, logr.Discard())
+	compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, c, logr.Discard())
 	assert.NoError(t, err)
 	assert.Equal(t, compositeInfo{Exists: false, Name: "redis"}, compInfo)
 

pkg/controller/webhooks/generic_deletionprotection_handler.go

@@ -14,8 +14,9 @@ import (
 var _ webhook.CustomValidator = &GenericDeletionProtectionHandler{}
 
 type GenericDeletionProtectionHandler struct {
-	client client.Client
-	log    logr.Logger
+	client             client.Client
+	controlPlaneClient client.Client
+	log                logr.Logger
 }
 
 // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type
@@ -40,7 +41,7 @@ func (p *GenericDeletionProtectionHandler) ValidateDelete(ctx context.Context, o
 
 	l := p.log.WithValues("object", resource.GetName(), "object", resource.GetNamespace(), "GVK", resource.GetObjectKind().GroupVersionKind().String())
 
-	compInfo, err := checkManagedObject(ctx, resource, p.client, l)
+	compInfo, err := checkManagedObject(ctx, resource, p.client, p.controlPlaneClient, l)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/webhooks/namespace.go

@@ -1,20 +1,59 @@
 package webhooks
 
 import (
+	"bufio"
+	"io"
+	"os"
+
+	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 //+kubebuilder:webhook:verbs=delete,path=/validate--v1-namespace,mutating=false,failurePolicy=fail,groups="",resources=namespaces,versions=v1,name=namespace.vshn.appcat.vshn.io,sideEffects=None,admissionReviewVersions=v1
 
 // SetupNamespaceDeletionProtectionHandlerWithManager registers the validation webhook with the manager.
 func SetupNamespaceDeletionProtectionHandlerWithManager(mgr ctrl.Manager) error {
+	cpClient := mgr.GetClient()
+	if viper.IsSet("CONTROL_PLANE_KUBECONFIG") {
+		kubeconfigPath := viper.GetString("CONTROL_PLANE_KUBECONFIG")
+		file, err := os.Open(kubeconfigPath)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		// Get the file size
+		stat, err := file.Stat()
+		if err != nil {
+			return err
+		}
 
+		// Read the file into a byte slice
+		kubeconfig := make([]byte, stat.Size())
+		_, err = bufio.NewReader(file).Read(kubeconfig)
+		if err != nil && err != io.EOF {
+			return err
+		}
+		config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig))
+		if err != nil {
+			return nil
+		}
+		cpClient, err = client.New(config, client.Options{
+			Scheme: mgr.GetScheme(),
+		})
+		if err != nil {
+			return err
+		}
+	}
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(&corev1.Namespace{}).
 		WithValidator(&GenericDeletionProtectionHandler{
-			client: mgr.GetClient(),
-			log:    mgr.GetLogger().WithName("webhook").WithName("namespace"),
+			client:             mgr.GetClient(),
+			controlPlaneClient: cpClient,
+			log:                mgr.GetLogger().WithName("webhook").WithName("namespace"),
 		}).
 		Complete()
 }

pkg/controller/webhooks/pvc.go

@@ -19,8 +19,9 @@ var _ webhook.CustomValidator = &PVCDeletionProtectionHandler{}
 
 // PVCDeletionProtectionHandler
 type PVCDeletionProtectionHandler struct {
-	client client.Client
-	log    logr.Logger
+	client   client.Client
+	cpClient client.Client
+	log      logr.Logger
 }
 
 // SetupPVCDeletionProtectionHandlerWithManager registers the validation webhook with the manager.
@@ -57,7 +58,7 @@ func (p *PVCDeletionProtectionHandler) ValidateDelete(ctx context.Context, obj r
 
 	l := p.log.WithValues("object", pvc.GetName(), "namespace", pvc.GetNamespace(), "GVK", pvc.GetObjectKind().GroupVersionKind().String())
 
-	compInfo, err := checkUnmanagedObject(ctx, pvc, p.client, l)
+	compInfo, err := checkUnmanagedObject(ctx, pvc, p.client, p.cpClient, l)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/webhooks/xobjectbuckets.go

@@ -20,8 +20,9 @@ var _ webhook.CustomValidator = &XObjectbucketDeletionProtectionHandler{}
 
 // XObjectbucketDeletionProtectionHandler
 type XObjectbucketDeletionProtectionHandler struct {
-	client client.Client
-	log    logr.Logger
+	client   client.Client
+	cpClient client.Client
+	log      logr.Logger
 }
 
 // SetupXObjectbucketCDeletionProtectionHandlerWithManager registers the validation webhook with the manager.
@@ -68,7 +69,7 @@ func (p *XObjectbucketDeletionProtectionHandler) ValidateDelete(ctx context.Cont
 
 	l := p.log.WithValues("object", bucket.GetName(), "namespace", bucket.GetNamespace(), "GVK", bucket.GetObjectKind().GroupVersionKind().String())
 
-	compInfo, err := checkManagedObject(ctx, bucket, p.client, l)
+	compInfo, err := checkManagedObject(ctx, bucket, p.client, p.cpClient, l)
 	if err != nil {
 		return nil, err
 	}