From 14a00239e7955a9abf1dac0f389c14cbc42e9452 Mon Sep 17 00:00:00 2001 From: Nicolas Bigler Date: Wed, 30 Oct 2024 11:21:39 +0100 Subject: [PATCH] Enhance support for control-plane split Signed-off-by: Nicolas Bigler --- cmd/controller.go | 76 ++++++++++--------- .../functions/common/instance_namespace.go | 5 +- .../functions/vshnpostgres/objectbucket.go | 5 ++ pkg/comp-functions/runtime/function_mgr.go | 23 ++++++ pkg/controller/webhooks/deletionprotection.go | 53 +++++++++++-- .../webhooks/deletionprotection_test.go | 8 +- .../generic_deletionprotection_handler.go | 7 +- pkg/controller/webhooks/keycloak.go | 1 - pkg/controller/webhooks/namespace.go | 9 ++- pkg/controller/webhooks/object.go | 20 +++-- pkg/controller/webhooks/pvc.go | 7 +- pkg/controller/webhooks/xobjectbuckets.go | 7 +- 12 files changed, 159 insertions(+), 62 deletions(-) diff --git a/cmd/controller.go b/cmd/controller.go index 1d2333ec1a..30e046d4aa 100644 --- a/cmd/controller.go +++ b/cmd/controller.go @@ -18,14 +18,16 @@ import ( ) type controller struct { - scheme *runtime.Scheme - metricsAddr, healthAddr string - leaderElect bool - enableWebhooks bool - enableAppcatWebhooks bool - enableQuotas bool - enableEventForwarding bool - certDir string + scheme *runtime.Scheme + metricsAddr, healthAddr string + leaderElect bool + enableWebhooks bool + enableAppcatWebhooks bool + enableProviderWebhooks bool + enableLegacyObjectWebhooks bool + enableQuotas bool + enableEventForwarding bool + certDir string } var c = controller{ @@ -46,6 +48,8 @@ func init() { "Enabling this will ensure there is only one active controller manager.") ControllerCMD.Flags().BoolVar(&c.enableWebhooks, "webhooks", true, "Disable the validation webhooks.") ControllerCMD.Flags().BoolVar(&c.enableAppcatWebhooks, "appcat-webhooks", true, "Disable the appcat validation webhooks") + ControllerCMD.Flags().BoolVar(&c.enableProviderWebhooks, "provider-webhooks", true, "Disable the provider validation webhooks") + ControllerCMD.Flags().BoolVar(&c.enableLegacyObjectWebhooks, "objects-legacy-webhooks", false, "Enable legacy objects validation webhooks") ControllerCMD.Flags().StringVar(&c.certDir, "certdir", "/etc/webhook/certs", "Set the webhook certificate directory") ControllerCMD.Flags().BoolVar(&c.enableQuotas, "quotas", false, "Enable the quota webhooks, is only active if webhooks is also true") ControllerCMD.Flags().BoolVar(&c.enableEventForwarding, "event-forwarding", true, "Disable event-forwarding") @@ -96,7 +100,7 @@ func (c *controller) executeController(cmd *cobra.Command, _ []string) error { return fmt.Errorf("PLANS_NAMEPSACE env variable needs to be set for quota support") } - err := setupWebhooks(mgr, c.enableQuotas, c.enableAppcatWebhooks) + err := setupWebhooks(mgr, c.enableQuotas, c.enableAppcatWebhooks, c.enableProviderWebhooks, c.enableLegacyObjectWebhooks) if err != nil { return err } @@ -112,7 +116,7 @@ func (c *controller) executeController(cmd *cobra.Command, _ []string) error { return mgr.Start(ctrl.SetupSignalHandler()) } -func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool) error { +func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool, withProviderWebhooks bool, withLegacyObjectWebhooks bool) error { if withAppcatWebhooks { err := webhooks.SetupPostgreSQLWebhookHandlerWithManager(mgr, withQuota) if err != nil { @@ -149,31 +153,35 @@ func setupWebhooks(mgr manager.Manager, withQuota bool, withAppcatWebhooks bool) } } - err := webhooks.SetupNamespaceDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err - } - err = webhooks.SetupReleaseDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err - } - err = webhooks.SetupMysqlDatabaseDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err - } - err = webhooks.SetupMysqlGrantDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err - } - err = webhooks.SetupMysqlUserDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err - } - err = webhooks.SetupObjectDeletionProtectionHandlerWithManager(mgr) - if err != nil { - return err + if withProviderWebhooks { + err := webhooks.SetupReleaseDeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + err = webhooks.SetupMysqlDatabaseDeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + err = webhooks.SetupMysqlGrantDeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + err = webhooks.SetupMysqlUserDeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + err = webhooks.SetupObjectDeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + if withLegacyObjectWebhooks { + err = webhooks.SetupObjectv1alpha1DeletionProtectionHandlerWithManager(mgr) + if err != nil { + return err + } + } } - err = webhooks.SetupObjectv1alpha1DeletionProtectionHandlerWithManager(mgr) + err := webhooks.SetupNamespaceDeletionProtectionHandlerWithManager(mgr) if err != nil { return err } diff --git a/pkg/comp-functions/functions/common/instance_namespace.go b/pkg/comp-functions/functions/common/instance_namespace.go index 84b9c75ba3..c9e9ee784b 100644 --- a/pkg/comp-functions/functions/common/instance_namespace.go +++ b/pkg/comp-functions/functions/common/instance_namespace.go @@ -89,8 +89,11 @@ func createNamespaceObserver(claimNs string, instance string, svc *runtime.Servi Name: claimNs, }, } + labels := map[string]string{ + "appcat.vshn.io/ignore-provider-config": "true", + } - return svc.SetDesiredKubeObserveObject(ns, instance+claimNsObserverSuffix) + return svc.SetDesiredKubeObserveObjectWithLabels(ns, instance+claimNsObserverSuffix, labels) } // Create the namespace for the service instance diff --git a/pkg/comp-functions/functions/vshnpostgres/objectbucket.go b/pkg/comp-functions/functions/vshnpostgres/objectbucket.go index 65b07d0ee0..503789063d 100644 --- a/pkg/comp-functions/functions/vshnpostgres/objectbucket.go +++ b/pkg/comp-functions/functions/vshnpostgres/objectbucket.go @@ -19,12 +19,17 @@ func EnsureObjectBucketLabels(ctx context.Context, comp *vshnv1.VSHNPostgreSQL, return runtime.NewFatalResult(fmt.Errorf("Cannot get composite from function io: %w", err)) } + labels := map[string]string{ + "appcat.vshn.io/ignore-provider-config": "true", + } + bucket := &appcatv1.XObjectBucket{} err = svc.GetDesiredComposedResourceByName(bucket, "pg-bucket") if err != nil { return runtime.NewWarningResult("cannot get xobjectbucket") } + bucket.SetLabels(labels) err = svc.SetDesiredComposedResourceWithName(bucket, "pg-bucket") if err != nil { diff --git a/pkg/comp-functions/runtime/function_mgr.go b/pkg/comp-functions/runtime/function_mgr.go index bccde3b421..c42574d319 100644 --- a/pkg/comp-functions/runtime/function_mgr.go +++ b/pkg/comp-functions/runtime/function_mgr.go @@ -560,6 +560,18 @@ func (s *ServiceRuntime) SetDesiredKubeObserveObject(obj client.Object, objectNa return s.SetDesiredComposedResourceWithName(kobj, objectName) } +// SetDesiredKubeObserveObjectWithLabels takes any `runtime.Object`, puts it into a provider-kubernetes Object and then +// adds it to the desired composed resources. +func (s *ServiceRuntime) SetDesiredKubeObserveObjectWithLabels(obj client.Object, objectName string, labels map[string]string, refs ...xkube.Reference) error { + + kobj, err := s.putIntoObjectWithLabels(true, obj, objectName, objectName, labels, refs...) + if err != nil { + return err + } + + return s.SetDesiredComposedResourceWithName(kobj, objectName) +} + // putIntoObject adds or updates the desired resource into its kube object // It will inject the same labels as any managed resource gets. func (s *ServiceRuntime) putIntoObject(observeOnly bool, o client.Object, kon, resourceName string, refs ...xkube.Reference) (*xkube.Object, error) { @@ -631,6 +643,17 @@ func (s *ServiceRuntime) putIntoObject(observeOnly bool, o client.Object, kon, r return ko, nil } +// putIntoObjectWithLabels adds or updates the desired resource into its kube object +// It will inject the same labels as any managed resource gets as well as add any additional labels specified. +func (s *ServiceRuntime) putIntoObjectWithLabels(observeOnly bool, o client.Object, kon, resourceName string, labels map[string]string, refs ...xkube.Reference) (*xkube.Object, error) { + ko, err := s.putIntoObject(observeOnly, o, kon, resourceName, refs...) + if err != nil { + return nil, err + } + ko.SetLabels(labels) + return ko, nil +} + // GetObservedComposite returns the observed composite and unmarshals it into the given object. func (s *ServiceRuntime) GetObservedComposite(obj client.Object) error { comp, err := request.GetObservedCompositeResource(s.req) diff --git a/pkg/controller/webhooks/deletionprotection.go b/pkg/controller/webhooks/deletionprotection.go index f6850c7d36..79c571967c 100644 --- a/pkg/controller/webhooks/deletionprotection.go +++ b/pkg/controller/webhooks/deletionprotection.go @@ -1,15 +1,21 @@ package webhooks import ( + "bufio" "context" "fmt" + "io" + "os" "github.com/go-logr/logr" + "github.com/spf13/viper" "github.com/vshn/appcat/v4/pkg/comp-functions/runtime" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -38,7 +44,7 @@ type DeletionProtectionInfo interface { // checkManagedObject will find the highest composite for any object that is deployed via Crossplane or provider-kubernetes. // For example: XObjectBucket, Release, Namespace, etc. // Anything that either contains an owner reference to a composite or an owner annotation. -func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, l logr.Logger) (compositeInfo, error) { +func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, cpClient client.Client, l logr.Logger) (compositeInfo, error) { ownerKind, ok := obj.GetLabels()[runtime.OwnerKindAnnotation] if !ok || ownerKind == "" { @@ -86,7 +92,7 @@ func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, return compositeInfo{Exists: false, Name: ownerName}, fmt.Errorf("object is not a valid client object: %s", ownerName) } - err = c.Get(ctx, client.ObjectKey{Name: ownerName}, comp) + err = cpClient.Get(ctx, client.ObjectKey{Name: ownerName}, comp) if err != nil { if apierrors.IsNotFound(err) { return compositeInfo{Exists: false, Name: ownerName}, nil @@ -109,10 +115,10 @@ func checkManagedObject(ctx context.Context, obj client.Object, c client.Client, // It checks if the namespace it belongs to is managed by a composite, if that's the case it uses the same logic to // determine the state of the deletion protection. // Such objects would be: any helm generated object, pvcs for sts and any other 3rd party managed objects. -func checkUnmanagedObject(ctx context.Context, obj client.Object, c client.Client, l logr.Logger) (compositeInfo, error) { +func checkUnmanagedObject(ctx context.Context, obj client.Object, c client.Client, cpClient client.Client, l logr.Logger) (compositeInfo, error) { namespace := &corev1.Namespace{} - err := c.Get(ctx, client.ObjectKey{Name: obj.GetNamespace()}, namespace) + err := cpClient.Get(ctx, client.ObjectKey{Name: obj.GetNamespace()}, namespace) if err != nil { if apierrors.IsNotFound(err) { return compositeInfo{Exists: false}, nil @@ -120,7 +126,7 @@ func checkUnmanagedObject(ctx context.Context, obj client.Object, c client.Clien return compositeInfo{Exists: false}, err } - compInfo, err := checkManagedObject(ctx, namespace, c, l) + compInfo, err := checkManagedObject(ctx, namespace, c, cpClient, l) if err != nil { return compositeInfo{}, err } @@ -145,3 +151,40 @@ func isDeletionProtected(obj client.Object) bool { // If the value is "true" the protection is disabled. return !(val == "true") } + +func getControlPlaneClient(mgr ctrl.Manager) (client.Client, error) { + + cpClient := mgr.GetClient() + if viper.IsSet("CONTROL_PLANE_KUBECONFIG") { + kubeconfigPath := viper.GetString("CONTROL_PLANE_KUBECONFIG") + file, err := os.Open(kubeconfigPath) + if err != nil { + return cpClient, err + } + defer file.Close() + + // Get the file size + stat, err := file.Stat() + if err != nil { + return cpClient, err + } + + // Read the file into a byte slice + kubeconfig := make([]byte, stat.Size()) + _, err = bufio.NewReader(file).Read(kubeconfig) + if err != nil && err != io.EOF { + return cpClient, err + } + config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig)) + if err != nil { + return cpClient, err + } + cpClient, err = client.New(config, client.Options{ + Scheme: mgr.GetScheme(), + }) + if err != nil { + return cpClient, err + } + } + return cpClient, nil +} diff --git a/pkg/controller/webhooks/deletionprotection_test.go b/pkg/controller/webhooks/deletionprotection_test.go index b70b3841a2..7df3906060 100644 --- a/pkg/controller/webhooks/deletionprotection_test.go +++ b/pkg/controller/webhooks/deletionprotection_test.go @@ -86,7 +86,7 @@ func Test_checkManagedObject(t *testing.T) { Build() // Then expect parent - compInfo, err := checkManagedObject(context.TODO(), obj, c, logr.Discard()) + compInfo, err := checkManagedObject(context.TODO(), obj, c, c, logr.Discard()) assert.NoError(t, err) assert.Equal(t, compositeInfo{Exists: true, Name: "redis"}, compInfo) @@ -96,7 +96,7 @@ func Test_checkManagedObject(t *testing.T) { obj.SetLabels(labels) // Then don't expect parent - compInfo, err = checkManagedObject(context.TODO(), obj, c, logr.Discard()) + compInfo, err = checkManagedObject(context.TODO(), obj, c, c, logr.Discard()) assert.NoError(t, err) assert.Equal(t, compositeInfo{Exists: false, Name: "redis"}, compInfo) @@ -127,7 +127,7 @@ func Test_checkManagedObject(t *testing.T) { assert.NoError(t, err) // Then expect parent - compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, logr.Discard()) + compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, c, logr.Discard()) assert.NoError(t, err) assert.Equal(t, compositeInfo{Exists: true, Name: "redis"}, compInfo) @@ -137,7 +137,7 @@ func Test_checkManagedObject(t *testing.T) { pvc.SetLabels(labels) // Then expect no parent - compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, logr.Discard()) + compInfo, err = checkUnmanagedObject(context.TODO(), pvc, c, c, logr.Discard()) assert.NoError(t, err) assert.Equal(t, compositeInfo{Exists: false, Name: "redis"}, compInfo) diff --git a/pkg/controller/webhooks/generic_deletionprotection_handler.go b/pkg/controller/webhooks/generic_deletionprotection_handler.go index ef703cc50d..f7c02d8014 100644 --- a/pkg/controller/webhooks/generic_deletionprotection_handler.go +++ b/pkg/controller/webhooks/generic_deletionprotection_handler.go @@ -14,8 +14,9 @@ import ( var _ webhook.CustomValidator = &GenericDeletionProtectionHandler{} type GenericDeletionProtectionHandler struct { - client client.Client - log logr.Logger + client client.Client + controlPlaneClient client.Client + log logr.Logger } // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type @@ -40,7 +41,7 @@ func (p *GenericDeletionProtectionHandler) ValidateDelete(ctx context.Context, o l := p.log.WithValues("object", resource.GetName(), "object", resource.GetNamespace(), "GVK", resource.GetObjectKind().GroupVersionKind().String()) - compInfo, err := checkManagedObject(ctx, resource, p.client, l) + compInfo, err := checkManagedObject(ctx, resource, p.client, p.controlPlaneClient, l) if err != nil { return nil, err } diff --git a/pkg/controller/webhooks/keycloak.go b/pkg/controller/webhooks/keycloak.go index fc940eea7e..eaf6ef7441 100644 --- a/pkg/controller/webhooks/keycloak.go +++ b/pkg/controller/webhooks/keycloak.go @@ -25,7 +25,6 @@ type KeycloakWebhookHandler struct { // SetupKeycloakWebhookHandlerWithManager registers the validation webhook with the manager. func SetupKeycloakWebhookHandlerWithManager(mgr ctrl.Manager, withQuota bool) error { - return ctrl.NewWebhookManagedBy(mgr). For(&vshnv1.VSHNKeycloak{}). WithValidator(&KeycloakWebhookHandler{ diff --git a/pkg/controller/webhooks/namespace.go b/pkg/controller/webhooks/namespace.go index 4b829d8efd..823d66019d 100644 --- a/pkg/controller/webhooks/namespace.go +++ b/pkg/controller/webhooks/namespace.go @@ -9,12 +9,17 @@ import ( // SetupNamespaceDeletionProtectionHandlerWithManager registers the validation webhook with the manager. func SetupNamespaceDeletionProtectionHandlerWithManager(mgr ctrl.Manager) error { + cpClient, err := getControlPlaneClient(mgr) + if err != nil { + return err + } return ctrl.NewWebhookManagedBy(mgr). For(&corev1.Namespace{}). WithValidator(&GenericDeletionProtectionHandler{ - client: mgr.GetClient(), - log: mgr.GetLogger().WithName("webhook").WithName("namespace"), + client: mgr.GetClient(), + controlPlaneClient: cpClient, + log: mgr.GetLogger().WithName("webhook").WithName("namespace"), }). Complete() } diff --git a/pkg/controller/webhooks/object.go b/pkg/controller/webhooks/object.go index c58fdd47df..80137d03d8 100644 --- a/pkg/controller/webhooks/object.go +++ b/pkg/controller/webhooks/object.go @@ -15,24 +15,32 @@ import ( // SetupObjectDeletionProtectionHandlerWithManager registers the validation webhook with the manager. func SetupObjectDeletionProtectionHandlerWithManager(mgr ctrl.Manager) error { - + cpClient, err := getControlPlaneClient(mgr) + if err != nil { + return err + } return ctrl.NewWebhookManagedBy(mgr). For(&xkubev1alpha2.Object{}). WithValidator(&GenericDeletionProtectionHandler{ - client: mgr.GetClient(), - log: mgr.GetLogger().WithName("webhook").WithName("object"), + client: mgr.GetClient(), + controlPlaneClient: cpClient, + log: mgr.GetLogger().WithName("webhook").WithName("object"), }). Complete() } // SetupObjectv1alpha1DeletionProtectionHandlerWithManager registers the validation webhook with the manager. func SetupObjectv1alpha1DeletionProtectionHandlerWithManager(mgr ctrl.Manager) error { - + cpClient, err := getControlPlaneClient(mgr) + if err != nil { + return err + } return ctrl.NewWebhookManagedBy(mgr). For(&xkubev1alpha1.Object{}). WithValidator(&GenericDeletionProtectionHandler{ - client: mgr.GetClient(), - log: mgr.GetLogger().WithName("webhook").WithName("object"), + client: mgr.GetClient(), + controlPlaneClient: cpClient, + log: mgr.GetLogger().WithName("webhook").WithName("object"), }). Complete() } diff --git a/pkg/controller/webhooks/pvc.go b/pkg/controller/webhooks/pvc.go index 6bd67dd6e7..049d071390 100644 --- a/pkg/controller/webhooks/pvc.go +++ b/pkg/controller/webhooks/pvc.go @@ -19,8 +19,9 @@ var _ webhook.CustomValidator = &PVCDeletionProtectionHandler{} // PVCDeletionProtectionHandler type PVCDeletionProtectionHandler struct { - client client.Client - log logr.Logger + client client.Client + cpClient client.Client + log logr.Logger } // SetupPVCDeletionProtectionHandlerWithManager registers the validation webhook with the manager. @@ -57,7 +58,7 @@ func (p *PVCDeletionProtectionHandler) ValidateDelete(ctx context.Context, obj r l := p.log.WithValues("object", pvc.GetName(), "namespace", pvc.GetNamespace(), "GVK", pvc.GetObjectKind().GroupVersionKind().String()) - compInfo, err := checkUnmanagedObject(ctx, pvc, p.client, l) + compInfo, err := checkUnmanagedObject(ctx, pvc, p.client, p.cpClient, l) if err != nil { return nil, err } diff --git a/pkg/controller/webhooks/xobjectbuckets.go b/pkg/controller/webhooks/xobjectbuckets.go index e74c9b9e57..0dfd7dac17 100644 --- a/pkg/controller/webhooks/xobjectbuckets.go +++ b/pkg/controller/webhooks/xobjectbuckets.go @@ -20,8 +20,9 @@ var _ webhook.CustomValidator = &XObjectbucketDeletionProtectionHandler{} // XObjectbucketDeletionProtectionHandler type XObjectbucketDeletionProtectionHandler struct { - client client.Client - log logr.Logger + client client.Client + cpClient client.Client + log logr.Logger } // SetupXObjectbucketCDeletionProtectionHandlerWithManager registers the validation webhook with the manager. @@ -68,7 +69,7 @@ func (p *XObjectbucketDeletionProtectionHandler) ValidateDelete(ctx context.Cont l := p.log.WithValues("object", bucket.GetName(), "namespace", bucket.GetNamespace(), "GVK", bucket.GetObjectKind().GroupVersionKind().String()) - compInfo, err := checkManagedObject(ctx, bucket, p.client, l) + compInfo, err := checkManagedObject(ctx, bucket, p.client, p.cpClient, l) if err != nil { return nil, err }