From bdd22b95e8196b820075a8c9f70cf9a40c051034 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Oct 2024 13:53:55 +0200 Subject: [PATCH] test: Added e2e tests for bidirectional annotation & label sync --- .../syncer/networkpolicies/networkpolicies.go | 1 - test/e2e/syncer/pods/pods.go | 84 ++++++++++- test/e2e/syncer/services/services.go | 136 ++++++++++++++---- test/framework/framework.go | 4 +- test/framework/util.go | 17 +++ 5 files changed, 211 insertions(+), 31 deletions(-) diff --git a/test/e2e/syncer/networkpolicies/networkpolicies.go b/test/e2e/syncer/networkpolicies/networkpolicies.go index 4551c6bd52..0e430bc7b1 100644 --- a/test/e2e/syncer/networkpolicies/networkpolicies.go +++ b/test/e2e/syncer/networkpolicies/networkpolicies.go @@ -169,7 +169,6 @@ var _ = ginkgo.Describe("NetworkPolicies are created as expected", func() { time.Sleep(time.Second * 10) framework.DefaultFramework.TestServiceIsEventuallyUnreachable(curlPod, nginxService) }) - }) func updateNetworkPolicyWithRetryOnConflict(f *framework.Framework, networkPolicy *networkingv1.NetworkPolicy, mutator func(np *networkingv1.NetworkPolicy)) error { diff --git a/test/e2e/syncer/pods/pods.go b/test/e2e/syncer/pods/pods.go index d7e09b5a49..817b67a53d 100644 --- a/test/e2e/syncer/pods/pods.go +++ b/test/e2e/syncer/pods/pods.go @@ -89,7 +89,7 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { // version 1.22 and lesser than that needs legacy flag enabled if version != nil { - i, err := strconv.Atoi(strings.Replace(version.Minor, "+", "", -1)) + i, err := strconv.Atoi(strings.ReplaceAll(version.Minor, "+", "")) framework.ExpectNoError(err) if i > 22 { vpod.Spec.EphemeralContainers = []corev1.EphemeralContainer{{ @@ -211,7 +211,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { }, Data: map[string]string{ cmKey: cmKeyValue, - }}, metav1.CreateOptions{}) + }, + }, metav1.CreateOptions{}) framework.ExpectNoError(err) pod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{ @@ -295,7 +296,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { }, Data: map[string][]byte{ secretKey: []byte(secretKeyValue), - }}, metav1.CreateOptions{}) + }, + }, metav1.CreateOptions{}) framework.ExpectNoError(err) pod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{ @@ -550,4 +552,80 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() { } } }) + + ginkgo.It("should perform a bidirectional sync on labels and annotations", func() { + podName := "test-annotations" + vPod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Annotations: map[string]string{ + "vcluster-annotation": "from vCluster with love", + }, + Labels: map[string]string{ + "vcluster-specific-label": "with_its_value", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: testingContainerName, + Image: testingContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: f.GetDefaultSecurityContext(), + }, + }, + }, + }, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = f.WaitForPodRunning(podName, ns) + framework.ExpectNoError(err, "A pod created in the vcluster is expected to be in the Running phase eventually.") + + // get current physical Pod resource + pPodName := translate.Default.HostName(nil, vPod.Name, vPod.Namespace) + pPod, err := f.HostClient.CoreV1().Pods(pPodName.Namespace).Get(f.Context, pPodName.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // get current vCluster Pod resource + vPod, err = f.VClusterClient.CoreV1().Pods(ns).Get(f.Context, podName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // update host cluster pod with additional information + additionalLabelKey := "another-one" + additionalLabelValue := "good-syncer" + additionalAnnotationKey := "annotation-key" + additionalAnnotationValue := "annotation-value" + + err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) { + pPod.Labels[additionalLabelKey] = additionalLabelValue + pPod.Annotations[additionalAnnotationKey] = additionalAnnotationValue + pPod, err = f.HostClient.CoreV1().Pods(pPod.Namespace).Update(f.Context, pPod, metav1.UpdateOptions{}) + if err != nil { + if kerrors.IsConflict(err) { + return false, nil + } + return false, err + } + + return true, nil + }) + framework.ExpectNoError(err) + + // wait for the syncer to update the pod + err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(ctx context.Context) (bool, error) { + pod, err := f.VClusterClient.CoreV1().Pods(vPod.Namespace).Get(ctx, vPod.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pod.ResourceVersion != vPod.ResourceVersion && pod.Annotations[additionalAnnotationKey] != "", nil + }) + framework.ExpectNoError(err) + + vPod, err = f.VClusterClient.CoreV1().Pods(vPod.Namespace).Get(f.Context, vPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + framework.ExpectEqual(vPod.Annotations[additionalAnnotationKey], pPod.Annotations[additionalAnnotationKey]) + framework.ExpectEqual(vPod.Labels[additionalLabelKey], pPod.Labels[additionalLabelKey]) + }) }) diff --git a/test/e2e/syncer/services/services.go b/test/e2e/syncer/services/services.go index 3e6063ce9b..60bab03244 100644 --- a/test/e2e/syncer/services/services.go +++ b/test/e2e/syncer/services/services.go @@ -12,11 +12,13 @@ import ( "github.com/loft-sh/vcluster/test/framework" "github.com/onsi/ginkgo/v2" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/cache" @@ -157,20 +159,20 @@ var _ = ginkgo.Describe("Services are created as expected", func() { defer cancel() _, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) { if svc, ok := event.Object.(*corev1.Service); ok { - found := svc.ObjectMeta.Name == testService.ObjectMeta.Name && - svc.ObjectMeta.Namespace == ns && + found := svc.Name == testService.Name && + svc.Namespace == ns && svc.Labels["test-service-static"] == "true" if !found { - f.Log.Infof("observed Service %v in namespace %v with labels: %v & ports %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Spec.Ports) + f.Log.Infof("observed Service %v in namespace %v with labels: %v & ports %v", svc.Name, svc.Namespace, svc.Labels, svc.Spec.Ports) return false, nil } - f.Log.Infof("Found Service %v in namespace %v with labels: %v & ports %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Spec.Ports) + f.Log.Infof("Found Service %v in namespace %v with labels: %v & ports %v", svc.Name, svc.Namespace, svc.Labels, svc.Spec.Ports) return found, nil } f.Log.Infof("Observed event: %+v", event.Object) return false, nil }) - framework.ExpectNoError(err, "Failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns) + framework.ExpectNoError(err, "Failed to locate Service %v in namespace %v", testService.Name, ns) f.Log.Infof("Service %s created", testSvcName) ginkgo.By("Getting /status") @@ -202,14 +204,14 @@ var _ = ginkgo.Describe("Services are created as expected", func() { defer cancel() _, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) { if svc, ok := event.Object.(*corev1.Service); ok { - found := svc.ObjectMeta.Name == testService.ObjectMeta.Name && - svc.ObjectMeta.Namespace == ns && + found := svc.Name == testService.Name && + svc.Namespace == ns && svc.Annotations["patchedstatus"] == "true" if !found { - f.Log.Infof("observed Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer) + f.Log.Infof("observed Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer) return false, nil } - f.Log.Infof("Found Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer) + f.Log.Infof("Found Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer) return found, nil } f.Log.Infof("Observed event: %+v", event.Object) @@ -242,28 +244,28 @@ var _ = ginkgo.Describe("Services are created as expected", func() { defer cancel() _, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) { if svc, ok := event.Object.(*corev1.Service); ok { - found := svc.ObjectMeta.Name == testService.ObjectMeta.Name && - svc.ObjectMeta.Namespace == ns && + found := svc.Name == testService.Name && + svc.Namespace == ns && svc.Annotations["patchedstatus"] == "true" if !found { - f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer) + f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer) return false, nil } for _, cond := range svc.Status.Conditions { if cond.Type == "StatusUpdate" && cond.Reason == "E2E" && cond.Message == "Set from e2e test" { - f.Log.Infof("Found Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.Conditions) + f.Log.Infof("Found Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.Conditions) return found, nil } } - f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer) + f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer) return false, nil } f.Log.Infof("Observed event: %+v", event.Object) return false, nil }) - framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns) + framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.Name, ns) f.Log.Infof("Service %s has service status updated", testSvcName) ginkgo.By("patching the service") @@ -283,20 +285,20 @@ var _ = ginkgo.Describe("Services are created as expected", func() { defer cancel() _, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) { if svc, ok := event.Object.(*corev1.Service); ok { - found := svc.ObjectMeta.Name == testService.ObjectMeta.Name && - svc.ObjectMeta.Namespace == ns && + found := svc.Name == testService.Name && + svc.Namespace == ns && svc.Labels["test-service"] == "patched" if !found { - f.Log.Infof("observed Service %v in namespace %v with labels: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels) + f.Log.Infof("observed Service %v in namespace %v with labels: %v", svc.Name, svc.Namespace, svc.Labels) return false, nil } - f.Log.Infof("Found Service %v in namespace %v with labels: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels) + f.Log.Infof("Found Service %v in namespace %v with labels: %v", svc.Name, svc.Namespace, svc.Labels) return found, nil } f.Log.Infof("Observed event: %+v", event.Object) return false, nil }) - framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns) + framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.Name, ns) f.Log.Infof("Service %s patched", testSvcName) // Delete service @@ -309,14 +311,14 @@ var _ = ginkgo.Describe("Services are created as expected", func() { switch event.Type { case watch.Deleted: if svc, ok := event.Object.(*corev1.Service); ok { - found := svc.ObjectMeta.Name == testService.ObjectMeta.Name && - svc.ObjectMeta.Namespace == ns && + found := svc.Name == testService.Name && + svc.Namespace == ns && svc.Labels["test-service-static"] == "true" if !found { - f.Log.Infof("observed Service %v in namespace %v with labels: %v & annotations: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Annotations) + f.Log.Infof("observed Service %v in namespace %v with labels: %v & annotations: %v", svc.Name, svc.Namespace, svc.Labels, svc.Annotations) return false, nil } - f.Log.Infof("Found Service %v in namespace %v with labels: %v & annotations: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Annotations) + f.Log.Infof("Found Service %v in namespace %v with labels: %v & annotations: %v", svc.Name, svc.Namespace, svc.Labels, svc.Annotations) return found, nil } default: @@ -324,7 +326,91 @@ var _ = ginkgo.Describe("Services are created as expected", func() { } return false, nil }) - framework.ExpectNoError(err, "failed to delete Service %v in namespace %v", testService.ObjectMeta.Name, ns) + framework.ExpectNoError(err, "failed to delete Service %v in namespace %v", testService.Name, ns) f.Log.Infof("Service %s deleted", testSvcName) }) + + ginkgo.It("should sync labels and annotation bidirectionally", func() { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myservice-with-annotations", + Namespace: ns, + Annotations: map[string]string{ + "some-annotation": "that is set from the vCluster", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "None", + }, + } + + vService, err := f.VClusterClient.CoreV1().Services(ns).Create(f.Context, service, metav1.CreateOptions{}) + framework.ExpectNoError(err) + err = f.WaitForService(vService.Name, vService.Namespace) + framework.ExpectNoError(err) + + // get physical service + pServiceName := translate.Default.HostName(nil, vService.Name, vService.Namespace) + pService, err := f.HostClient.CoreV1().Services(pServiceName.Namespace).Get(f.Context, pServiceName.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // update physical service + err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) { + pService.Annotations["some-annotation"] += " and update from the host cluster" + pService.Labels["host-cluster-label"] = "some_host_label_value" + pService, err = f.HostClient.CoreV1().Services(pServiceName.Namespace).Update(f.Context, pService, metav1.UpdateOptions{}) + if err != nil { + if kerrors.IsConflict(err) { + return false, nil + } + + return false, err + } + + return true, nil + }) + framework.ExpectNoError(err) + + // wait for the change to be synced into the vCluster + err = f.WaitForServiceToUpdate(f.VClusterClient, vService.Name, vService.Namespace, vService.ResourceVersion) + framework.ExpectNoError(err) + + // refetch the vCluster service object + vService, err = f.VClusterClient.CoreV1().Services(ns).Get(f.Context, vService.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // check that labels and annotations are the same + framework.ExpectEqual(vService.Annotations["some-annotation"], pService.Annotations["some-annotation"]) + framework.ExpectEqual(vService.Labels["host-cluster-label"], pService.Labels["host-cluster-label"]) + + // update vCluster service + err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) { + vService.Annotations["some-annotation"] += " and another update from the vCluster" + vService.Labels["vcluster-label"] = "some_vcluster_value" + vService, err = f.VClusterClient.CoreV1().Services(vService.Namespace).Update(f.Context, vService, metav1.UpdateOptions{}) + if err != nil { + if kerrors.IsConflict(err) { + return false, nil + } + + return false, err + } + + return true, nil + }) + framework.ExpectNoError(err) + + // wait for the change to be synced into the host cluster + err = f.WaitForServiceToUpdate(f.HostClient, pService.Name, pService.Namespace, pService.ResourceVersion) + framework.ExpectNoError(err) + + // refetch the host cluster service object + pService, err = f.HostClient.CoreV1().Services(pService.Namespace).Get(f.Context, pService.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // check that labels and annotations are the same + framework.ExpectEqual(vService.Annotations["some-annotation"], pService.Annotations["some-annotation"]) + framework.ExpectEqual(vService.Labels["vcluster-label"], pService.Labels["vcluster-label"]) + }) }) diff --git a/test/framework/framework.go b/test/framework/framework.go index b54ec792d5..5e12569253 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -117,7 +117,7 @@ func CreateFramework(ctx context.Context, scheme *runtime.Scheme) error { suffix := os.Getenv("VCLUSTER_SUFFIX") if suffix == "" { - //TODO: maybe implement some autodiscovery of the suffix value that would work with dev and prod setups + // TODO: maybe implement some autodiscovery of the suffix value that would work with dev and prod setups suffix = "vcluster" } translate.VClusterName = suffix @@ -131,7 +131,7 @@ func CreateFramework(ctx context.Context, scheme *runtime.Scheme) error { translate.Default = translate.NewSingleNamespaceTranslator(ns) } - l.Infof("Testing Vcluster named: %s in namespace: %s", name, ns) + l.Infof("Testing vCluster named: %s in namespace: %s", name, ns) hostConfig, err := ctrl.GetConfig() if err != nil { diff --git a/test/framework/util.go b/test/framework/util.go index 9e0a9b76c8..5e6e0d8855 100644 --- a/test/framework/util.go +++ b/test/framework/util.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" ) @@ -159,6 +160,22 @@ func (f *Framework) WaitForService(serviceName string, ns string) error { }) } +// WaitForServiceToUpdate waits for a Kubernetes service to update by periodically fetching it using the provided client. +// It compares the current resource version of the service to the specified version and returns when they are different. +func (f *Framework) WaitForServiceToUpdate(client *kubernetes.Clientset, serviceName string, ns string, resourceVersion string) error { + return wait.PollUntilContextTimeout(f.Context, time.Second, PollTimeout, true, func(ctx context.Context) (bool, error) { + svc, err := client.CoreV1().Services(ns).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return svc.ResourceVersion != resourceVersion, nil + }) +} + // Some vcluster operations list Service, e.g. pod translation. // To ensure expected results of such operation we need to wait until newly created Service is in syncer controller cache, // otherwise syncer will operate on slightly outdated resources, which is not good for test stability.