Skip to content

Commit

Permalink
test: Added e2e tests for bidirectional annotation & label sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@ThomasK33
ThomasK33 committed Oct 11, 2024
1 parent aabc672 commit bdd22b9
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 31 deletions.
1 change: 0 additions & 1 deletion test/e2e/syncer/networkpolicies/networkpolicies.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ var _ = ginkgo.Describe("NetworkPolicies are created as expected", func() {
time.Sleep(time.Second * 10)
framework.DefaultFramework.TestServiceIsEventuallyUnreachable(curlPod, nginxService)
})

})

func updateNetworkPolicyWithRetryOnConflict(f *framework.Framework, networkPolicy *networkingv1.NetworkPolicy, mutator func(np *networkingv1.NetworkPolicy)) error {
Expand Down
84 changes: 81 additions & 3 deletions test/e2e/syncer/pods/pods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {

// version 1.22 and lesser than that needs legacy flag enabled
if version != nil {
i, err := strconv.Atoi(strings.Replace(version.Minor, "+", "", -1))
i, err := strconv.Atoi(strings.ReplaceAll(version.Minor, "+", ""))
framework.ExpectNoError(err)
if i > 22 {
vpod.Spec.EphemeralContainers = []corev1.EphemeralContainer{{
Expand Down Expand Up @@ -211,7 +211,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
},
Data: map[string]string{
cmKey: cmKeyValue,
}}, metav1.CreateOptions{})
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err)

pod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{
Expand Down Expand Up @@ -295,7 +296,8 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
},
Data: map[string][]byte{
secretKey: []byte(secretKeyValue),
}}, metav1.CreateOptions{})
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err)

pod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{
Expand Down Expand Up @@ -550,4 +552,80 @@ var _ = ginkgo.Describe("Pods are running in the host cluster", func() {
}
}
})

ginkgo.It("should perform a bidirectional sync on labels and annotations", func() {
podName := "test-annotations"
vPod, err := f.VClusterClient.CoreV1().Pods(ns).Create(f.Context, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Annotations: map[string]string{
"vcluster-annotation": "from vCluster with love",
},
Labels: map[string]string{
"vcluster-specific-label": "with_its_value",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: testingContainerName,
Image: testingContainerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: f.GetDefaultSecurityContext(),
},
},
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err)

err = f.WaitForPodRunning(podName, ns)
framework.ExpectNoError(err, "A pod created in the vcluster is expected to be in the Running phase eventually.")

// get current physical Pod resource
pPodName := translate.Default.HostName(nil, vPod.Name, vPod.Namespace)
pPod, err := f.HostClient.CoreV1().Pods(pPodName.Namespace).Get(f.Context, pPodName.Name, metav1.GetOptions{})
framework.ExpectNoError(err)

// get current vCluster Pod resource
vPod, err = f.VClusterClient.CoreV1().Pods(ns).Get(f.Context, podName, metav1.GetOptions{})
framework.ExpectNoError(err)

// update host cluster pod with additional information
additionalLabelKey := "another-one"
additionalLabelValue := "good-syncer"
additionalAnnotationKey := "annotation-key"
additionalAnnotationValue := "annotation-value"

err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) {
pPod.Labels[additionalLabelKey] = additionalLabelValue
pPod.Annotations[additionalAnnotationKey] = additionalAnnotationValue
pPod, err = f.HostClient.CoreV1().Pods(pPod.Namespace).Update(f.Context, pPod, metav1.UpdateOptions{})
if err != nil {
if kerrors.IsConflict(err) {
return false, nil
}
return false, err
}

return true, nil
})
framework.ExpectNoError(err)

// wait for the syncer to update the pod
err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(ctx context.Context) (bool, error) {
pod, err := f.VClusterClient.CoreV1().Pods(vPod.Namespace).Get(ctx, vPod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}

return pod.ResourceVersion != vPod.ResourceVersion && pod.Annotations[additionalAnnotationKey] != "", nil
})
framework.ExpectNoError(err)

vPod, err = f.VClusterClient.CoreV1().Pods(vPod.Namespace).Get(f.Context, vPod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)

framework.ExpectEqual(vPod.Annotations[additionalAnnotationKey], pPod.Annotations[additionalAnnotationKey])
framework.ExpectEqual(vPod.Labels[additionalLabelKey], pPod.Labels[additionalLabelKey])
})
})
136 changes: 111 additions & 25 deletions test/e2e/syncer/services/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ import (
"github.com/loft-sh/vcluster/test/framework"
"github.com/onsi/ginkgo/v2"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/cache"
Expand Down Expand Up @@ -157,20 +159,20 @@ var _ = ginkgo.Describe("Services are created as expected", func() {
defer cancel()
_, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) {
if svc, ok := event.Object.(*corev1.Service); ok {
found := svc.ObjectMeta.Name == testService.ObjectMeta.Name &&
svc.ObjectMeta.Namespace == ns &&
found := svc.Name == testService.Name &&
svc.Namespace == ns &&
svc.Labels["test-service-static"] == "true"
if !found {
f.Log.Infof("observed Service %v in namespace %v with labels: %v & ports %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Spec.Ports)
f.Log.Infof("observed Service %v in namespace %v with labels: %v & ports %v", svc.Name, svc.Namespace, svc.Labels, svc.Spec.Ports)
return false, nil
}
f.Log.Infof("Found Service %v in namespace %v with labels: %v & ports %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Spec.Ports)
f.Log.Infof("Found Service %v in namespace %v with labels: %v & ports %v", svc.Name, svc.Namespace, svc.Labels, svc.Spec.Ports)
return found, nil
}
f.Log.Infof("Observed event: %+v", event.Object)
return false, nil
})
framework.ExpectNoError(err, "Failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns)
framework.ExpectNoError(err, "Failed to locate Service %v in namespace %v", testService.Name, ns)
f.Log.Infof("Service %s created", testSvcName)

ginkgo.By("Getting /status")
Expand Down Expand Up @@ -202,14 +204,14 @@ var _ = ginkgo.Describe("Services are created as expected", func() {
defer cancel()
_, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) {
if svc, ok := event.Object.(*corev1.Service); ok {
found := svc.ObjectMeta.Name == testService.ObjectMeta.Name &&
svc.ObjectMeta.Namespace == ns &&
found := svc.Name == testService.Name &&
svc.Namespace == ns &&
svc.Annotations["patchedstatus"] == "true"
if !found {
f.Log.Infof("observed Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer)
f.Log.Infof("observed Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer)
return false, nil
}
f.Log.Infof("Found Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer)
f.Log.Infof("Found Service %v in namespace %v with annotations: %v & LoadBalancer: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer)
return found, nil
}
f.Log.Infof("Observed event: %+v", event.Object)
Expand Down Expand Up @@ -242,28 +244,28 @@ var _ = ginkgo.Describe("Services are created as expected", func() {
defer cancel()
_, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) {
if svc, ok := event.Object.(*corev1.Service); ok {
found := svc.ObjectMeta.Name == testService.ObjectMeta.Name &&
svc.ObjectMeta.Namespace == ns &&
found := svc.Name == testService.Name &&
svc.Namespace == ns &&
svc.Annotations["patchedstatus"] == "true"
if !found {
f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer)
f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer)
return false, nil
}
for _, cond := range svc.Status.Conditions {
if cond.Type == "StatusUpdate" &&
cond.Reason == "E2E" &&
cond.Message == "Set from e2e test" {
f.Log.Infof("Found Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.Conditions)
f.Log.Infof("Found Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.Conditions)
return found, nil
}
}
f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Annotations, svc.Status.LoadBalancer)
f.Log.Infof("Observed Service %v in namespace %v with annotations: %v & Conditions: %v", svc.Name, svc.Namespace, svc.Annotations, svc.Status.LoadBalancer)
return false, nil
}
f.Log.Infof("Observed event: %+v", event.Object)
return false, nil
})
framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns)
framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.Name, ns)
f.Log.Infof("Service %s has service status updated", testSvcName)

ginkgo.By("patching the service")
Expand All @@ -283,20 +285,20 @@ var _ = ginkgo.Describe("Services are created as expected", func() {
defer cancel()
_, err = watchtools.Until(ctx, svcList.ResourceVersion, w, func(event watch.Event) (bool, error) {
if svc, ok := event.Object.(*corev1.Service); ok {
found := svc.ObjectMeta.Name == testService.ObjectMeta.Name &&
svc.ObjectMeta.Namespace == ns &&
found := svc.Name == testService.Name &&
svc.Namespace == ns &&
svc.Labels["test-service"] == "patched"
if !found {
f.Log.Infof("observed Service %v in namespace %v with labels: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels)
f.Log.Infof("observed Service %v in namespace %v with labels: %v", svc.Name, svc.Namespace, svc.Labels)
return false, nil
}
f.Log.Infof("Found Service %v in namespace %v with labels: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels)
f.Log.Infof("Found Service %v in namespace %v with labels: %v", svc.Name, svc.Namespace, svc.Labels)
return found, nil
}
f.Log.Infof("Observed event: %+v", event.Object)
return false, nil
})
framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.ObjectMeta.Name, ns)
framework.ExpectNoError(err, "failed to locate Service %v in namespace %v", testService.Name, ns)
f.Log.Infof("Service %s patched", testSvcName)

// Delete service
Expand All @@ -309,22 +311,106 @@ var _ = ginkgo.Describe("Services are created as expected", func() {
switch event.Type {
case watch.Deleted:
if svc, ok := event.Object.(*corev1.Service); ok {
found := svc.ObjectMeta.Name == testService.ObjectMeta.Name &&
svc.ObjectMeta.Namespace == ns &&
found := svc.Name == testService.Name &&
svc.Namespace == ns &&
svc.Labels["test-service-static"] == "true"
if !found {
f.Log.Infof("observed Service %v in namespace %v with labels: %v & annotations: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Annotations)
f.Log.Infof("observed Service %v in namespace %v with labels: %v & annotations: %v", svc.Name, svc.Namespace, svc.Labels, svc.Annotations)
return false, nil
}
f.Log.Infof("Found Service %v in namespace %v with labels: %v & annotations: %v", svc.ObjectMeta.Name, svc.ObjectMeta.Namespace, svc.Labels, svc.Annotations)
f.Log.Infof("Found Service %v in namespace %v with labels: %v & annotations: %v", svc.Name, svc.Namespace, svc.Labels, svc.Annotations)
return found, nil
}
default:
f.Log.Infof("Observed event: %+v", event.Type)
}
return false, nil
})
framework.ExpectNoError(err, "failed to delete Service %v in namespace %v", testService.ObjectMeta.Name, ns)
framework.ExpectNoError(err, "failed to delete Service %v in namespace %v", testService.Name, ns)
f.Log.Infof("Service %s deleted", testSvcName)
})

ginkgo.It("should sync labels and annotation bidirectionally", func() {
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "myservice-with-annotations",
Namespace: ns,
Annotations: map[string]string{
"some-annotation": "that is set from the vCluster",
},
},
Spec: corev1.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "None",
},
}

vService, err := f.VClusterClient.CoreV1().Services(ns).Create(f.Context, service, metav1.CreateOptions{})
framework.ExpectNoError(err)
err = f.WaitForService(vService.Name, vService.Namespace)
framework.ExpectNoError(err)

// get physical service
pServiceName := translate.Default.HostName(nil, vService.Name, vService.Namespace)
pService, err := f.HostClient.CoreV1().Services(pServiceName.Namespace).Get(f.Context, pServiceName.Name, metav1.GetOptions{})
framework.ExpectNoError(err)

// update physical service
err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) {
pService.Annotations["some-annotation"] += " and update from the host cluster"
pService.Labels["host-cluster-label"] = "some_host_label_value"
pService, err = f.HostClient.CoreV1().Services(pServiceName.Namespace).Update(f.Context, pService, metav1.UpdateOptions{})
if err != nil {
if kerrors.IsConflict(err) {
return false, nil
}

return false, err
}

return true, nil
})
framework.ExpectNoError(err)

// wait for the change to be synced into the vCluster
err = f.WaitForServiceToUpdate(f.VClusterClient, vService.Name, vService.Namespace, vService.ResourceVersion)
framework.ExpectNoError(err)

// refetch the vCluster service object
vService, err = f.VClusterClient.CoreV1().Services(ns).Get(f.Context, vService.Name, metav1.GetOptions{})
framework.ExpectNoError(err)

// check that labels and annotations are the same
framework.ExpectEqual(vService.Annotations["some-annotation"], pService.Annotations["some-annotation"])
framework.ExpectEqual(vService.Labels["host-cluster-label"], pService.Labels["host-cluster-label"])

// update vCluster service
err = wait.PollUntilContextTimeout(f.Context, time.Second, framework.PollTimeout, true, func(context.Context) (bool, error) {
vService.Annotations["some-annotation"] += " and another update from the vCluster"
vService.Labels["vcluster-label"] = "some_vcluster_value"
vService, err = f.VClusterClient.CoreV1().Services(vService.Namespace).Update(f.Context, vService, metav1.UpdateOptions{})
if err != nil {
if kerrors.IsConflict(err) {
return false, nil
}

return false, err
}

return true, nil
})
framework.ExpectNoError(err)

// wait for the change to be synced into the host cluster
err = f.WaitForServiceToUpdate(f.HostClient, pService.Name, pService.Namespace, pService.ResourceVersion)
framework.ExpectNoError(err)

// refetch the host cluster service object
pService, err = f.HostClient.CoreV1().Services(pService.Namespace).Get(f.Context, pService.Name, metav1.GetOptions{})
framework.ExpectNoError(err)

// check that labels and annotations are the same
framework.ExpectEqual(vService.Annotations["some-annotation"], pService.Annotations["some-annotation"])
framework.ExpectEqual(vService.Labels["vcluster-label"], pService.Labels["vcluster-label"])
})
})
4 changes: 2 additions & 2 deletions test/framework/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func CreateFramework(ctx context.Context, scheme *runtime.Scheme) error {

suffix := os.Getenv("VCLUSTER_SUFFIX")
if suffix == "" {
//TODO: maybe implement some autodiscovery of the suffix value that would work with dev and prod setups
// TODO: maybe implement some autodiscovery of the suffix value that would work with dev and prod setups
suffix = "vcluster"
}
translate.VClusterName = suffix
Expand All @@ -131,7 +131,7 @@ func CreateFramework(ctx context.Context, scheme *runtime.Scheme) error {
translate.Default = translate.NewSingleNamespaceTranslator(ns)
}

l.Infof("Testing Vcluster named: %s in namespace: %s", name, ns)
l.Infof("Testing vCluster named: %s in namespace: %s", name, ns)

hostConfig, err := ctrl.GetConfig()
if err != nil {
Expand Down
17 changes: 17 additions & 0 deletions test/framework/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/ptr"
)

Expand Down Expand Up @@ -159,6 +160,22 @@ func (f *Framework) WaitForService(serviceName string, ns string) error {
})
}

// WaitForServiceToUpdate waits for a Kubernetes service to update by periodically fetching it using the provided client.
// It compares the current resource version of the service to the specified version and returns when they are different.
func (f *Framework) WaitForServiceToUpdate(client *kubernetes.Clientset, serviceName string, ns string, resourceVersion string) error {
return wait.PollUntilContextTimeout(f.Context, time.Second, PollTimeout, true, func(ctx context.Context) (bool, error) {
svc, err := client.CoreV1().Services(ns).Get(ctx, serviceName, metav1.GetOptions{})
if err != nil {
if kerrors.IsNotFound(err) {
return false, nil
}
return false, err
}

return svc.ResourceVersion != resourceVersion, nil
})
}

// Some vcluster operations list Service, e.g. pod translation.
// To ensure expected results of such operation we need to wait until newly created Service is in syncer controller cache,
// otherwise syncer will operate on slightly outdated resources, which is not good for test stability.
Expand Down

0 comments on commit bdd22b9

Please sign in to comment.