From b66ef882f6b3443a6b2b49bc05c16b475ed6279a Mon Sep 17 00:00:00 2001 From: Yu-Lin Chen Date: Fri, 12 Jan 2024 13:05:05 +0800 Subject: [PATCH] Remove unrelated e2e test --- .../admission_controller_suite_test.go | 238 ---- .../admission_controller_test.go | 491 -------- .../basic_scheduling_suite_test.go | 58 - .../basic_scheduling/basic_scheduling_test.go | 133 -- .../e2e/bin_packing/bin_packing_suite_test.go | 131 -- test/e2e/bin_packing/bin_packing_test.go | 200 --- .../gang_scheduling_suite_test.go | 91 -- .../gang_scheduling/gang_scheduling_test.go | 758 ------------ .../node_resources_suite_test.go | 85 -- .../e2e/node_resources/node_resources_test.go | 92 -- .../persistent_volume_suite_test.go | 50 - .../persistent_volume_test.go | 352 ------ test/e2e/predicates/predicates_suite_test.go | 98 -- test/e2e/predicates/predicates_test.go | 1099 ----------------- .../priority_scheduling_suite_test.go | 141 --- .../priority_scheduling_test.go | 515 -------- .../queue_quota_mgmt_suite_test.go | 94 -- .../queue_quota_mgmt/queue_quota_mgmt_test.go | 260 ---- .../recovery_and_restart_suite_test.go | 52 - .../recovery_and_restart_test.go | 389 ------ .../resource_fairness_suite_test.go | 86 -- .../resource_fairness_test.go | 202 --- .../simple_preemptor_suite_test.go | 50 - .../simple_preemptor/simple_preemptor_test.go | 247 ---- .../spark_jobs_scheduling_suite_test.go | 88 -- .../spark_jobs_scheduling_test.go | 176 --- .../drip_feed_schedule_test.go | 153 --- .../fallback_test.go | 118 -- .../state_aware_app_scheduling_suite_test.go | 96 -- .../user_group_limit_suite_test.go | 50 - .../user_group_limit/user_group_limit_test.go | 658 ---------- 31 files changed, 7251 deletions(-) delete mode 100644 test/e2e/admission_controller/admission_controller_suite_test.go delete mode 100644 test/e2e/admission_controller/admission_controller_test.go delete mode 100644 test/e2e/basic_scheduling/basic_scheduling_suite_test.go delete mode 100644 test/e2e/basic_scheduling/basic_scheduling_test.go delete mode 100644 test/e2e/bin_packing/bin_packing_suite_test.go delete mode 100644 test/e2e/bin_packing/bin_packing_test.go delete mode 100644 test/e2e/gang_scheduling/gang_scheduling_suite_test.go delete mode 100644 test/e2e/gang_scheduling/gang_scheduling_test.go delete mode 100644 test/e2e/node_resources/node_resources_suite_test.go delete mode 100644 test/e2e/node_resources/node_resources_test.go delete mode 100644 test/e2e/persistent_volume/persistent_volume_suite_test.go delete mode 100644 test/e2e/persistent_volume/persistent_volume_test.go delete mode 100644 test/e2e/predicates/predicates_suite_test.go delete mode 100644 test/e2e/predicates/predicates_test.go delete mode 100644 test/e2e/priority_scheduling/priority_scheduling_suite_test.go delete mode 100644 test/e2e/priority_scheduling/priority_scheduling_test.go delete mode 100644 test/e2e/queue_quota_mgmt/queue_quota_mgmt_suite_test.go delete mode 100644 test/e2e/queue_quota_mgmt/queue_quota_mgmt_test.go delete mode 100644 test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go delete mode 100644 test/e2e/recovery_and_restart/recovery_and_restart_test.go delete mode 100644 test/e2e/resource_fairness/resource_fairness_suite_test.go delete mode 100644 test/e2e/resource_fairness/resource_fairness_test.go delete mode 100644 test/e2e/simple_preemptor/simple_preemptor_suite_test.go delete mode 100644 test/e2e/simple_preemptor/simple_preemptor_test.go delete mode 100644 test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_suite_test.go delete mode 100644 test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_test.go delete mode 100644 test/e2e/state_aware_app_scheduling/drip_feed_schedule_test.go delete mode 100644 test/e2e/state_aware_app_scheduling/fallback_test.go delete mode 100644 test/e2e/state_aware_app_scheduling/state_aware_app_scheduling_suite_test.go delete mode 100644 test/e2e/user_group_limit/user_group_limit_suite_test.go delete mode 100644 test/e2e/user_group_limit/user_group_limit_test.go diff --git a/test/e2e/admission_controller/admission_controller_suite_test.go b/test/e2e/admission_controller/admission_controller_suite_test.go deleted file mode 100644 index 047ac12c3..000000000 --- a/test/e2e/admission_controller/admission_controller_suite_test.go +++ /dev/null @@ -1,238 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package admission_controller_test - -import ( - "fmt" - "path/filepath" - "runtime" - "testing" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - v1 "k8s.io/api/core/v1" - schedulingv1 "k8s.io/api/scheduling/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -const appName = "sleep" - -var suiteName string -var kubeClient k8s.KubeCtl -var ns string -var bypassNs = "kube-system" -var restClient yunikorn.RClient -var oldConfigMap = new(v1.ConfigMap) -var one = int32(1) -var preemptPolicyNever = v1.PreemptNever -var preemptPolicyPreemptLower = v1.PreemptLowerPriority -var annotation = "ann-" + common.RandSeq(10) - -var testPod = v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sleepjob", - Labels: map[string]string{"app": appName}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "sleepjob", - Image: "alpine:latest", - Command: []string{"sleep", "30"}, - }, - }, - }, -} - -var testDeployment = appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Replicas: &one, - Selector: &metav1.LabelSelector{ - MatchLabels: testPod.Labels, - }, - Template: getPodSpec(v1.RestartPolicyAlways), - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testJob = batchv1.Job{ - Spec: batchv1.JobSpec{ - Parallelism: &one, - Completions: &one, - Template: getPodSpec(v1.RestartPolicyNever), - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testStatefulSet = appsv1.StatefulSet{ - Spec: appsv1.StatefulSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: testPod.Labels, - }, - Template: getPodSpec(v1.RestartPolicyAlways), - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testReplicaSet = appsv1.ReplicaSet{ - Spec: appsv1.ReplicaSetSpec{ - Replicas: &one, - Selector: &metav1.LabelSelector{ - MatchLabels: testPod.Labels, - }, - Template: getPodSpec(v1.RestartPolicyAlways), - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testDaemonSet = appsv1.DaemonSet{ - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: testPod.Labels, - }, - Template: getPodSpec(v1.RestartPolicyAlways), - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testCronJob = batchv1.CronJob{ - Spec: batchv1.CronJobSpec{ - Schedule: "* * * * *", - JobTemplate: batchv1.JobTemplateSpec{ - Spec: batchv1.JobSpec{ - Parallelism: &one, - Completions: &one, - Template: getPodSpec(v1.RestartPolicyNever), - }, - ObjectMeta: testPod.ObjectMeta, - }, - }, - ObjectMeta: testPod.ObjectMeta, -} - -var testPreemptPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-preempt", - Annotations: map[string]string{constants.AnnotationAllowPreemption: constants.True}, - }, - Value: 2000, - PreemptionPolicy: &preemptPolicyPreemptLower, -} - -var testNonPreemptPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-non-preempt", - Annotations: map[string]string{constants.AnnotationAllowPreemption: constants.False}, - }, - Value: 1000, - PreemptionPolicy: &preemptPolicyNever, -} - -var testNonYkPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-non-yk", - }, - Value: 1500, - PreemptionPolicy: &preemptPolicyPreemptLower, -} - -func getPodSpec(restartPolicy v1.RestartPolicy) v1.PodTemplateSpec { - p := testPod.DeepCopy() - p.Spec.RestartPolicy = restartPolicy - return v1.PodTemplateSpec{ - ObjectMeta: p.ObjectMeta, - Spec: p.Spec, - } -} - -func TestAdmissionController(t *testing.T) { - ReportAfterSuite("TestAdmissionController", func(report Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-admission_controller_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - RegisterFailHandler(Fail) - RunSpecs(t, "Admission Controller Suite") -} - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - restClient = yunikorn.RClient{} - - kubeClient = k8s.KubeCtl{} - Expect(kubeClient.SetClient()).To(BeNil()) - - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) - - By("Port-forward the scheduler pod") - err := kubeClient.PortForwardYkSchedulerPod() - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Creating priority class %s", testPreemptPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&testPreemptPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Creating priority class %s", testNonPreemptPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&testNonPreemptPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Creating priority class %s", testNonYkPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&testNonYkPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) -}) - -var _ = AfterSuite(func() { - kubeClient = k8s.KubeCtl{} - Expect(kubeClient.SetClient()).To(BeNil()) - - By(fmt.Sprintf("Removing priority class %s", testNonYkPriorityClass.Name)) - err := kubeClient.DeletePriorityClass(testNonYkPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Removing priority class %s", testNonPreemptPriorityClass.Name)) - err = kubeClient.DeletePriorityClass(testNonPreemptPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Removing priority class %s", testPreemptPriorityClass.Name)) - err = kubeClient.DeletePriorityClass(testPreemptPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) diff --git a/test/e2e/admission_controller/admission_controller_test.go b/test/e2e/admission_controller/admission_controller_test.go deleted file mode 100644 index 82097f6ac..000000000 --- a/test/e2e/admission_controller/admission_controller_test.go +++ /dev/null @@ -1,491 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package admission_controller_test - -import ( - "fmt" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - amConf "github.com/apache/yunikorn-k8shim/pkg/admission/conf" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -const userInfoAnnotation = constants.DomainYuniKorn + "user.info" -const nonExistentNode = "non-existent-node" -const defaultPodTimeout = 10 * time.Second -const cronJobPodTimeout = 65 * time.Second - -var _ = ginkgo.Describe("AdmissionController", func() { - ginkgo.BeforeEach(func() { - kubeClient = k8s.KubeCtl{} - gomega.Expect(kubeClient.SetClient()).To(gomega.BeNil()) - ns = "ns-" + common.RandSeq(10) - ginkgo.By(fmt.Sprintf("Creating namespace: %s for admission controller tests", ns)) - var ns1, err1 = kubeClient.CreateNamespace(ns, nil) - gomega.Ω(err1).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns1.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - }) - - ginkgo.It("Verifying pod with preempt priority class", func() { - ginkgo.By("has correct properties set") - podCopy := testPod.DeepCopy() - podCopy.Name = "preempt-pod" - podCopy.Spec.PriorityClassName = testPreemptPriorityClass.Name - podCopy.Spec.NodeName = nonExistentNode - pod, err := kubeClient.CreatePod(podCopy, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - gomega.Ω(*pod.Spec.Priority).Should(gomega.Equal(testPreemptPriorityClass.Value)) - gomega.Ω(*pod.Spec.PreemptionPolicy).Should(gomega.Equal(*testPreemptPriorityClass.PreemptionPolicy)) - - value, ok := pod.Annotations[constants.AnnotationAllowPreemption] - gomega.Ω(ok).Should(gomega.BeTrue()) - gomega.Ω(value).Should(gomega.Equal(constants.True)) - }) - - ginkgo.It("Verifying pod with non-preempt priority class", func() { - ginkgo.By("has correct properties set") - podCopy := testPod.DeepCopy() - podCopy.Name = nonExistentNode - podCopy.Spec.PriorityClassName = testNonPreemptPriorityClass.Name - podCopy.Spec.NodeName = nonExistentNode - pod, err := kubeClient.CreatePod(podCopy, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - gomega.Ω(*pod.Spec.Priority).Should(gomega.Equal(testNonPreemptPriorityClass.Value)) - gomega.Ω(*pod.Spec.PreemptionPolicy).Should(gomega.Equal(*testNonPreemptPriorityClass.PreemptionPolicy)) - - value, ok := pod.Annotations[constants.AnnotationAllowPreemption] - gomega.Ω(ok).Should(gomega.BeTrue()) - gomega.Ω(value).Should(gomega.Equal(constants.False)) - }) - - ginkgo.It("Verifying pod with non-YK priority class", func() { - ginkgo.By("has correct properties set") - podCopy := testPod.DeepCopy() - podCopy.Name = "non-yk-pod" - podCopy.Spec.PriorityClassName = testNonYkPriorityClass.Name - podCopy.Spec.NodeName = nonExistentNode - pod, err := kubeClient.CreatePod(podCopy, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - gomega.Ω(*pod.Spec.Priority).Should(gomega.Equal(testNonYkPriorityClass.Value)) - gomega.Ω(*pod.Spec.PreemptionPolicy).Should(gomega.Equal(*testNonYkPriorityClass.PreemptionPolicy)) - - value, ok := pod.Annotations[constants.AnnotationAllowPreemption] - gomega.Ω(ok).Should(gomega.BeTrue()) - gomega.Ω(value).Should(gomega.Equal(constants.True)) - }) - - ginkgo.It("Verifying pod with no priority class", func() { - ginkgo.By("has correct properties set") - podCopy := testPod.DeepCopy() - podCopy.Name = "no-priority" - podCopy.Spec.NodeName = nonExistentNode - pod, err := kubeClient.CreatePod(podCopy, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - value, ok := pod.Annotations[constants.AnnotationAllowPreemption] - gomega.Ω(ok).Should(gomega.BeTrue()) - gomega.Ω(value).Should(gomega.Equal(constants.True)) - }) - - ginkgo.It("Verifying a pod is created in the test namespace", func() { - ginkgo.By("has 1 running pod whose SchedulerName is yunikorn") - pod, err := kubeClient.CreatePod(&testPod, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - // Wait for pod to move into running state - err = kubeClient.WaitForPodBySelectorRunning(ns, - fmt.Sprintf("app=%s", appName), 10) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - gomega.Ω(pod.Spec.SchedulerName).Should(gomega.BeEquivalentTo(constants.SchedulerName)) - }) - - ginkgo.It("Verifying a pod is created in the bypass namespace", func() { - ginkgo.By("Create a pod in the bypass namespace") - pod, err := kubeClient.CreatePod(&testPod, bypassNs) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, bypassNs) - - err = kubeClient.WaitForPodBySelectorRunning(bypassNs, - fmt.Sprintf("app=%s", appName), 10) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - gomega.Ω(pod.Spec.SchedulerName).ShouldNot(gomega.BeEquivalentTo(constants.SchedulerName)) - - }) - - ginkgo.It("Verifying the scheduler configuration is overridden", func() { - invalidConfigMap := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ConfigMapName, - Namespace: configmanager.YuniKornTestConfig.YkNamespace, - }, - Data: make(map[string]string), - } - - res, err := restClient.ValidateSchedulerConfig(invalidConfigMap) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - gomega.Ω(res.Allowed).Should(gomega.BeEquivalentTo(false)) - }) - - ginkgo.It("Configure the scheduler with an empty configmap", func() { - emptyConfigMap := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ConfigMapName, - Namespace: configmanager.YuniKornTestConfig.YkNamespace, - }, - Data: make(map[string]string), - } - cm, err := kubeClient.UpdateConfigMap(&emptyConfigMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - gomega.Ω(cm).ShouldNot(gomega.BeNil()) - }) - - ginkgo.It("Configure the scheduler with invalid configmap", func() { - invalidConfigMap := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ConfigMapName, - Namespace: configmanager.YuniKornTestConfig.YkNamespace, - }, - Data: map[string]string{"queues.yaml": "invalid"}, - } - invalidCm, err := kubeClient.UpdateConfigMap(&invalidConfigMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).Should(gomega.HaveOccurred()) - gomega.Ω(invalidCm).ShouldNot(gomega.BeNil()) - }) - - ginkgo.It("Check that annotation is added to a pod & cannot be modified", func() { - ginkgo.By("Create a pod") - pod, err := kubeClient.CreatePod(&testPod, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer deletePod(pod, ns) - - err = kubeClient.WaitForPodBySelector(ns, - fmt.Sprintf("app=%s", appName), 10*time.Second) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - pod, err = kubeClient.GetPod(pod.Name, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - userinfo := pod.Annotations[constants.DomainYuniKorn+"user.info"] - gomega.Ω(userinfo).Should(gomega.Not(gomega.BeNil())) - - ginkgo.By("Attempt to update userinfo annotation") - _, err = kubeClient.UpdatePodWithAnnotation(pod, ns, constants.DomainYuniKorn+"user.info", "shouldnotsucceed") - gomega.Ω(err).Should(gomega.HaveOccurred()) - }) - - ginkgo.It("Check that annotation is added to a deployment", func() { - create := func() (string, error) { - dep, err := kubeClient.CreateDeployment(&testDeployment, ns) - name := "" - if dep != nil { - name = dep.Name - } - return name, err - } - - runWorkloadTest(k8s.Deployment, create, defaultPodTimeout) - }) - - ginkgo.It("Check that annotation is added to a StatefulSet", func() { - create := func() (string, error) { - sfs, err := kubeClient.CreateStatefulSet(&testStatefulSet, ns) - name := "" - if sfs != nil { - name = sfs.Name - } - return name, err - } - - runWorkloadTest(k8s.StatefulSet, create, defaultPodTimeout) - }) - - ginkgo.It("Check that annotation is added to a DaemonSet", func() { - create := func() (string, error) { - ds, err := kubeClient.CreateDaemonSet(&testDaemonSet, ns) - name := "" - if ds != nil { - name = ds.Name - } - return name, err - } - - runWorkloadTest(k8s.DaemonSet, create, defaultPodTimeout) - }) - - ginkgo.It("Check that annotation is added to a ReplicaSet", func() { - create := func() (string, error) { - rs, err := kubeClient.CreateReplicaSet(&testReplicaSet, ns) - name := "" - if rs != nil { - name = rs.Name - } - return name, err - } - - runWorkloadTest(k8s.ReplicaSet, create, defaultPodTimeout) - }) - - ginkgo.It("Check that annotation is added to a Job", func() { - create := func() (string, error) { - job, err := kubeClient.CreateJob(&testJob, ns) - name := "" - if job != nil { - name = job.Name - } - return name, err - } - - runWorkloadTest(k8s.Job, create, defaultPodTimeout) - }) - - ginkgo.It("Check that annotation is added to a CronJob", func() { - create := func() (string, error) { - cj, err := kubeClient.CreateCronJob(&testCronJob, ns) - name := "" - if cj != nil { - name = cj.Name - } - return name, err - } - - runWorkloadTest(k8s.CronJob, create, cronJobPodTimeout) - }) - - ginkgo.It("Check that deployment is rejected when controller users are not trusted", func() { - ginkgo.By("Retrieve existing configmap") - configMap, err := kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - if configMap.Data == nil { - configMap.Data = make(map[string]string) - } - configMap.Data[amConf.AMAccessControlTrustControllers] = "false" - ginkgo.By("Update configmap") - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.By("Create a deployment") - deployment, err2 := kubeClient.CreateDeployment(&testDeployment, ns) - gomega.Ω(err2).ShouldNot(gomega.HaveOccurred()) - defer kubeClient.DeleteWorkloadAndPods(deployment.Name, k8s.Deployment, ns) - - // pod is not expected to appear - ginkgo.By("Check for sleep pods (should time out)") - err = kubeClient.WaitForPodBySelector(ns, fmt.Sprintf("app=%s", testDeployment.ObjectMeta.Labels["app"]), - 10*time.Second) - fmt.Fprintf(ginkgo.GinkgoWriter, "Error: %v\n", err) - gomega.Ω(err).Should(gomega.HaveOccurred()) - ginkgo.By("Check deployment status") - deployment, err = kubeClient.GetDeployment(testDeployment.Name, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - fmt.Fprintf(ginkgo.GinkgoWriter, "Replicas: %d, AvailableReplicas: %d, ReadyReplicas: %d\n", - deployment.Status.Replicas, deployment.Status.AvailableReplicas, deployment.Status.ReadyReplicas) - gomega.Ω(deployment.Status.Replicas).To(gomega.Equal(int32(0))) - gomega.Ω(deployment.Status.AvailableReplicas).To(gomega.Equal(int32(0))) - gomega.Ω(deployment.Status.ReadyReplicas).To(gomega.Equal(int32(0))) - - // restore setting - ginkgo.By("Restore trustController setting") - configMap, err = kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - configMap.Data[amConf.AMAccessControlTrustControllers] = "true" - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - // pod is expected to appear - ginkgo.By("Check for sleep pod") - err = kubeClient.WaitForPodBySelector(ns, fmt.Sprintf("app=%s", testDeployment.ObjectMeta.Labels["app"]), - 60*time.Second) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.It("Check that deployment is rejected when external user is not trusted", func() { - ginkgo.By("Retrieve existing configmap") - configMap, err := kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - if configMap.Data == nil { - configMap.Data = make(map[string]string) - } - configMap.Data[amConf.AMAccessControlExternalUsers] = "" - ginkgo.By("Update configmap") - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.By("Create a deployment") - deployment := testDeployment.DeepCopy() - deployment.Spec.Template.Annotations = make(map[string]string) - deployment.Spec.Template.Annotations[userInfoAnnotation] = "{\"user\":\"test\",\"groups\":[\"devops\",\"system:authenticated\"]}" - _, err = kubeClient.CreateDeployment(deployment, ns) - fmt.Fprintf(ginkgo.GinkgoWriter, "Error received from API server: %v\n", err) - gomega.Ω(err).Should(gomega.HaveOccurred()) - gomega.Ω(err).To(gomega.BeAssignableToTypeOf(&errors.StatusError{})) - - // modify setting - ginkgo.By("Changing allowed externalUser setting") - configMap, err = kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - configMap.Data[amConf.AMAccessControlExternalUsers] = "(^minikube-user$|^kubernetes-admin$)" // works with Minikube & KIND - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - - }) - - // submit deployment again - ginkgo.By("Submit deployment again") - _, err = kubeClient.CreateDeployment(deployment, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer kubeClient.DeleteWorkloadAndPods(deployment.Name, k8s.Deployment, ns) - - // pod is expected to appear - ginkgo.By("Check for sleep pod") - err = kubeClient.WaitForPodBySelector(ns, fmt.Sprintf("app=%s", testDeployment.ObjectMeta.Labels["app"]), - 60*time.Second) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.It("Check that replicaset is not modified if submitted by system user", func() { - ginkgo.By("Retrieve existing configmap") - configMap, err := kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - if configMap.Data == nil { - configMap.Data = make(map[string]string) - } - configMap.Data[amConf.AMAccessControlBypassAuth] = "true" - ginkgo.By("Update configmap (bypassAuth -> true)") - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.By("Submit a deployment") - deployment := testDeployment.DeepCopy() - _, err = kubeClient.CreateDeployment(deployment, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer kubeClient.DeleteWorkloadAndPods(deployment.Name, k8s.Deployment, ns) - ginkgo.By("Check for sleep pod") - err = kubeClient.WaitForPodBySelector(ns, fmt.Sprintf("app=%s", deployment.ObjectMeta.Labels["app"]), - 60*time.Second) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - - ginkgo.By("Update configmap (bypassAuth -> false)") - configMap, err = kubeClient.GetConfigMap(constants.ConfigMapName, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - configMap.Data[amConf.AMAccessControlBypassAuth] = "false" - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - _, err = kubeClient.UpdateConfigMap(configMap, configmanager.YuniKornTestConfig.YkNamespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - }) - - ginkgo.By("Update container image in deployment") - deployment, err = kubeClient.GetDeployment(deployment.Name, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - container := deployment.Spec.Template.Spec.Containers[0] - container.Image = "nginx:1.16.1" - deployment.Spec.Template.Spec.Containers[0] = container - _, err = kubeClient.UpdateDeployment(deployment, ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - - ginkgo.By("Wait for sleep pod") - err = kubeClient.WaitForPodBySelectorRunning(ns, fmt.Sprintf("app=%s", testDeployment.ObjectMeta.Labels["app"]), - 60) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - - ginkgo.By("Check for number of replicasets") - replicaSetList, err2 := kubeClient.GetReplicaSets(ns) - gomega.Ω(err2).ShouldNot(gomega.HaveOccurred()) - for _, rs := range replicaSetList.Items { - fmt.Fprintf(ginkgo.GinkgoWriter, "%-20s\tReplicas: %d\tReady: %d\tAvailable: %d\n", - rs.Name, - rs.Status.Replicas, - rs.Status.ReadyReplicas, - rs.Status.AvailableReplicas) - } - gomega.Ω(len(replicaSetList.Items)).To(gomega.Equal(2)) - }) - - ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - ginkgo.By("Tear down namespace: " + ns) - err := kubeClient.TearDownNamespace(ns) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - // call the healthCheck api to check scheduler health - ginkgo.By("Check YuniKorn's health") - checks, err2 := yunikorn.GetFailedHealthChecks() - gomega.Ω(err2).ShouldNot(gomega.HaveOccurred()) - gomega.Ω(checks).Should(gomega.Equal(""), checks) - }) -}) - -func runWorkloadTest(workloadType k8s.WorkloadType, create func() (string, error), - podTimeout time.Duration) { - ginkgo.By("Create a " + string(workloadType)) - name, err := create() - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - defer kubeClient.DeleteWorkloadAndPods(name, workloadType, ns) - err = kubeClient.WaitForPodBySelector(ns, "app="+appName, podTimeout) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - - ginkgo.By("Get at least one running pod") - var pods *v1.PodList - pods, err = kubeClient.GetPods(ns) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - fmt.Fprintf(ginkgo.GinkgoWriter, "Running pod is %s\n", pods.Items[0].Name) - pod, err2 := kubeClient.GetPod(pods.Items[0].Name, ns) - gomega.Ω(err2).ShouldNot(gomega.HaveOccurred()) - userinfo := pod.Annotations[constants.DomainYuniKorn+"user.info"] - gomega.Ω(userinfo).Should(gomega.Not(gomega.BeNil())) -} - -func deletePod(pod *v1.Pod, namespace string) { - if pod != nil { - ginkgo.By("Delete pod " + pod.Name) - err := kubeClient.DeletePod(pod.Name, namespace) - gomega.Ω(err).ShouldNot(gomega.HaveOccurred()) - } -} diff --git a/test/e2e/basic_scheduling/basic_scheduling_suite_test.go b/test/e2e/basic_scheduling/basic_scheduling_suite_test.go deleted file mode 100644 index ad560005f..000000000 --- a/test/e2e/basic_scheduling/basic_scheduling_suite_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package basicscheduling_test - -import ( - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/v2/reporters" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestBasicScheduling(t *testing.T) { - ginkgo.ReportAfterSuite("TestBasicScheduling", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-basic_scheduling_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestBasicScheduling", ginkgo.Label("TestBasicScheduling")) -} - -var By = ginkgo.By - -var Ω = gomega.Ω -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred -var BeEquivalentTo = gomega.BeEquivalentTo diff --git a/test/e2e/basic_scheduling/basic_scheduling_test.go b/test/e2e/basic_scheduling/basic_scheduling_test.go deleted file mode 100644 index cca1674b9..000000000 --- a/test/e2e/basic_scheduling/basic_scheduling_test.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package basicscheduling_test - -import ( - "runtime" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var suiteName string -var kClient k8s.KubeCtl -var restClient yunikorn.RClient -var sleepRespPod *v1.Pod -var dev = "dev" + common.RandSeq(5) -var appsInfo *dao.ApplicationDAOInfo -var annotation = "ann-" + common.RandSeq(10) -var oldConfigMap = new(v1.ConfigMap) - -// Define sleepPod -var sleepPodConfigs = k8s.SleepPodConfig{Name: "sleepjob", NS: dev} - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - // Initializing kubectl client - kClient = k8s.KubeCtl{} - gomega.Ω(kClient.SetClient()).To(gomega.BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - - yunikorn.EnsureYuniKornConfigsPresent() - - By("Port-forward the scheduler pod") - err := kClient.PortForwardYkSchedulerPod() - Ω(err).NotTo(HaveOccurred()) - - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo", annotation) - - ginkgo.By("create development namespace") - ns1, err := kClient.CreateNamespace(dev, nil) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns1.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - - ginkgo.By("Deploy the sleep pod to the development namespace") - initPod, podErr := k8s.InitSleepPod(sleepPodConfigs) - gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod, err = kClient.CreatePod(initPod, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - // Wait for pod to move to running state - err = kClient.WaitForPodRunning(dev, sleepPodConfigs.Name, 30*time.Second) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - appsInfo, err = restClient.GetAppInfo("default", "root."+dev, sleepRespPod.ObjectMeta.Labels["applicationId"]) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(appsInfo).NotTo(gomega.BeNil()) -}) - -var _ = ginkgo.AfterSuite(func() { - ginkgo.By("Tear down namespace: " + dev) - err := kClient.TearDownNamespace(dev) - Ω(err).NotTo(HaveOccurred()) - - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -var _ = ginkgo.Describe("", func() { - - ginkgo.It("Verify_App_Queue_Info", func() { - ginkgo.By("Verify that the sleep pod is mapped to development queue") - gomega.Ω(appsInfo.ApplicationID).To(gomega.Equal(sleepRespPod.ObjectMeta.Labels["applicationId"])) - gomega.Ω(appsInfo.QueueName).To(gomega.ContainSubstring(sleepRespPod.ObjectMeta.Namespace)) - }) - - ginkgo.It("Verify_Job_State", func() { - ginkgo.By("Verify that the job is scheduled & starting by YuniKorn") - gomega.Ω(appsInfo.State).To(gomega.Equal("Starting")) - gomega.Ω("yunikorn").To(gomega.Equal(sleepRespPod.Spec.SchedulerName)) - }) - - ginkgo.It("Verify_Pod_Alloc_Props", func() { - ginkgo.By("Verify the pod allocation properties") - gomega.Ω(appsInfo.Allocations).NotTo(gomega.BeNil()) - gomega.Ω(len(appsInfo.Allocations)).NotTo(gomega.BeZero()) - allocation := appsInfo.Allocations[0] - gomega.Ω(allocation).NotTo(gomega.BeNil()) - gomega.Ω(allocation.AllocationKey).NotTo(gomega.BeNil()) - gomega.Ω(allocation.NodeID).NotTo(gomega.BeNil()) - gomega.Ω(allocation.Partition).NotTo(gomega.BeNil()) - gomega.Ω(allocation.AllocationID).NotTo(gomega.BeNil()) - gomega.Ω(allocation.ApplicationID).To(gomega.Equal(sleepRespPod.ObjectMeta.Labels["applicationId"])) - core := sleepRespPod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue() - mem := sleepRespPod.Spec.Containers[0].Resources.Requests.Memory().Value() - resMap := allocation.ResourcePerAlloc - Ω(len(resMap)).NotTo(gomega.BeZero()) - Ω(resMap["memory"]).To(gomega.Equal(mem)) - Ω(resMap["vcore"]).To(gomega.Equal(core)) - }) - - ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev}) - // call the healthCheck api to check scheduler health - ginkgo.By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(HaveOccurred()) - Ω(checks).To(gomega.Equal(""), checks) - }) -}) diff --git a/test/e2e/bin_packing/bin_packing_suite_test.go b/test/e2e/bin_packing/bin_packing_suite_test.go deleted file mode 100644 index 9bc30b876..000000000 --- a/test/e2e/bin_packing/bin_packing_suite_test.go +++ /dev/null @@ -1,131 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package bin_packing - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-core/pkg/common/configs" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestBinPacking(t *testing.T) { - ginkgo.ReportAfterSuite("TestBinPacking", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-bin_packing_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestBinPacking", ginkgo.Label("TestBinPacking")) -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) -var kClient = k8s.KubeCtl{} //nolint - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - - Ω(kClient.SetClient()).To(BeNil()) - /* Sample configMap. Post-update, Yunikorn will use binpacking node sort and fair app sort - partitions: - - name: default - queues: - - name: root - nodesortpolicy: - binpacking - properties: - application.sort.policy: fifo - timestamp: "1619566965" - submitacl: '*' - placementrules: - - name: tag - create: true - value: namespace - */ - By("Enabling new scheduling config with binpacking node sort policy") - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "fifo", annotation, func(sc *configs.SchedulerConfig) error { - setErr := common.SetSchedulingPolicy(sc, "default", "root", "fifo") - if setErr != nil { - return setErr - } - setErr = common.SetNodeSortPolicy(sc, "default", configs.NodeSortingPolicy{Type: "binpacking"}) - if setErr != nil { - return setErr - } - _, tsErr := common.SetQueueTimestamp(sc, "default", "root") - if tsErr != nil { - return tsErr - } - _, yamlErr := common.ToYAML(sc) - if yamlErr != nil { - return yamlErr - } - return nil - }) - - // Restart yunikorn and port-forward - // Required to change node sort policy. - ginkgo.By("Restart the scheduler pod") - yunikorn.RestartYunikorn(&kClient) - - ginkgo.By("Port-forward scheduler pod after restart") - yunikorn.RestorePortForwarding(&kClient) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -var Describe = ginkgo.Describe -var It = ginkgo.It -var By = ginkgo.By -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var BeNumerically = gomega.BeNumerically -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/bin_packing/bin_packing_test.go b/test/e2e/bin_packing/bin_packing_test.go deleted file mode 100644 index 43b8ec6c8..000000000 --- a/test/e2e/bin_packing/bin_packing_test.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package bin_packing - -import ( - "fmt" - "sort" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" -) - -var _ = Describe("", func() { - var ns string - - BeforeEach(func() { - ns = "ns-" + common.RandSeq(10) - By(fmt.Sprintf("Create namespace: %s", ns)) - var ns1, err1 = kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - // Verify BinPacking Node Order Memory - // 1. Sort nodes by increasing available memory: [nodeA, nodeB...] - // 2. Increase nodeA by 20% of available memory. Add node=nodeA pod label. - // 3. Increase nodeB by 10% of available memory - // 4. Submit jobA of 3 pods. Verify all are allocated to nodeA - // 5. Submit jobB of 3 pods with anti-affinity of node=nodeA. Verify all are allocated to nodeB - It("Verify_BinPacking_Node_Order_Memory", func() { - domRes := v1.ResourceMemory - nodes, err := kClient.GetNodes() - Ω(err).NotTo(HaveOccurred()) - - // Sort nodes by available dominant resource in increasing order - nodesAvailRes := kClient.GetNodesAvailRes(*nodes) - sortedWorkerNodes := k8s.GetWorkerNodes(*nodes) - sort.Slice(sortedWorkerNodes, func(i, j int) bool { - node1, node2 := sortedWorkerNodes[i].Name, sortedWorkerNodes[j].Name - q1, q2 := nodesAvailRes[node1][domRes], nodesAvailRes[node2][domRes] - return q1.Cmp(q2) == -1 - }) - var nodeNames []string - for _, node := range sortedWorkerNodes { - nodeNames = append(nodeNames, node.Name) - } - By(fmt.Sprintf("Sorted nodes by available memory: %v", nodeNames)) - - Ω(len(sortedWorkerNodes)).To(BeNumerically(">=", 2), - "At least 2 nodes required") - - // Submit pods to nodeA/nodeB to stabilize node order - padPct := []float64{0.2, 0.1} - for i := 0; i < 2; i++ { - // Determine memory padding amount - nodeName := sortedWorkerNodes[i].Name - nodeAvailMem := nodesAvailRes[nodeName][domRes] - memPadding := int64(float64(nodeAvailMem.Value()) * padPct[i]) - podName := fmt.Sprintf("%s-padding", nodeName) - - // Create pod with nodeSelector - podConf := k8s.TestPodConfig{ - Name: podName, - Namespace: ns, - Labels: map[string]string{ - "node": nodeName, - "app": "app-" + common.RandSeq(5), - "applicationId": "appid-" + podName, - }, - NodeSelector: map[string]string{"kubernetes.io/hostname": nodeName}, - Resources: &v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "cpu": resource.MustParse("0m"), - "memory": *resource.NewQuantity(memPadding, resource.BinarySI), - }, - }, - } - - By(fmt.Sprintf("[%s] Submit pod %s", podConf.Labels["applicationId"], podName)) - initPod, err := k8s.InitTestPod(podConf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - err = kClient.WaitForPodRunning(ns, podName, time.Duration(90)*time.Second) - Ω(err).NotTo(HaveOccurred()) - } - - // Sort nodes by available dominant resource in increasing order - nodesAvailRes = kClient.GetNodesAvailRes(*nodes) - sortedWorkerNodes = k8s.GetWorkerNodes(*nodes) - sort.Slice(sortedWorkerNodes, func(i, j int) bool { - node1, node2 := sortedWorkerNodes[i].Name, sortedWorkerNodes[j].Name - q1, q2 := nodesAvailRes[node1][domRes], nodesAvailRes[node2][domRes] - return q1.Cmp(q2) == -1 - }) - nodeNames = []string{} - for _, node := range sortedWorkerNodes { - nodeNames = append(nodeNames, node.Name) - } - By(fmt.Sprintf("Sorted nodes by available memory after padding: %v", nodeNames)) - - // JobA Specs - jobAPodSpec := k8s.TestPodConfig{ - Labels: map[string]string{ - "app": "sleep-" + common.RandSeq(5), - "applicationId": "appid-joba-" + common.RandSeq(5), - }, - } - jobAConf := k8s.JobConfig{ - Name: "joba-" + common.RandSeq(5), - Namespace: ns, - Parallelism: int32(3), - PodConfig: jobAPodSpec, - } - - // JobB Specs with anti-affinity to top node (highly utilised node). Should be scheduled to 2nd most-utilized node. - jobBPodSpec := k8s.TestPodConfig{ - Labels: map[string]string{ - "app": "sleep-" + common.RandSeq(5), - "applicationId": "appid-jobb-" + common.RandSeq(5), - }, - Affinity: &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "node", - Operator: metav1.LabelSelectorOpIn, - Values: []string{sortedWorkerNodes[0].Name}, - }, - }, - }, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - }, - } - jobBConf := k8s.JobConfig{ - Name: "jobb-" + common.RandSeq(5), - Namespace: ns, - Parallelism: int32(3), - PodConfig: jobBPodSpec, - } - - for i, jobConf := range []k8s.JobConfig{jobAConf, jobBConf} { - // Submit each job - By(fmt.Sprintf("[%s] Create job %s with 3 pods", jobConf.PodConfig.Labels["applicationId"], jobConf.Name)) - job, jobErr := k8s.InitJobConfig(jobConf) - Ω(jobErr).NotTo(HaveOccurred()) - _, jobErr = kClient.CreateJob(job, ns) - Ω(jobErr).NotTo(HaveOccurred()) - createErr := kClient.WaitForJobPodsCreated(ns, jobConf.Name, int(jobConf.Parallelism), 30*time.Second) - Ω(createErr).NotTo(HaveOccurred()) - timeoutErr := kClient.WaitForJobPods(ns, jobConf.Name, int(jobConf.Parallelism), 90*time.Second) - Ω(timeoutErr).NotTo(HaveOccurred()) - - // List job pods and verify running on expected node - jobPods, lstErr := kClient.ListPods(ns, fmt.Sprintf("job-name=%s", jobConf.Name)) - Ω(lstErr).NotTo(HaveOccurred()) - Ω(jobPods).NotTo(BeNil()) - Ω(len(jobPods.Items)).Should(Equal(int(3)), "Pods count should be 3") - for _, pod := range jobPods.Items { - Ω(pod.Spec.NodeName).To(Equal(sortedWorkerNodes[i].Name), - "job pods not scheduled to correct node") - } - } - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - By("Tear down namespace: " + ns) - err := kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/gang_scheduling/gang_scheduling_suite_test.go b/test/e2e/gang_scheduling/gang_scheduling_suite_test.go deleted file mode 100644 index 1a73ac06a..000000000 --- a/test/e2e/gang_scheduling/gang_scheduling_suite_test.go +++ /dev/null @@ -1,91 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package gangscheduling_test - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestGangScheduling(t *testing.T) { - ginkgo.ReportAfterSuite("TestGangScheduling", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-gang_scheduling_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestGangScheduling", ginkgo.Label("TestGangScheduling")) -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) -var kClient = k8s.KubeCtl{} //nolint - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -// Declarations for Ginkgo DSL -var Describe = ginkgo.Describe - -var It = ginkgo.It -var PIt = ginkgo.PIt -var By = ginkgo.By -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach -var DescribeTable = ginkgo.Describe -var Entry = ginkgo.Entry - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var BeNumerically = gomega.BeNumerically -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/gang_scheduling/gang_scheduling_test.go b/test/e2e/gang_scheduling/gang_scheduling_test.go deleted file mode 100644 index 8fd4ececd..000000000 --- a/test/e2e/gang_scheduling/gang_scheduling_test.go +++ /dev/null @@ -1,758 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package gangscheduling_test - -import ( - "fmt" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - batchv1 "k8s.io/api/batch/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - "github.com/apache/yunikorn-k8shim/pkg/cache" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" -) - -const ( - groupA = "groupa" - groupB = "groupb" - groupC = "groupc" -) - -var ( - restClient yunikorn.RClient - ns string - nsQueue string - appID string - minResource map[string]resource.Quantity - jobNames []string - unsatisfiableNodeSelector = map[string]string{"kubernetes.io/hostname": "unsatisfiable_node"} -) - -var _ = Describe("", func() { - BeforeEach(func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - - ns = "ns-" + common.RandSeq(10) - nsQueue = "root." + ns - By(fmt.Sprintf("Creating namespace: %s for sleep jobs", ns)) - namespace, err := kClient.CreateNamespace(ns, nil) - Ω(err).NotTo(HaveOccurred()) - Ω(namespace.Status.Phase).To(Equal(v1.NamespaceActive)) - - minResource = map[string]resource.Quantity{ - v1.ResourceCPU.String(): resource.MustParse("10m"), - v1.ResourceMemory.String(): resource.MustParse("20M"), - } - jobNames = []string{} - appID = "appid-" + common.RandSeq(5) - }) - - // Test to verify annotation with task group definition - // 1. Deploy 1 job with tg definition - // 2. Poll for 5 placeholders - // 3. App running - // 4. Deploy 1 job with 5 real pods of task group - // 5. Check placeholders deleted - // 6. Real pods running and app running - It("Verify_Annotation_TaskGroup_Def", func() { - // Define gang member template with 5 members, 1 real pod (not part of tg) - annotations := k8s.PodAnnotation{ - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(5), MinResource: minResource}, - }, - } - createJob(appID, minResource, annotations, 1) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - // Ensure placeholders are created - appDaoInfo, appDaoInfoErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - checkPlaceholderData(appDaoInfo, groupA, 5, 0, 0) - - // Deploy job, now with 5 pods part of taskGroup - By("Deploy second job with 5 real taskGroup pods") - annotations.TaskGroupName = groupA - realJob := createJob(appID, minResource, annotations, 5) - - // Check all placeholders deleted. - By("Wait for all placeholders terminated") - phTermErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-"+groupA+"-", 0, 3*time.Minute, nil) - Ω(phTermErr).NotTo(gomega.HaveOccurred()) - - // Check real gang members now running - By("Wait for all gang members running") - jobRunErr := kClient.WaitForJobPods(ns, realJob.Name, int(*realJob.Spec.Parallelism), 3*time.Minute) - Ω(jobRunErr).NotTo(HaveOccurred()) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - // Ensure placeholders are replaced - appDaoInfo, appDaoInfoErr = restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - checkPlaceholderData(appDaoInfo, groupA, 5, 5, 0) - }) - - // Test to verify multiple task group nodes - // 1. Deploy 1 job with 3 task group's definitions - // 2. Poll for task group's placeholders - // 3. Store the nodes of placeholders by task group - // 4. Deploy 1 job with real pods - // 5. Nodes distributions of real pods and placeholders should be the same. - It("Verify_Multiple_TaskGroups_Nodes", func() { - annotations := k8s.PodAnnotation{ - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource}, - {Name: groupB, MinMember: int32(5), MinResource: minResource}, - {Name: groupC, MinMember: int32(7), MinResource: minResource}, - }, - } - createJob(appID, minResource, annotations, 1) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - // Wait for placeholders to become running - stateRunning := v1.PodRunning - By("Wait for all placeholders running") - phErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-", 15, 2*time.Minute, &stateRunning) - Ω(phErr).NotTo(HaveOccurred()) - - // Check placeholder node distribution is same as real pods' - phPods, phListErr := kClient.ListPods(ns, "placeholder=true") - Ω(phListErr).NotTo(HaveOccurred()) - taskGroupNodes := map[string]map[string]int{} - for _, ph := range phPods.Items { - tg, ok := ph.Annotations[constants.AnnotationTaskGroupName] - if !ok { - continue - } - if _, ok = taskGroupNodes[tg]; !ok { - taskGroupNodes[tg] = map[string]int{} - } - if _, ok = taskGroupNodes[tg][ph.Spec.NodeName]; !ok { - taskGroupNodes[tg][ph.Spec.NodeName] = 0 - } - taskGroupNodes[tg][ph.Spec.NodeName]++ - } - - // Deploy real pods for each taskGroup - realJobNames := []string{} - for _, tg := range annotations.TaskGroups { - annotations.TaskGroupName = tg.Name - realJob := createJob(appID, minResource, annotations, tg.MinMember) - realJobNames = append(realJobNames, realJob.Name) - } - - By("Wait for all placeholders terminated") - phTermErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-", 0, 3*time.Minute, nil) - Ω(phTermErr).NotTo(HaveOccurred()) - - // Check real gang members now running on same node distribution - By("Verify task group node distribution") - realPodNodes := map[string]map[string]int{} - for i, tg := range annotations.TaskGroups { - jobPods, lstErr := kClient.ListPods(ns, fmt.Sprintf("job-name=%s", realJobNames[i])) - Ω(lstErr).NotTo(HaveOccurred()) - Ω(len(jobPods.Items)).Should(BeNumerically("==", tg.MinMember)) - realPodNodes[tg.Name] = map[string]int{} - for _, pod := range jobPods.Items { - podRunErr := kClient.WaitForPodRunning(ns, pod.Name, time.Minute*5) - Ω(podRunErr).NotTo(HaveOccurred()) - pod, getErr := kClient.GetPod(pod.Name, ns) - Ω(getErr).NotTo(HaveOccurred()) - realPodNodes[tg.Name][pod.Spec.NodeName]++ - } - } - Ω(realPodNodes).Should(Equal(taskGroupNodes)) - - checkAppStatus(appID, yunikorn.States().Application.Running) - }) - - // Test to verify task group with more than min members - // 1. Deploy 1 job with more taskGroup pods than minMembers - // 2. Verify all pods running - It("Verify_TG_with_More_Than_minMembers", func() { - annotations := k8s.PodAnnotation{ - TaskGroupName: groupA, - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource}, - }, - } - createJob(appID, minResource, annotations, 6) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - By("Wait for all placeholders terminated") - phTermErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-", 0, 3*time.Minute, nil) - Ω(phTermErr).NotTo(HaveOccurred()) - - // Ensure placeholders are replaced and allocations count is correct - appDaoInfo, appDaoInfoErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - checkPlaceholderData(appDaoInfo, groupA, 3, 3, 0) - Ω(len(appDaoInfo.Allocations)).To(Equal(int(6)), "Allocations count is not correct") - }) - - // Test to verify soft GS style behaviour - // 1. Submit gang job with 2 gangs - // a) ganga - 3 placeholders - // b) gangb - 1 placeholder, unsatisfiable nodeSelector - // 2. After placeholder timeout, real pods should be scheduled. - It("Verify_Default_GS_Style", func() { - pdTimeout := 20 - annotations := k8s.PodAnnotation{ - SchedulingPolicyParams: fmt.Sprintf("%s=%d", constants.SchedulingPolicyTimeoutParam, pdTimeout), - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource}, - {Name: groupB, MinMember: int32(1), MinResource: minResource, NodeSelector: unsatisfiableNodeSelector}, - }, - } - job := createJob(appID, minResource, annotations, 3) - - // Wait for placeholder timeout - time.Sleep(time.Duration(pdTimeout) * time.Second) - - By(fmt.Sprintf("[%s] Verify pods are scheduled", appID)) - jobRunErr := kClient.WaitForJobPods(ns, job.Name, int(*job.Spec.Parallelism), 2*time.Minute) - Ω(jobRunErr).NotTo(HaveOccurred()) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - // Ensure placeholders are timed out and allocations count is correct as app started running normal because of 'soft' gang style - appDaoInfo, appDaoInfoErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(2), "Placeholder count is not correct") - checkPlaceholderData(appDaoInfo, groupA, 3, 0, 3) - checkPlaceholderData(appDaoInfo, groupB, 1, 0, 1) - Ω(len(appDaoInfo.Allocations)).To(Equal(int(3)), "Allocations count is not correct") - for _, alloc := range appDaoInfo.Allocations { - Ω(alloc.Placeholder).To(Equal(false), "Allocation should be non placeholder") - Ω(alloc.PlaceholderUsed).To(Equal(false), "Allocation should not be replacement of ph") - } - }) - - // Test to verify Hard GS style behaviour - // 1. Deploy 1 job with 2 task group's: - // a) 1 tg with un runnable placeholders - // b) 1 tg with runnable placeholders - // 2. Verify appState = Accepted - // 3. Once ph's are timed out, app should move to failed state - It("Verify_Hard_GS_Failed_State", func() { - pdTimeout := 20 - gsStyle := "Hard" - placeholderTimeoutStr := fmt.Sprintf("%s=%d", constants.SchedulingPolicyTimeoutParam, pdTimeout) - gsStyleStr := fmt.Sprintf("%s=%s", constants.SchedulingPolicyStyleParam, gsStyle) - - annotations := k8s.PodAnnotation{ - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource, NodeSelector: unsatisfiableNodeSelector}, - {Name: groupB, MinMember: int32(3), MinResource: minResource}, - }, - SchedulingPolicyParams: fmt.Sprintf("%s %s", placeholderTimeoutStr, gsStyleStr), - } - createJob(appID, minResource, annotations, 1) - - // Wait for placeholder timeout - time.Sleep(time.Duration(pdTimeout) * time.Second) - - checkCompletedAppStatus(appID, yunikorn.States().Application.Failed) - - // Ensure placeholders are timed out and allocations count is correct as app started running normal because of 'soft' gang style - appDaoInfo, appDaoInfoErr := restClient.GetCompletedAppInfo(configmanager.DefaultPartition, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(2), "Placeholder count is not correct") - checkPlaceholderData(appDaoInfo, groupA, 3, 0, 3) - checkPlaceholderData(appDaoInfo, groupB, 3, 0, 3) - }) - - // Test to verify Gang Apps FIFO order - // Update namespace with quota of 300m and 300M - // 1. Deploy appA with 1 pod - // 2. Deploy appB with gang of 3 pods - // 3. Deploy appC with 1 pod - // 4. Delete appA - // 5. appA = Completing, appB = Running, appC = Accepted - It("Verify_GangApp_FIFO_Order", func() { - By(fmt.Sprintf("Update namespace %s with quota cpu 300m and memory 300M", ns)) - namespace, nsErr := kClient.UpdateNamespace(ns, map[string]string{ - constants.NamespaceQuota: "{\"cpu\": \"300m\", \"memory\": \"300M\"}"}) - Ω(nsErr).NotTo(HaveOccurred()) - Ω(namespace.Status.Phase).To(Equal(v1.NamespaceActive)) - - appIDA := "app-a-" + common.RandSeq(5) - appIDB := "app-b-" + common.RandSeq(5) - appIDC := "app-c-" + common.RandSeq(5) - - minResource[v1.ResourceCPU.String()] = resource.MustParse("100m") - minResource[v1.ResourceMemory.String()] = resource.MustParse("100M") - - annotationsA := k8s.PodAnnotation{ - TaskGroupName: groupA, - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(0), MinResource: minResource}, - }, - } - annotationsB := k8s.PodAnnotation{ - TaskGroupName: groupB, - TaskGroups: []cache.TaskGroup{ - {Name: groupB, MinMember: int32(3), MinResource: minResource}, - }, - } - annotationsC := k8s.PodAnnotation{ - TaskGroupName: groupC, - TaskGroups: []cache.TaskGroup{ - {Name: groupC, MinMember: int32(0), MinResource: minResource}, - }, - } - jobA := createJob(appIDA, minResource, annotationsA, 1) - time.Sleep(1 * time.Second) // To ensure there is minor gap between applications - createJob(appIDB, minResource, annotationsB, 3) - time.Sleep(1 * time.Second) // To ensure there is minor gap between applications - createJob(appIDC, minResource, annotationsC, 1) - - // AppB should have 2/3 placeholders running - By("Wait for 2 placeholders running in app " + appIDB) - statusRunning := v1.PodRunning - phErr := kClient.WaitForPlaceholders(ns, "tg-"+appIDB+"-", 2, 30*time.Second, &statusRunning) - Ω(phErr).NotTo(HaveOccurred()) - - // Delete appA - deleteErr := kClient.DeleteJob(jobA.Name, ns) - Ω(deleteErr).NotTo(HaveOccurred()) - jobNames = jobNames[1:] // remove jobA, so we don't delete again in AfterEach - - // Now, appA=Completing, appB=Running, appC=Accepted - checkAppStatus(appIDA, yunikorn.States().Application.Completing) - checkAppStatus(appIDB, yunikorn.States().Application.Running) - checkAppStatus(appIDC, yunikorn.States().Application.Accepted) - - appDaoInfo, appDaoInfoErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appIDA) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.Allocations)).To(Equal(0), "Allocations count is not correct") - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(0), "Placeholder count is not correct") - - appDaoInfo, appDaoInfoErr = restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appIDB) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.Allocations)).To(Equal(3), "Allocations count is not correct") - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(1), "Placeholder count is not correct") - Ω(int(appDaoInfo.PlaceholderData[0].Count)).To(Equal(int(3)), "Placeholder count is not correct") - - appDaoInfo, appDaoInfoErr = restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appIDC) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.Allocations)).To(Equal(0), "Allocations count is not correct") - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(0), "Placeholder count is not correct") - }) - - // Test validates that lost placeholders resources are decremented by Yunikorn. - // 1. Submit gang job with 2 gangs - // a) ganga - 3 placeholders, nodeSelector=nodeA - // b) gangb - 1 placeholder, unsatisfiable nodeSelector - // c) 3 real ganga pods - // 2. Delete all gangA placeholders after all are running - // 3. Verify no pods from gang-app in nodeA allocations. - // Verify no pods in app allocations - // Verify queue used capacity = 0 - // Verify app is failed after app timeout met - It("Verify_Deleted_Placeholders", func() { - nodes, err := kClient.GetNodes() - Ω(err).NotTo(HaveOccurred()) - workerNodes := k8s.GetWorkerNodes(*nodes) - Ω(len(workerNodes)).NotTo(Equal(0)) - - pdTimeout := 60 - annotations := k8s.PodAnnotation{ - SchedulingPolicyParams: fmt.Sprintf("%s=%d", constants.SchedulingPolicyTimeoutParam, pdTimeout), - TaskGroups: []cache.TaskGroup{ - { - Name: groupA, - MinMember: int32(3), - MinResource: minResource, - NodeSelector: map[string]string{"kubernetes.io/hostname": workerNodes[0].Name}, - }, - { - Name: groupB, - MinMember: int32(1), - MinResource: minResource, - NodeSelector: unsatisfiableNodeSelector, - }, - }, - } - createJob(appID, minResource, annotations, 3) - - checkAppStatus(appID, yunikorn.States().Application.Accepted) - - // Wait for groupa placeholder pods running - By(fmt.Sprintf("Wait for %s placeholders running", groupA)) - stateRunning := v1.PodRunning - runErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-"+groupA+"-", 3, 30*time.Second, &stateRunning) - Ω(runErr).NotTo(HaveOccurred()) - - // Delete all groupa placeholder pods - phPods, listErr := kClient.ListPlaceholders(ns, "tg-"+appID+"-"+groupA+"-") - Ω(listErr).NotTo(HaveOccurred()) - for _, ph := range phPods { - By(fmt.Sprintf("Delete placeholder %s", ph.Name)) - deleteErr := kClient.DeletePod(ph.Name, ns) - Ω(deleteErr).NotTo(HaveOccurred()) - } - - // Wait for Yunikorn allocation removal after K8s deletion - time.Sleep(5 * time.Second) - - // Verify app allocations correctly decremented - appInfo, appErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appErr).NotTo(HaveOccurred()) - Ω(len(appInfo.Allocations)).To(Equal(0), "Placeholder allocation not removed from app") - - // Verify no app allocation in nodeA - ykNodes, nodeErr := restClient.GetNodes(configmanager.DefaultPartition) - Ω(nodeErr).NotTo(HaveOccurred()) - for _, node := range *ykNodes { - for _, alloc := range node.Allocations { - Ω(alloc.ApplicationID).NotTo(Equal(appID), "Placeholder allocation not removed from node") - } - } - - // Verify queue resources = 0 - qInfo, qErr := restClient.GetQueue(configmanager.DefaultPartition, nsQueue) - Ω(qErr).NotTo(HaveOccurred()) - var usedResource yunikorn.ResourceUsage - var usedPercentageResource yunikorn.ResourceUsage - usedResource.ParseResourceUsage(qInfo.AllocatedResource) - Ω(usedResource.GetResourceValue(siCommon.CPU)).Should(Equal(int64(0)), "Placeholder allocation not removed from queue") - Ω(usedResource.GetResourceValue(siCommon.Memory)).Should(Equal(int64(0)), "Placeholder allocation not removed from queue") - usedPercentageResource.ParseResourceUsage(qInfo.AbsUsedCapacity) - Ω(usedPercentageResource.GetResourceValue(siCommon.CPU)).Should(Equal(int64(0)), "Placeholder allocation not removed from queue") - Ω(usedPercentageResource.GetResourceValue(siCommon.Memory)).Should(Equal(int64(0)), "Placeholder allocation not removed from queue") - }) - - // Test to verify completed placeholders cleanup - // 1. Deploy 1 job with 2 task group's: - // a) 1 tg with un runnable placeholders - // b) 1 tg with runnable placeholders - // 2. Delete job - // 3. Verify app is completing - // 4. Verify placeholders deleted - // 5. Verify app allocation is empty - It("Verify_Completed_Job_Placeholders_Cleanup", func() { - annotations := k8s.PodAnnotation{ - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource, NodeSelector: unsatisfiableNodeSelector}, - {Name: groupB, MinMember: int32(3), MinResource: minResource}, - }, - } - job := createJob(appID, minResource, annotations, 1) - - checkAppStatus(appID, yunikorn.States().Application.Accepted) - - By("Wait for groupB placeholders running") - stateRunning := v1.PodRunning - runErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-"+groupB+"-", 3, 30*time.Second, &stateRunning) - Ω(runErr).NotTo(HaveOccurred()) - - By("List placeholders") - tgPods, listErr := kClient.ListPlaceholders(ns, "tg-"+appID+"-") - Ω(listErr).NotTo(HaveOccurred()) - - By("Delete job pods") - deleteErr := kClient.DeleteJob(job.Name, ns) - Ω(deleteErr).NotTo(HaveOccurred()) - jobNames = []string{} // remove job names to prevent delete job again in AfterEach - - checkAppStatus(appID, yunikorn.States().Application.Completing) - - By("Verify placeholders deleted") - for _, ph := range tgPods { - deleteErr = kClient.WaitForPodTerminated(ns, ph.Name, 30*time.Second) - Ω(deleteErr).NotTo(HaveOccurred(), "Placeholder %s still running", ph) - } - - By("Verify app allocation is empty") - appInfo, restErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(restErr).NotTo(HaveOccurred()) - Ω(len(appInfo.Allocations)).To(Equal(0)) - }) - - // Test to verify originator deletion will trigger placeholders cleanup - // 1. Create an originator pod - // 2. Not set pod ownerreference - // 3. Delete originator pod to trigger placeholders deletion - // 4. Verify placeholders deleted - // 5. Verify app allocation is empty - It("Verify_OriginatorDeletion_Trigger_Placeholders_Cleanup", func() { - // case 1: originator pod without ownerreference - verifyOriginatorDeletionCase(false) - }) - - // Test to verify originator deletion will trigger placeholders cleanup - // 1. Create an originator pod - // 2. Set pod ownerreference with ownerreference, take configmap for example - // 3. Delete originator pod to trigger placeholders deletion - // 4. Verify placeholders deleted - // 5. Verify app allocation is empty - It("Verify_OriginatorDeletionWithOwnerreference_Trigger_Placeholders_Cleanup", func() { - // case 2: originator pod with ownerreference - verifyOriginatorDeletionCase(true) - }) - - // Test placeholder with hugepages - // 1. Deploy 1 job with hugepages-2Mi - // 2. Verify all pods running - It("Verify_HugePage", func() { - hugepageKey := fmt.Sprintf("%s2Mi", v1.ResourceHugePagesPrefix) - nodes, err := kClient.GetNodes() - Ω(err).NotTo(HaveOccurred()) - hasHugePages := false - for _, node := range nodes.Items { - if v, ok := node.Status.Capacity[v1.ResourceName(hugepageKey)]; ok { - if v.Value() != 0 { - hasHugePages = true - break - } - } - } - if !hasHugePages { - ginkgo.Skip("Skip hugepages test as no node has hugepages") - } - - // add hugepages to request - minResource[hugepageKey] = resource.MustParse("100Mi") - annotations := k8s.PodAnnotation{ - TaskGroupName: groupA, - TaskGroups: []cache.TaskGroup{ - {Name: groupA, MinMember: int32(3), MinResource: minResource}, - }, - } - job := createJob(appID, minResource, annotations, 3) - - By("Verify all job pods are running") - jobRunErr := kClient.WaitForJobPods(ns, job.Name, int(*job.Spec.Parallelism), 2*time.Minute) - Ω(jobRunErr).NotTo(HaveOccurred()) - - checkAppStatus(appID, yunikorn.States().Application.Running) - - // Ensure placeholders are replaced and allocations count is correct - appDaoInfo, appDaoInfoErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(appDaoInfoErr).NotTo(HaveOccurred()) - Ω(len(appDaoInfo.PlaceholderData)).To(Equal(1), "Placeholder count is not correct") - checkPlaceholderData(appDaoInfo, groupA, 3, 3, 0) - Ω(len(appDaoInfo.Allocations)).To(Equal(int(3)), "Allocations count is not correct") - Ω(appDaoInfo.UsedResource[hugepageKey]).To(Equal(int64(314572800)), "Used huge page resource is not correct") - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By(fmt.Sprintf("Cleanup jobs: %v", jobNames)) - for _, jobName := range jobNames { - err := kClient.DeleteJob(jobName, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - } - - By("Tear down namespace: " + ns) - err := kClient.TearDownNamespace(ns) - - Ω(err).NotTo(HaveOccurred()) - }) - -}) - -func createJob(applicationID string, minResource map[string]resource.Quantity, annotations k8s.PodAnnotation, parallelism int32) (job *batchv1.Job) { - var ( - err error - requests = v1.ResourceList{} - limits = v1.ResourceList{} - ) - for k, v := range minResource { - key := v1.ResourceName(k) - requests[key] = v - if strings.HasPrefix(k, v1.ResourceHugePagesPrefix) { - limits[key] = v - } - } - - podConf := k8s.TestPodConfig{ - Labels: map[string]string{ - "app": "sleep-" + common.RandSeq(5), - "applicationId": applicationID, - }, - Annotations: &annotations, - Resources: &v1.ResourceRequirements{ - Requests: requests, - Limits: limits, - }, - } - jobConf := k8s.JobConfig{ - Name: fmt.Sprintf("gangjob-%s-%s", applicationID, common.RandSeq(5)), - Namespace: ns, - Parallelism: parallelism, - PodConfig: podConf, - } - - job, err = k8s.InitJobConfig(jobConf) - Ω(err).NotTo(HaveOccurred()) - - taskGroupsMap, err := k8s.PodAnnotationToMap(podConf.Annotations) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("[%s] Deploy job %s with task-groups: %+v", applicationID, jobConf.Name, taskGroupsMap[k8s.TaskGroups])) - job, err = kClient.CreateJob(job, ns) - Ω(err).NotTo(HaveOccurred()) - - err = kClient.WaitForJobPodsCreated(ns, job.Name, int(*job.Spec.Parallelism), 30*time.Second) - Ω(err).NotTo(HaveOccurred()) - - jobNames = append(jobNames, job.Name) // for cleanup in afterEach function - return job -} - -func checkAppStatus(applicationID, state string) { - By(fmt.Sprintf("Verify application %s status is %s", applicationID, state)) - timeoutErr := restClient.WaitForAppStateTransition(configmanager.DefaultPartition, nsQueue, applicationID, state, 120) - Ω(timeoutErr).NotTo(HaveOccurred()) -} - -func checkCompletedAppStatus(applicationID, state string) { - By(fmt.Sprintf("Verify application %s status is %s", applicationID, state)) - timeoutErr := restClient.WaitForCompletedAppStateTransition(configmanager.DefaultPartition, applicationID, state, 120) - Ω(timeoutErr).NotTo(HaveOccurred()) -} - -func checkPlaceholderData(appDaoInfo *dao.ApplicationDAOInfo, tgName string, count, replaced, timeout int) { - verified := false - for _, placeholderData := range appDaoInfo.PlaceholderData { - if tgName == placeholderData.TaskGroupName { - Ω(int(placeholderData.Count)).To(Equal(count), "Placeholder count is not correct") - Ω(int(placeholderData.Replaced)).To(Equal(replaced), "Placeholder replaced is not correct") - Ω(int(placeholderData.TimedOut)).To(Equal(timeout), "Placeholder timeout is not correct") - verified = true - break - } - } - Ω(verified).To(Equal(true), fmt.Sprintf("Can't find task group %s in app info", tgName)) -} - -func verifyOriginatorDeletionCase(withOwnerRef bool) { - podConf := k8s.TestPodConfig{ - Name: "gang-driver-pod" + common.RandSeq(5), - Labels: map[string]string{ - "app": "sleep-" + common.RandSeq(5), - "applicationId": appID, - }, - Annotations: &k8s.PodAnnotation{ - TaskGroups: []cache.TaskGroup{ - { - Name: groupA, - MinMember: int32(3), - MinResource: minResource, - NodeSelector: unsatisfiableNodeSelector, - }, - { - Name: groupB, - MinMember: int32(3), - MinResource: minResource, - }, - }, - }, - Resources: &v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "cpu": minResource["cpu"], - "memory": minResource["memory"], - }, - }, - } - - podTest, err := k8s.InitTestPod(podConf) - Ω(err).NotTo(HaveOccurred()) - - if withOwnerRef { - // create a configmap as ownerreference - testConfigmap := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cm", - UID: "test-cm-uid", - }, - Data: map[string]string{ - "test": "test", - }, - } - defer func() { - err := kClient.DeleteConfigMap(testConfigmap.Name, ns) - Ω(err).NotTo(HaveOccurred()) - }() - - testConfigmap, err := kClient.CreateConfigMap(testConfigmap, ns) - Ω(err).NotTo(HaveOccurred()) - - podTest.OwnerReferences = []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "ConfigMap", - Name: testConfigmap.Name, - UID: testConfigmap.UID, - }, - } - } - - taskGroupsMap, annErr := k8s.PodAnnotationToMap(podConf.Annotations) - Ω(annErr).NotTo(HaveOccurred()) - By(fmt.Sprintf("Deploy pod %s with task-groups: %+v", podTest.Name, taskGroupsMap[k8s.TaskGroups])) - originator, err := kClient.CreatePod(podTest, ns) - Ω(err).NotTo(HaveOccurred()) - - checkAppStatus(appID, yunikorn.States().Application.Accepted) - - By("Wait for groupB placeholders running") - stateRunning := v1.PodRunning - runErr := kClient.WaitForPlaceholders(ns, "tg-"+appID+"-"+groupB+"-", 3, 30*time.Second, &stateRunning) - Ω(runErr).NotTo(HaveOccurred()) - - By("List placeholders") - tgPods, listErr := kClient.ListPlaceholders(ns, "tg-"+appID+"-") - Ω(listErr).NotTo(HaveOccurred()) - - By("Delete originator pod") - deleteErr := kClient.DeletePod(originator.Name, ns) - Ω(deleteErr).NotTo(HaveOccurred()) - - By("Verify placeholders deleted") - for _, ph := range tgPods { - deleteErr = kClient.WaitForPodTerminated(ns, ph.Name, 30*time.Second) - Ω(deleteErr).NotTo(HaveOccurred(), "Placeholder %s still running", ph) - } - - By("Verify app allocation is empty") - appInfo, restErr := restClient.GetAppInfo(configmanager.DefaultPartition, nsQueue, appID) - Ω(restErr).NotTo(HaveOccurred()) - Ω(len(appInfo.Allocations)).To(BeNumerically("==", 0)) -} diff --git a/test/e2e/node_resources/node_resources_suite_test.go b/test/e2e/node_resources/node_resources_suite_test.go deleted file mode 100644 index de1b95ef8..000000000 --- a/test/e2e/node_resources/node_resources_suite_test.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package node_resources_test - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestNodeResources(t *testing.T) { - ginkgo.ReportAfterSuite("TestNodeResources", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-node_resources_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestNodeResources", ginkgo.Label("TestNodeResources")) -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -// Declarations for Ginkgo DSL -var Describe = ginkgo.Describe - -var It = ginkgo.It -var By = ginkgo.By -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/node_resources/node_resources_test.go b/test/e2e/node_resources/node_resources_test.go deleted file mode 100644 index 61c7689f4..000000000 --- a/test/e2e/node_resources/node_resources_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package node_resources_test - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" -) - -var _ = Describe("", func() { - var kClient k8s.KubeCtl //nolint - var restClient yunikorn.RClient - var ns string - - BeforeEach(func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - ns = "ns-" + common.RandSeq(10) - By(fmt.Sprintf("Creating namespace: %s", ns)) - var ns1, err1 = kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By("Tear down namespace: " + ns) - err := kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) - - // To verify the node capacity - // 1. Map node capacity to node name from kClient - // 2. Pull Yunikorn node capacities through rest - // 3. Validate values are equal - It("Verify_Node_Capacity", func() { - // Get kClient node capacities - kClientNodes, getErr := kClient.GetNodes() - Ω(getErr).NotTo(HaveOccurred()) - kClientNodeCapacities := kClient.GetNodesCapacity(*kClientNodes) - - // Get Yunikorn node capacities via rest. - ykNodes, restErr := restClient.GetNodes("default") - Ω(restErr).NotTo(HaveOccurred()) - - // For each yk node capacity, compare to official Kubernetes value - for _, ykNode := range *ykNodes { - nodeName := ykNode.NodeID - var ykCapacity yunikorn.ResourceUsage - ykCapacity.ParseResourceUsage(ykNode.Capacity) - - kubeNodeCPU := kClientNodeCapacities[nodeName]["cpu"] - kubeNodeMem := kClientNodeCapacities[nodeName][siCommon.Memory] - kubeNodeMem.RoundUp(resource.Mega) // round to nearest megabyte for comparison - - // Compare memory to nearest megabyte - roundedYKMem := ykCapacity.GetMemory() - roundedYKMem.RoundUp(resource.Mega) - cmpRes := kubeNodeMem.Cmp(roundedYKMem) - Ω(cmpRes).To(Equal(0)) - - // Compare cpu cores to nearest millicore - cmpRes = kubeNodeCPU.Cmp(ykCapacity.GetCPU()) - Ω(cmpRes).To(Equal(0)) - } - }) -}) diff --git a/test/e2e/persistent_volume/persistent_volume_suite_test.go b/test/e2e/persistent_volume/persistent_volume_suite_test.go deleted file mode 100644 index e20f973c7..000000000 --- a/test/e2e/persistent_volume/persistent_volume_suite_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package persistent_volume - -import ( - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestPersistentVolume(t *testing.T) { - ginkgo.ReportAfterSuite("TestPersistentVolume", func(report ginkgo.Report) { - err := reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-persistent_volume_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestPersistentVolume", ginkgo.Label("TestPersistentVolume")) -} - -var Ω = gomega.Ω -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/persistent_volume/persistent_volume_test.go b/test/e2e/persistent_volume/persistent_volume_test.go deleted file mode 100644 index f131dc89a..000000000 --- a/test/e2e/persistent_volume/persistent_volume_test.go +++ /dev/null @@ -1,352 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package persistent_volume - -import ( - "runtime" - "time" - - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/resource" - - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/onsi/ginkgo/v2" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var suiteName string -var kClient k8s.KubeCtl -var restClient yunikorn.RClient -var dev = "dev-" + common.RandSeq(5) - -const ( - LocalTypePv = "Local" - StandardScName = "standard" -) - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - - // Initializing rest client - restClient = yunikorn.RClient{} - Ω(restClient).NotTo(gomega.BeNil()) - - yunikorn.EnsureYuniKornConfigsPresent() - - // Create namespace - ginkgo.By("Create namespace " + dev) - ns, err := kClient.CreateNamespace(dev, nil) - Ω(err).NotTo(HaveOccurred()) - Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) -}) - -var _ = ginkgo.AfterSuite(func() { - // Clean up - ginkgo.By("Deleting PVCs and PVs") - err := kClient.DeletePVCs(dev) - err2 := kClient.DeletePVs(dev) - ginkgo.By("Tearing down namespace: " + dev) - err3 := kClient.TearDownNamespace(dev) - - Ω(err).NotTo(HaveOccurred()) - Ω(err2).NotTo(HaveOccurred()) - Ω(err3).NotTo(HaveOccurred()) -}) - -var _ = ginkgo.Describe("PersistentVolume", func() { - ginkgo.It("Verify_static_binding_of_local_pv", func() { - pvName := "local-pv-" + common.RandSeq(5) - conf := k8s.PvConfig{ - Name: pvName, - Capacity: "1Gi", - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - Type: LocalTypePv, - Path: "/tmp", - StorageClass: StandardScName, - } - - ginkgo.By("Create local type pv " + pvName) - pvObj, err := k8s.InitPersistentVolume(conf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePersistentVolume(pvObj) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPersistentVolumeAvailable(pvName, 60*time.Second)).NotTo(HaveOccurred()) - - pvcName := "pvc-" + common.RandSeq(5) - pvcConf := k8s.PvcConfig{ - Name: pvcName, - Capacity: "1Gi", - VolumeName: pvName, - } - - ginkgo.By("Create pvc " + pvcName + ", which binds to " + pvName) - pvcObj, err := k8s.InitPersistentVolumeClaim(pvcConf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePersistentVolumeClaim(pvcObj, dev) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPersistentVolumeClaimPresent(dev, pvcName, 60*time.Second)).NotTo(HaveOccurred()) - - podName := "pod-" + common.RandSeq(5) - podConf := k8s.TestPodConfig{ - Name: podName, - Namespace: dev, - PvcName: pvcName, - } - - ginkgo.By("Create pod " + podName + ", which uses pvc " + pvcName) - podObj, err := k8s.InitTestPod(podConf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(podObj, dev) - Ω(err).NotTo(HaveOccurred()) - - ginkgo.By("Check pod " + podName + " is successfully running") - err = kClient.WaitForPodRunning(dev, podName, 60*time.Second) - Ω(err).NotTo(HaveOccurred()) - }) - - ginkgo.It("Verify_dynamic_bindng_with_nfs_server", func() { - ginkgo.By("Start creating nfs provisioner.") - - // Create nfs server and related rbac - saName := "nfs-service-account" - crName := "nfs-cluster-role" - crbName := "nfs-cluster-role-binding" //nolint:gosec - serverName := "nfs-provisioner" - scName := "nfs-sc" - createNfsRbac(saName, crName, crbName) - createNfsProvisioner(saName, serverName, scName) - - // Create pvc using storageclass - pvcName := "pvc-" + common.RandSeq(5) - pvcConf := k8s.PvcConfig{ - Name: pvcName, - Capacity: "1Gi", - StorageClassName: scName, - } - - ginkgo.By("Create pvc " + pvcName + ", which uses storage class " + scName) - pvcObj, err := k8s.InitPersistentVolumeClaim(pvcConf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePersistentVolumeClaim(pvcObj, dev) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPersistentVolumeClaimPresent(dev, pvcName, 60*time.Second)).NotTo(HaveOccurred()) - - // Create pod - podName := "pod-" + common.RandSeq(5) - podConf := k8s.TestPodConfig{ - Name: podName, - Namespace: dev, - PvcName: pvcName, - } - - ginkgo.By("Create pod " + podName + " with pvc " + pvcName) - podObj, err := k8s.InitTestPod(podConf) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(podObj, dev) - Ω(err).NotTo(HaveOccurred()) - - ginkgo.By("Check pod " + podName + " is successfully running") - err = kClient.WaitForPodRunning(dev, podName, 60*time.Second) - Ω(err).NotTo(HaveOccurred()) - - deleteNfsRelatedRoles(saName, crName, crbName) - deleteNfsProvisioner(serverName, scName) - }) -}) - -func createNfsRbac(svaName string, crName string, crbName string) { - // Create service account, cluster role and role binding - ginkgo.By("Create service account " + svaName) - _, err := kClient.CreateServiceAccount(svaName, dev) - Ω(err).NotTo(HaveOccurred()) - - nfsClusterRole := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: crName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"*"}, - Resources: []string{ - "nodes", "nodes/proxy", - "namespaces", "services", "pods", "pods/exec", - "deployments", "deployments/finalizers", - "replicationcontrollers", "replicasets", - "statefulsets", "daemonsets", - "events", "endpoints", "configmaps", "secrets", "jobs", "cronjobs", - "storageclasses", "persistentvolumeclaims", "persistentvolumes", - }, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"openebs.io"}, - Resources: []string{"*"}, - Verbs: []string{"*"}, - }, - }, - } - ginkgo.By("Create cluster role " + crName) - _, err = kClient.CreateClusterRole(nfsClusterRole) - Ω(err).NotTo(HaveOccurred()) - - ginkgo.By("Create cluster role binding " + crbName) - _, err = kClient.CreateClusterRoleBinding(crbName, crName, dev, svaName) - Ω(err).NotTo(HaveOccurred()) -} - -func createNfsProvisioner(svaName string, serverName string, scName string) { - // Create nfs provisioner - nfsProvisioner := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: serverName, - Namespace: dev, - Labels: map[string]string{ - "name": serverName, - }, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": serverName, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": serverName, - }, - }, - Spec: v1.PodSpec{ - ServiceAccountName: svaName, - Containers: []v1.Container{ - { - Name: "nfs-provisioner", - Image: "openebs/provisioner-nfs:0.10.0", - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "OPENEBS_NAMESPACE", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "OPENEBS_SERVICE_ACCOUNT", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.serviceAccountName", - }, - }, - }, - { - Name: "OPENEBS_IO_ENABLE_ANALYTICS", - Value: "true", - }, - { - Name: "OPENEBS_IO_NFS_SERVER_USE_CLUSTERIP", - Value: "true", - }, - { - Name: "OPENEBS_IO_INSTALLER_TYPE", - Value: "openebs-operator-nfs", - }, - { - Name: "OPENEBS_IO_NFS_SERVER_IMG", - Value: "openebs/nfs-server-alpine:0.10.0", - }, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "cpu": resource.MustParse("50m"), - "memory": resource.MustParse("50M"), - }, - Limits: v1.ResourceList{ - "cpu": resource.MustParse("200m"), - "memory": resource.MustParse("200M"), - }, - }, - }, - }, - }, - }, - }, - } - - ginkgo.By("Create nfs provisioner " + serverName) - _, err := kClient.CreateDeployment(nfsProvisioner, dev) - Ω(err).NotTo(HaveOccurred()) - - // Create storage class - sc := &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: scName, - Annotations: map[string]string{ - "openebs.io/cas-type": "nfsrwx", - "cas.openebs.io/config": "- name: NFSServerType\n value: \"kernel\"\n- name: BackendStorageClass\n value: \"standard\"\n", - }, - }, - Provisioner: "openebs.io/nfsrwx", - } - - ginkgo.By("Create storage class " + scName) - _, err = kClient.CreateStorageClass(sc) - Ω(err).NotTo(HaveOccurred()) -} - -func deleteNfsRelatedRoles(serviceAccount string, clusterRole string, clusterRoleBinding string) { - ginkgo.By("Deleting NFS related roles and bindings") - err := kClient.DeleteClusterRoleBindings(clusterRoleBinding) - err2 := kClient.DeleteClusterRole(clusterRole) - err3 := kClient.DeleteServiceAccount(serviceAccount, dev) - - Ω(err).NotTo(HaveOccurred()) - Ω(err2).NotTo(HaveOccurred()) - Ω(err3).NotTo(HaveOccurred()) -} - -func deleteNfsProvisioner(deployName string, scName string) { - ginkgo.By("Deleting NFS deployment and storage class") - err := kClient.DeleteDeployment(deployName, dev) - err2 := kClient.DeleteStorageClass(scName) - - Ω(err).NotTo(HaveOccurred()) - Ω(err2).NotTo(HaveOccurred()) -} diff --git a/test/e2e/predicates/predicates_suite_test.go b/test/e2e/predicates/predicates_suite_test.go deleted file mode 100644 index 36d543f06..000000000 --- a/test/e2e/predicates/predicates_suite_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package predicates_test - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation string - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -func TestPredicates(t *testing.T) { - ginkgo.ReportAfterSuite("TestPredicates", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-predicates_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestPredicates", ginkgo.Label("TestPredicates")) -} - -// type Benchmarker ginkgo.Benchmarker -var Fail = ginkgo.Fail - -var Describe = ginkgo.Describe -var DescribeTable = ginkgo.DescribeTable -var Entry = ginkgo.Entry -var Context = ginkgo.Context -var It = ginkgo.It -var By = ginkgo.By -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach - -// Declarations for Gomega DSL -var RegisterFailHandler = gomega.RegisterFailHandler -var Ω = gomega.Ω - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var BeNil = gomega.BeNil -var BeTrue = gomega.BeTrue -var HaveOccurred = gomega.HaveOccurred -var MatchRegexp = gomega.MatchRegexp -var BeZero = gomega.BeZero -var BeEquivalentTo = gomega.BeEquivalentTo -var ContainElement = gomega.ContainElement -var HaveKeyWithValue = gomega.HaveKeyWithValue diff --git a/test/e2e/predicates/predicates_test.go b/test/e2e/predicates/predicates_test.go deleted file mode 100644 index 6ffb1b003..000000000 --- a/test/e2e/predicates/predicates_test.go +++ /dev/null @@ -1,1099 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package predicates_test - -import ( - "fmt" - "reflect" - "time" - - "github.com/onsi/ginkgo/v2" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -// variable populated in BeforeEach, never modified afterwards -var workerNodes []string - -func getNodeThatCanRunPodWithoutToleration(k *k8s.KubeCtl, namespace string) string { - By("Trying to launch a pod without a toleration to get a node which can launch it.") - return runPodAndGetNodeName(k, k8s.SleepPodConfig{Name: "without-toleration", NS: namespace}) -} - -// GetNodeThatCanRunPod trying to launch a pod without a label to get a node which can launch it -func GetNodeThatCanRunPod(k *k8s.KubeCtl, namespace string) string { - By("Trying to launch a pod without a label to get a node which can launch it.") - return runPodAndGetNodeName(k, k8s.SleepPodConfig{Name: "without-toleration", NS: namespace}) -} - -func runPodAndGetNodeName(k *k8s.KubeCtl, conf k8s.SleepPodConfig) string { - // launch a pod to find a node which can launch a pod. We intentionally do - // not just take the node list and choose the first of them. Depending on the - // cluster and the scheduler it might be that a "normal" pod cannot be - // scheduled onto it. - pod := runTestPod(k, conf) - - By("Explicitly delete pod here to free the resource it takes.") - err := k.DeletePod(pod.Name, pod.Namespace) - Ω(err).NotTo(HaveOccurred()) - return pod.Spec.NodeName -} - -func runTestPod(k *k8s.KubeCtl, conf k8s.SleepPodConfig) *v1.Pod { - initPod, podErr := k8s.InitSleepPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - pod, err := k.CreatePod(initPod, conf.NS) - Ω(err).NotTo(HaveOccurred()) - Ω(k.WaitForPodRunning(pod.Namespace, pod.Name, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - pod1, err := k.GetPod(pod.Name, pod.Namespace) - Ω(err).NotTo(HaveOccurred()) - return pod1 -} - -var _ = Describe("Predicates", func() { - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var ns, anotherNS string - const LABELVALUE = "testing-label-value" - - BeforeEach(func() { - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - - nodes, nodesErr := kClient.GetNodes() - Ω(nodesErr).NotTo(HaveOccurred()) - - nodeList := k8s.GetWorkerNodes(*nodes) - Ω(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error occurred: %v", err)) - for _, n := range nodeList { - workerNodes = append(workerNodes, n.Name) - } - ns = "test-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace", ns)) - ns1, err1 := kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - - anotherNS = "test-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace", anotherNS)) - ns2, err2 := kClient.CreateNamespace(anotherNS, nil) - Ω(err2).NotTo(HaveOccurred()) - Ω(ns2.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns, anotherNS}) - - By("Cleanup") - for _, n := range []string{ns, anotherNS} { - ginkgo.By("Tear down namespace: " + n) - err = kClient.TearDownNamespace(n) - Ω(err).NotTo(HaveOccurred()) - } - }) - - // Test Nodes does not have any label, hence it should be impossible to schedule Pod with - // nonempty Selector set. - /* - Testname: Yunikorn Scheduler, node selector not matching - Description: - 1. Create a Pod with a NodeSelector set to a value that does not match a node in the cluster. - 2. Since there are no nodes matching the criteria the Pod MUST not be scheduled. - */ - It("Verify_Non_Matching_NodeSelector_Respected", func() { - By("Trying to schedule Pod with nonempty NodeSelector.") - podName := "blocked-pod" - - conf := k8s.TestPodConfig{ - Name: podName, - Labels: map[string]string{ - "name": "blocked", - "app": "blocked-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{ - "label": "nonempty", - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, podErr = kClient.CreatePod(initPod, ns) - Ω(podErr).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Verify pod:%s is in pending state", podName)) - podErr = kClient.WaitForPodPending(ns, podName, time.Duration(60)*time.Second) - Ω(podErr).NotTo(HaveOccurred()) - - By("Verify the YuniKorn request failed scheduling") - - podErr = restClient.WaitForAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podName, 60) - Ω(podErr).NotTo(HaveOccurred()) - log, podErr := restClient.GetAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podName) - Ω(podErr).NotTo(HaveOccurred()) - Ω(log).NotTo(BeNil(), "Log can't be empty") - logEntries := yunikorn.AllocLogToStrings(log) - Ω(logEntries).To(ContainElement(MatchRegexp(".*didn't match Pod's node affinity")), "Log entry message mismatch") - }) - - /* - Testname: Yunikorn Scheduler, node selector matching - Description: - 1. Create a label on the node {key: value}. - 2. Then create a Pod with a NodeSelector set to {key: value}. - 3. Check to see if the Pod is scheduled on that node by YK scheduler. - */ - It("Verify_Matching_NodeSelector_Respected", func() { - nodeName := GetNodeThatCanRunPod(&kClient, ns) - By("Trying to apply a random label on the found node") - key := fmt.Sprintf("kubernetes.io/e2e-%s", common.RandSeq(10)) - value := "101" - labelErr := kClient.SetNodeLabel(nodeName, key, value) - Ω(labelErr).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, key) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - }() - - By("Trying to launch the pod, now with labels.") - labelPodName := "with-labels" - conf := k8s.TestPodConfig{ - Name: labelPodName, - Namespace: ns, - Labels: map[string]string{ - "app": "blocked-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{ - key: value, - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, labelPodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPod, err1 := kClient.GetPod(labelPodName, ns) - Ω(err1).NotTo(HaveOccurred()) - Ω(labelPod.Spec.NodeName).Should(Equal(nodeName)) - }) - - // Tests for Node Affinity - It("Verify_Non_Matching_NodeAffinity_Respected", func() { - By("Trying to schedule Pod with nonempty NodeAffinity.") - podName := "blocked-pod" - - conf := k8s.TestPodConfig{ - Name: podName, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "foo", - Operator: v1.NodeSelectorOpIn, - Values: []string{"bar", "value2"}, - }, - }, - }, { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "diffkey", - Operator: v1.NodeSelectorOpIn, - Values: []string{"wrong", "value2"}, - }, - }, - }, - }, - }, - }, - }, - Labels: map[string]string{ - "name": "blocked", - "app": "blocked-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, podErr = kClient.CreatePod(initPod, ns) - Ω(podErr).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Verify pod:%s is in pending state", podName)) - podErr = kClient.WaitForPodPending(ns, podName, time.Duration(60)*time.Second) - Ω(podErr).NotTo(HaveOccurred()) - - By("Verify the YuniKorn request failed scheduling") - - podErr = restClient.WaitForAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podName, 60) - Ω(podErr).NotTo(HaveOccurred()) - log, podErr := restClient.GetAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podName) - Ω(podErr).NotTo(HaveOccurred()) - Ω(log).NotTo(BeNil(), "Log can't be empty") - logEntries := yunikorn.AllocLogToStrings(log) - Ω(logEntries).To(ContainElement(MatchRegexp(".*didn't match Pod's node affinity")), "Log entry message mismatch") - }) - - It("Verify_Matching_NodeAffinity_Respected", func() { - nodeName := GetNodeThatCanRunPod(&kClient, ns) - By("Trying to apply a random label on the found node") - key := fmt.Sprintf("kubernetes.io/e2e-%s", common.RandSeq(10)) - value := "102" - labelErr := kClient.SetNodeLabel(nodeName, key, value) - Ω(labelErr).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, key) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - }() - - By("Trying to launch the pod, now with labels.") - labelPodName := "with-labels" - conf := k8s.TestPodConfig{ - Name: labelPodName, - Namespace: ns, - Labels: map[string]string{ - "app": "blocked-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: key, - Operator: v1.NodeSelectorOpIn, - Values: []string{value}, - }, - }, - }, - }, - }, - }, - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, labelPodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPod, err1 := kClient.GetPod(labelPodName, ns) - Ω(err1).NotTo(HaveOccurred()) - Ω(labelPod.Spec.NodeName).Should(Equal(nodeName)) - }) - - // Tests for Taints & Tolerations - It("Verify_Matching_Taint_Tolerations_Respected", func() { - nodeName := getNodeThatCanRunPodWithoutToleration(&kClient, ns) - By("Trying to apply a random taint on the found node.") - taintKey := fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", common.RandSeq(10)) - taintValue := "testing-taint-value" - taintEffect := v1.TaintEffectNoSchedule - err = kClient.TaintNode(nodeName, taintKey, taintValue, taintEffect) - Ω(err).NotTo(HaveOccurred()) - defer func() { - err = kClient.UntaintNode(nodeName, taintKey) - Ω(err).NotTo(HaveOccurred()) - }() - - ginkgo.By("Trying to apply a random label on the found node.") - labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", common.RandSeq(10)) - labelValue := LABELVALUE - labelErr := kClient.SetNodeLabel(nodeName, labelKey, labelValue) - Ω(labelErr).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, labelKey) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, labelKey, labelValue) - Ω(err).NotTo(HaveOccurred()) - }() - - ginkgo.By("Trying to relaunch the pod, now with tolerations.") - tolerationPodName := "with-tolerations" - conf := k8s.TestPodConfig{ - Name: tolerationPodName, - Namespace: ns, - Labels: map[string]string{ - "app": "tolerations-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - Tolerations: []v1.Toleration{{Key: taintKey, Value: taintValue, Effect: taintEffect}}, - NodeSelector: map[string]string{labelKey: labelValue}, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, tolerationPodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPod, err1 := kClient.GetPod(tolerationPodName, ns) - Ω(err1).NotTo(HaveOccurred()) - Ω(labelPod.Spec.NodeName).Should(Equal(nodeName)) - }) - - It("Verify_Not_Matching_Taint_Tolerations_Respected", func() { - nodeName := getNodeThatCanRunPodWithoutToleration(&kClient, ns) - By("Trying to apply a random taint on the found node.") - taintKey := fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", common.RandSeq(10)) - taintValue := "testing-taint-value" - taintEffect := v1.TaintEffectNoSchedule - err = kClient.TaintNode(nodeName, taintKey, taintValue, taintEffect) - Ω(err).NotTo(HaveOccurred()) - defer func() { - err := kClient.UntaintNode(nodeName, taintKey) - Ω(err).NotTo(HaveOccurred()) - }() - - ginkgo.By("Trying to apply a random label on the found node.") - labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", common.RandSeq(10)) - labelValue := LABELVALUE - err := kClient.SetNodeLabel(nodeName, labelKey, labelValue) - Ω(err).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, labelKey) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, labelKey, labelValue) - Ω(err).NotTo(HaveOccurred()) - }() - - ginkgo.By("Trying to relaunch the pod with no tolerations.") - podNameNoTolerations := "with-no-tolerations" - conf := k8s.TestPodConfig{ - Name: podNameNoTolerations, - Namespace: ns, - Labels: map[string]string{ - "app": "no-tolerations-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{labelKey: labelValue}, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Verify pod:%s is in pending state", podNameNoTolerations)) - err = kClient.WaitForPodPending(ns, podNameNoTolerations, time.Duration(60)*time.Second) - Ω(err).NotTo(HaveOccurred()) - - By("Verify the YuniKorn request failed scheduling") - - err = restClient.WaitForAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podNameNoTolerations, 60) - Ω(err).NotTo(HaveOccurred()) - log, err := restClient.GetAllocationLog("default", "root."+ns, initPod.ObjectMeta.Labels["applicationId"], podNameNoTolerations) - Ω(err).NotTo(HaveOccurred()) - Ω(log).NotTo(BeNil(), "Log can't be empty") - logEntries := yunikorn.AllocLogToStrings(log) - Ω(logEntries).To(ContainElement(MatchRegexp(".*taint.*")), "Log entry message mismatch") - - // Remove taint off the node and verify the pod is scheduled on node. - err = kClient.UntaintNode(nodeName, taintKey) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, podNameNoTolerations, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPod, err := kClient.GetPod(podNameNoTolerations, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(labelPod.Spec.NodeName).Should(Equal(nodeName)) - }) - - // Tests for Pod Affinity & Anti Affinity - Context("PodAffinity & AntiAffinity Tests", func() { - type PodAffinityStruct struct { - name string - pod *v1.Pod - pods []*v1.Pod - fits bool - } - - podLabel := map[string]string{"service": "securityscan"} - podLabel2 := map[string]string{"security": "S1"} - var nodeName, labelKey string - sleepContainer := v1.Container{Name: "test-container", Image: "alpine:latest", Command: []string{"sleep", "60"}} - - labelKey = "kubernetes.io/hostname" - - BeforeEach(func() { - By("Finding a node that can fit the pods") - nodeName = GetNodeThatCanRunPod(&kClient, ns) - }) - - DescribeTable("", func(t PodAffinityStruct) { - By("Launching the pods with labels") - for _, pod := range t.pods { - pod.Labels["app"] = common.RandSeq(5) - pod.Labels["applicationId"] = common.RandSeq(10) - createdPod, err := kClient.CreatePod(pod, ns) - Ω(err).NotTo(HaveOccurred()) - - err = kClient.WaitForPodScheduled(createdPod.Namespace, createdPod.Name, time.Duration(60)*time.Second) - Ω(err).NotTo(HaveOccurred()) - } - - By("Launching the pod with Pod (anti)-affinity") - t.pod.Labels["app"] = common.RandSeq(5) - t.pod.Labels["applicationId"] = common.RandSeq(10) - testPod, err := kClient.CreatePod(t.pod, ns) - Ω(err).NotTo(HaveOccurred()) - var err1 error - if t.fits { - err1 = kClient.WaitForPodScheduled(testPod.Namespace, testPod.Name, time.Duration(60)*time.Second) - } else { - err1 = kClient.WaitForPodUnschedulable(testPod, time.Duration(60)*time.Second) - } - Ω(err1).NotTo(HaveOccurred()) - }, - Entry("Verify_Not_Matching_Inter_Pod-Affinity_Respected", - PodAffinityStruct{ - name: "Verify_Not_Matching_Inter_Pod-Affinity_Respected", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - fits: false, - }), - Entry("Verify_Matching_Inter_Pod-Affinity_Respected", - PodAffinityStruct{ - name: "Verify_Matching_Inter-Pod-Affinity_Respected", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName, - }, - }, - }, - fits: true, - }), - Entry("Verify_Matching_Inter_Pod-Affinity_Respected_For_NotIn_Operator", - PodAffinityStruct{ - name: "Verify_Matching_Inter-Pod-Affinity_Respected_For_NotIn_Operator", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"securityscan3", "value3"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: true, - }), - Entry("Verify_Matching_Inter-Pod-Affinity_Under_Diff_NS", - PodAffinityStruct{ - name: "validates that inter-pod-affinity is respected when pods have different Namespaces", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - Namespaces: []string{"diff-namespace"}, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel, - Namespace: anotherNS}}}, - fits: false, - }), - Entry("Verify_Inter-Pod-Affinity_With_UnMatching_LabelSelector", - PodAffinityStruct{ - name: "Doesn't satisfy the PodAffinity because of unmatching labelSelector with the existing pod", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"antivirusscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: false, - }), - /*If you specify multiple nodeSelectorTerms associated with nodeAffinity types, - then the pod can be scheduled onto a node if one of the nodeSelectorTerms can be satisfied. - - If you specify multiple matchExpressions associated with nodeSelectorTerms, - then the pod can be scheduled onto a node only if all matchExpressions is satisfied. - */ - Entry("Verify_Inter-Pod-Affinity_With_Multiple_Affinities", - PodAffinityStruct{ - name: "validates that InterPodAffinity is respected if matching with multiple affinities in multiple RequiredDuringSchedulingIgnoredDuringExecution ", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpExists, - }, { - Key: "wrongkey", - Operator: metav1.LabelSelectorOpDoesNotExist, - }, - }, - }, - TopologyKey: labelKey, - }, { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan"}, - }, { - Key: "service", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"WrongValue"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: true, - }), - Entry("Verify_Pod_Affinity_With_One_UnMatched_Expression", - PodAffinityStruct{ - name: "The labelSelector requirements(items of matchExpressions) are ANDed, the pod cannot schedule onto the node because one of the matchExpression items doesn't match.", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabel2, - Name: "fakename", - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpExists, - }, { - Key: "wrongkey", - Operator: metav1.LabelSelectorOpDoesNotExist, - }, - }, - }, - TopologyKey: labelKey, - }, { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan2"}, - }, { - Key: "service", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"WrongValue"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: false, - }), - Entry("Verfiy_Matched_Pod_Affinity_Anti_Affinity_Respected", - PodAffinityStruct{ - name: "validates that InterPod Affinity and AntiAffinity is respected if matching", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"antivirusscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: true, - }), - Entry("Verify_Matched_Pod_Affinity_Unmatched_Pod_Antiaffinity", - PodAffinityStruct{ - name: "satisfies the PodAffinity but doesn't satisfies the PodAntiAffinity with the existing pod", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakename", - Labels: podLabel2, - }, - Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: labelKey, - }, - }, - }, - }, - }, - }, - pods: []*v1.Pod{{Spec: v1.PodSpec{ - SchedulerName: configmanager.SchedulerName, - Containers: []v1.Container{sleepContainer}, - NodeName: nodeName}, ObjectMeta: metav1.ObjectMeta{ - Name: "fakename2", - Labels: podLabel}}}, - fits: false, - }), - ) - }) - - // Tests for PodFitsHost - It("Verify_PodFitsHost_Respected", func() { - nodeName := GetNodeThatCanRunPod(&kClient, ns) - - By("Trying to launch the pod, now with labels.") - PodName := "test-pod" - conf := k8s.TestPodConfig{ - Name: PodName, - NodeName: nodeName, - Labels: map[string]string{ - "app": "podFitsHost-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - Namespace: ns, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err := kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, PodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPod, err := kClient.GetPod(PodName, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(labelPod.Spec.NodeName).Should(Equal(nodeName)) - }) - - // Tests for PodFitsHostPorts - It("Verify_No_Conflict_With_Same_HostPort_Diff_IP_Protocol", func() { - nodeName := GetNodeThatCanRunPod(&kClient, ns) - - By("Trying to apply a random label on the found node") - key := fmt.Sprintf("kubernetes.io/e2e-hostport-%s", common.RandSeq(10)) - value := "103" - err := kClient.SetNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, key) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - }() - - port := int32(54321) - portMap := [][]interface{}{ - {"127.0.0.1", v1.ProtocolTCP}, - {"127.0.0.2", v1.ProtocolTCP}, - {"127.0.0.2", v1.ProtocolUDP}, - } - for _, p := range portMap { - s := reflect.ValueOf(p) - Port := port - Protocol, success := s.Index(1).Interface().(v1.Protocol) - Ω(success).To(BeTrue()) - HostIP, success := s.Index(0).Interface().(string) - Ω(success).To(BeTrue()) - labelPodName := "same-hostport-" + common.RandSeq(10) - By(fmt.Sprintf("Trying to create a pod(%s) with hostport %v, protocol %v and hostIP %v and expect scheduled", labelPodName, Port, Protocol, HostIP)) - conf := k8s.TestPodConfig{ - Name: labelPodName, - Namespace: ns, - Labels: map[string]string{ - "app": "sameHostPort-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{ - key: value, - }, - Ports: []v1.ContainerPort{ - { - HostPort: Port, - ContainerPort: 80, - Protocol: Protocol, - HostIP: HostIP, - }, - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err := kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, labelPodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - } - }) - - It("Verify_Conflict_With_Same_HostPort_Same_IP_Protocol", func() { - nodeName := GetNodeThatCanRunPod(&kClient, ns) - - By("Trying to apply a random label on the found node") - key := fmt.Sprintf("kubernetes.io/e2e-hostport-%s", common.RandSeq(10)) - value := "104" - err := kClient.SetNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - exists, existsErr := kClient.IsNodeLabelExists(nodeName, key) - Ω(existsErr).NotTo(HaveOccurred()) - Ω(exists).To(BeTrue()) - defer func() { - err = kClient.RemoveNodeLabel(nodeName, key, value) - Ω(err).NotTo(HaveOccurred()) - }() - - port := int32(54322) - labelPodName := "same-hostport-" + common.RandSeq(10) - By(fmt.Sprintf("Trying to create a pod(%s) with hostport %v, protocol %v and hostIP %v and expect scheduled", labelPodName, port, v1.ProtocolTCP, "0.0.0.0")) - conf := k8s.TestPodConfig{ - Name: labelPodName, - Namespace: ns, - Labels: map[string]string{ - "app": "sameHostPort-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{ - key: value, - }, - Ports: []v1.ContainerPort{ - { - HostPort: port, - ContainerPort: 80, - Protocol: v1.ProtocolTCP, - HostIP: "", - }, - }, - } - - initPod, podErr := k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(ns, labelPodName, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - labelPodName2 := "same-hostport-" + common.RandSeq(10) - By(fmt.Sprintf("Trying to create a pod(%s) with hostport %v, protocol %v and hostIP %v and expect not scheduled", labelPodName2, port, v1.ProtocolTCP, "127.0.0.1")) - conf = k8s.TestPodConfig{ - Name: labelPodName2, - Namespace: anotherNS, - Labels: map[string]string{ - "app": "sameHostPort-app-" + common.RandSeq(5), - "applicationId": common.RandSeq(10), - }, - NodeSelector: map[string]string{ - key: value, - }, - Ports: []v1.ContainerPort{ - { - HostPort: port, - ContainerPort: 80, - Protocol: v1.ProtocolTCP, - HostIP: "127.0.0.1", - }, - }, - } - - initPod, podErr = k8s.InitTestPod(conf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, anotherNS) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Verify pod:%s is in pending state", labelPodName2)) - err = kClient.WaitForPodPending(anotherNS, labelPodName2, time.Duration(60)*time.Second) - Ω(err).NotTo(HaveOccurred()) - - By("Verify the YuniKorn request failed scheduling") - - err = restClient.WaitForAllocationLog("default", "root."+anotherNS, initPod.ObjectMeta.Labels["applicationId"], labelPodName2, 60) - Ω(err).NotTo(HaveOccurred()) - log, err := restClient.GetAllocationLog("default", "root."+anotherNS, initPod.ObjectMeta.Labels["applicationId"], labelPodName2) - Ω(err).NotTo(HaveOccurred()) - Ω(log).NotTo(BeNil(), "Log can't be empty") - logEntries := yunikorn.AllocLogToStrings(log) - Ω(logEntries).To(ContainElement(MatchRegexp(".*free ports.*")), "Log entry message mismatch") - }) - - AfterEach(func() { - By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(HaveOccurred()) - Ω(checks).To(Equal(""), checks) - }) -}) diff --git a/test/e2e/priority_scheduling/priority_scheduling_suite_test.go b/test/e2e/priority_scheduling/priority_scheduling_suite_test.go deleted file mode 100644 index b5f40f775..000000000 --- a/test/e2e/priority_scheduling/priority_scheduling_suite_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package priority_test - -import ( - "fmt" - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - schedulingv1 "k8s.io/api/scheduling/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestPriorityScheduling(t *testing.T) { - ginkgo.ReportAfterSuite("TestPriorityScheduling", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-priority_scheduling_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestPriorityScheduling", ginkgo.Label("TestPriorityScheduling")) -} - -var suiteName string -var kubeClient k8s.KubeCtl - -var preemptPolicyNever = v1.PreemptNever - -var lowPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-low", - }, - Value: -100, - PreemptionPolicy: &preemptPolicyNever, -} - -var highPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-high", - }, - Value: 100, - PreemptionPolicy: &preemptPolicyNever, -} - -var normalPriorityClass = schedulingv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "yk-test-normal", - }, - Value: 0, - PreemptionPolicy: &preemptPolicyNever, -} - -var annotation = "ann-" + common.RandSeq(10) -var oldConfigMap = new(v1.ConfigMap) - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - var err error - kubeClient = k8s.KubeCtl{} - Expect(kubeClient.SetClient()).To(BeNil()) - - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) - - By(fmt.Sprintf("Creating priority class %s", lowPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&lowPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Creating priority class %s", highPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&highPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Creating priority class %s", normalPriorityClass.Name)) - _, err = kubeClient.CreatePriorityClass(&normalPriorityClass) - Ω(err).ShouldNot(HaveOccurred()) -}) - -var _ = ginkgo.AfterSuite(func() { - var err error - kubeClient = k8s.KubeCtl{} - Expect(kubeClient.SetClient()).To(BeNil()) - - By(fmt.Sprintf("Removing priority class %s", normalPriorityClass.Name)) - err = kubeClient.DeletePriorityClass(normalPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Removing priority class %s", highPriorityClass.Name)) - err = kubeClient.DeletePriorityClass(highPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Removing priority class %s", lowPriorityClass.Name)) - err = kubeClient.DeletePriorityClass(lowPriorityClass.Name) - Ω(err).ShouldNot(HaveOccurred()) - - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -var By = ginkgo.By - -var Ω = gomega.Ω -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred -var Expect = gomega.Expect -var Equal = gomega.Equal diff --git a/test/e2e/priority_scheduling/priority_scheduling_test.go b/test/e2e/priority_scheduling/priority_scheduling_test.go deleted file mode 100644 index dc3540dee..000000000 --- a/test/e2e/priority_scheduling/priority_scheduling_test.go +++ /dev/null @@ -1,515 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package priority_test - -import ( - "fmt" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - "github.com/apache/yunikorn-core/pkg/common/configs" - "github.com/apache/yunikorn-k8shim/pkg/cache" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" -) - -const ( - requestCPU = "100m" - requestMem = "100M" -) - -var ( - ns string - rr = &v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(requestCPU), - v1.ResourceMemory: resource.MustParse(requestMem), - }, - } -) - -var _ = ginkgo.Describe("PriorityScheduling", func() { - var namespace *v1.Namespace - var err error - var oldConfigMap = new(v1.ConfigMap) - var annotation string - var sleepPodConf, lowPodConf, normalPodConf, highPodConf k8s.TestPodConfig - - ginkgo.BeforeEach(func() { - ns = "test-" + common.RandSeq(10) - - By(fmt.Sprintf("Creating test namespace %s", ns)) - namespace, err = kubeClient.CreateNamespace(ns, map[string]string{}) - Ω(err).ShouldNot(HaveOccurred()) - Ω(namespace.Status.Phase).Should(Equal(v1.NamespaceActive)) - }) - - ginkgo.It("Verify_Static_Queue_App_Scheduling_Order", func() { - By("Setting custom YuniKorn configuration") - annotation = "ann-" + common.RandSeq(10) - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "fifo", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{ - Name: "fence", - Parent: true, - Resources: configs.Resources{Max: map[string]string{siCommon.CPU: requestCPU, siCommon.Memory: requestMem}}, - Properties: map[string]string{configs.PriorityPolicy: "fence"}, - }); err != nil { - return err - } - if err = common.AddQueue(sc, "default", "root.fence", configs.QueueConfig{Name: "child1"}); err != nil { - return err - } - if err = common.AddQueue(sc, "default", "root.fence", configs.QueueConfig{Name: "child2"}); err != nil { - return err - } - - return nil - }) - - sleepPodConf = k8s.TestPodConfig{ - Name: "test-sleep-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.fence.child2", - constants.LabelApplicationID: "app-sleep-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - - lowPodConf = k8s.TestPodConfig{ - Name: "test-low-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.fence.child1", - constants.LabelApplicationID: "app-low-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - PriorityClassName: lowPriorityClass.Name, - } - - normalPodConf = k8s.TestPodConfig{ - Name: "test-normal-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.fence.child2", - constants.LabelApplicationID: "app-normal-" + common.RandSeq(5)}, - Resources: rr, - Namespace: ns, - PriorityClassName: normalPriorityClass.Name, - } - - highPodConf = k8s.TestPodConfig{ - Name: "test-high-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.fence.child1", - constants.LabelApplicationID: "app-high-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - PriorityClassName: highPriorityClass.Name, - } - validatePodSchedulingOrder(ns, sleepPodConf, lowPodConf, normalPodConf, highPodConf) - }) - - ginkgo.It("Verify_Dynamic_Queue_App_Scheduling_Order", func() { - By("Setting custom YuniKorn configuration") - annotation = "ann-" + common.RandSeq(10) - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo", annotation) - - By(fmt.Sprintf("Update test namespace quota %s", ns)) - namespace, err = kubeClient.UpdateNamespace(ns, map[string]string{ - constants.NamespaceQuota: fmt.Sprintf("{\"%s\": \"%s\", \"%s\": \"%s\"}", v1.ResourceCPU, requestCPU, v1.ResourceMemory, requestMem), - }) - Ω(err).ShouldNot(HaveOccurred()) - Ω(namespace.Status.Phase).Should(Equal(v1.NamespaceActive)) - - sleepPodConf = k8s.TestPodConfig{ - Name: "test-sleep-" + common.RandSeq(5), - Labels: map[string]string{constants.LabelApplicationID: "app-sleep-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - - lowPodConf = k8s.TestPodConfig{ - Name: "test-low-priority-" + common.RandSeq(5), - Labels: map[string]string{constants.LabelApplicationID: "app-low-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - PriorityClassName: lowPriorityClass.Name, - } - - normalPodConf = k8s.TestPodConfig{ - Name: "test-normal-priority-" + common.RandSeq(5), - Labels: map[string]string{constants.LabelApplicationID: "app-normal-" + common.RandSeq(5)}, - Resources: rr, - Namespace: ns, - PriorityClassName: normalPriorityClass.Name, - } - - highPodConf = k8s.TestPodConfig{ - Name: "test-high-priority-" + common.RandSeq(5), - Labels: map[string]string{constants.LabelApplicationID: "app-high-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - PriorityClassName: highPriorityClass.Name, - } - validatePodSchedulingOrder(ns, sleepPodConf, lowPodConf, normalPodConf, highPodConf) - }) - - ginkgo.It("Verify_Priority_Offset_Queue_App_Scheduling_Order", func() { - By("Setting custom YuniKorn configuration") - annotation = "ann-" + common.RandSeq(10) - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "fifo", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{ - Name: "priority", - Parent: true, - Resources: configs.Resources{Max: map[string]string{siCommon.CPU: "100m", siCommon.Memory: "100M"}}, - Properties: map[string]string{configs.PriorityPolicy: "fence"}, - }); err != nil { - return err - } - if err = common.AddQueue(sc, "default", "root.priority", configs.QueueConfig{ - Name: "high", - Properties: map[string]string{configs.PriorityOffset: "100"}, - }); err != nil { - return err - } - if err = common.AddQueue(sc, "default", "root.priority", configs.QueueConfig{ - Name: "normal", - Properties: map[string]string{configs.PriorityOffset: "0"}, - }); err != nil { - return err - } - if err = common.AddQueue(sc, "default", "root.priority", configs.QueueConfig{ - Name: "low", - Properties: map[string]string{configs.PriorityOffset: "-100"}, - }); err != nil { - return err - } - - return nil - }) - - sleepPodConf = k8s.TestPodConfig{ - Name: "test-sleep-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.priority.high", - constants.LabelApplicationID: "app-sleep-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - - lowPodConf = k8s.TestPodConfig{ - Name: "test-low-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.priority.low", - constants.LabelApplicationID: "app-low-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - - normalPodConf = k8s.TestPodConfig{ - Name: "test-normal-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.priority.normal", - constants.LabelApplicationID: "app-normal-" + common.RandSeq(5)}, - Resources: rr, - Namespace: ns, - } - - highPodConf = k8s.TestPodConfig{ - Name: "test-high-priority-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.priority.high", - constants.LabelApplicationID: "app-high-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - validatePodSchedulingOrder(ns, sleepPodConf, lowPodConf, normalPodConf, highPodConf) - }) - - ginkgo.It("Verify_Gang_Scheduling_With_Priority", func() { - By("Setting custom YuniKorn configuration") - annotation = "ann-" + common.RandSeq(10) - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "fifo", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{ - Name: "default", - Parent: false, - Resources: configs.Resources{Max: map[string]string{siCommon.CPU: "100m", siCommon.Memory: "100M"}}, - }); err != nil { - return err - } - - return nil - }) - sleepPodConf = k8s.TestPodConfig{ - Name: "test-sleep-" + common.RandSeq(5), - Labels: map[string]string{ - constants.LabelQueueName: "root.default", - constants.LabelApplicationID: "app-sleep-" + common.RandSeq(5)}, - Namespace: ns, - Resources: rr, - } - - taskGroupMinResource := map[string]resource.Quantity{} - for k, v := range rr.Requests { - taskGroupMinResource[k.String()] = v - } - lowPodConf = createPodConfWithTaskGroup("low", lowPriorityClass.Name, taskGroupMinResource) - normalPodConf = createPodConfWithTaskGroup("normal", normalPriorityClass.Name, taskGroupMinResource) - highPodConf = createPodConfWithTaskGroup("high", highPriorityClass.Name, taskGroupMinResource) - - var sleepPod, lowPod, normalPod, highPod *v1.Pod - By("Create sleep pod to consume queue") - sleepPod, err = k8s.InitTestPod(sleepPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - sleepPod, err = kubeClient.CreatePod(sleepPod, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kubeClient.WaitForPodRunning(ns, sleepPod.Name, 1*time.Minute) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Submit low priority job") - lowPod, err = k8s.InitTestPod(lowPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - lowJob := k8s.InitTestJob(lowPod.Labels[constants.LabelApplicationID], 1, 1, lowPod) - lowJob, err = kubeClient.CreateJob(lowJob, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kubeClient.WaitForJobPodsCreated(ns, lowJob.Name, 1, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Submit normal priority job") - normalPod, err = k8s.InitTestPod(normalPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - normalJob := k8s.InitTestJob(normalPod.Labels[constants.LabelApplicationID], 1, 1, normalPod) - normalJob, err = kubeClient.CreateJob(normalJob, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kubeClient.WaitForJobPodsCreated(ns, normalJob.Name, 1, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Submit high priority job") - highPod, err = k8s.InitTestPod(highPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - highJob := k8s.InitTestJob(highPod.Labels[constants.LabelApplicationID], 1, 1, highPod) - highJob, err = kubeClient.CreateJob(highJob, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kubeClient.WaitForJobPodsCreated(ns, highJob.Name, 1, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for scheduler state to settle") - time.Sleep(10 * time.Second) - - var lowPods, normalPods, highPods *v1.PodList - lowPods, err = kubeClient.ListPods(ns, fmt.Sprintf("job-name=%s", lowJob.Name)) - Ω(err).NotTo(gomega.HaveOccurred()) - lowPod = &lowPods.Items[0] - - normalPods, err = kubeClient.ListPods(ns, fmt.Sprintf("job-name=%s", normalJob.Name)) - Ω(err).NotTo(gomega.HaveOccurred()) - normalPod = &normalPods.Items[0] - - highPods, err = kubeClient.ListPods(ns, fmt.Sprintf("job-name=%s", highJob.Name)) - Ω(err).NotTo(gomega.HaveOccurred()) - highPod = &highPods.Items[0] - - By("Ensure no test pods are running") - ensureNotRunning(ns, lowPod, normalPod, highPod) - - By("Kill sleep pod to make room for test pods") - err = kubeClient.DeletePod(sleepPod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for high-priority placeholders terminated") - err = kubeClient.WaitForPlaceholders(ns, "tg-"+highPodConf.Labels["applicationId"]+"-", 0, 30*time.Second, nil) - Ω(err).NotTo(HaveOccurred()) - - By("Wait for high-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, highPod.Name, 1*time.Minute) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Ensure low and normal priority pods are not running") - ensureNotRunning(ns, lowPod, normalPod) - - By("Kill high-priority job") - err = kubeClient.DeleteJob(highJob.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for normal-priority placeholders terminated") - err = kubeClient.WaitForPlaceholders(ns, "tg-"+normalPodConf.Labels["applicationId"]+"-", 0, 30*time.Second, nil) - Ω(err).NotTo(HaveOccurred()) - - By("Wait for normal-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, normalPod.Name, 1*time.Minute) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Ensure low priority pod is not running") - ensureNotRunning(ns, lowPod) - - By("Kill normal-priority job") - err = kubeClient.DeleteJob(normalJob.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for low-priority placeholders terminated") - err = kubeClient.WaitForPlaceholders(ns, "tg-"+lowPodConf.Labels["applicationId"]+"-", 0, 30*time.Second, nil) - Ω(err).NotTo(HaveOccurred()) - - By("Wait for low-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, lowPod.Name, 1*time.Minute) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Kill low-priority job") - err = kubeClient.DeleteJob(lowJob.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - // If there is any error test case, we need to delete all pods to make sure it doesn't influence other cases. - ginkgo.By("Delete all sleep pods") - err = kubeClient.DeletePods(ns) - if err != nil { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pods in namespace %s - reason is %s\n", ns, err.Error()) - } - - By(fmt.Sprintf("Tearing down namespace %s", ns)) - err = kubeClient.TearDownNamespace(ns) - Ω(err).ShouldNot(HaveOccurred()) - - By("Restoring YuniKorn configuration") - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) - }) -}) - -func validatePodSchedulingOrder(ns string, sleepPodConf, lowPodConf, normalPodConf, highPodConf k8s.TestPodConfig) { - var err error - var sleepPod *v1.Pod - - By("Create sleep pod to consume queue") - sleepPod, err = k8s.InitTestPod(sleepPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - sleepPod, err = kubeClient.CreatePod(sleepPod, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kubeClient.WaitForPodRunning(ns, sleepPod.Name, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Submit low priority pod") - lowPod, err := k8s.InitTestPod(lowPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - lowPod, err = kubeClient.CreatePod(lowPod, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - time.Sleep(1 * time.Second) - - By("Submit normal priority pod") - normalPod, err := k8s.InitTestPod(normalPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - normalPod, err = kubeClient.CreatePod(normalPod, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - time.Sleep(1 * time.Second) - - By("Submit high priority pod") - highPod, err := k8s.InitTestPod(highPodConf) - Ω(err).NotTo(gomega.HaveOccurred()) - highPod, err = kubeClient.CreatePod(highPod, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - time.Sleep(1 * time.Second) - - By("Wait for scheduler state to settle") - time.Sleep(10 * time.Second) - - By("Ensure no test pods are running") - ensureNotRunning(ns, lowPod, normalPod, highPod) - - By("Kill sleep pod to make room for test pods") - err = kubeClient.DeletePod(sleepPod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for high-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, highPod.Name, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Ensure low and normal priority pods are not running") - ensureNotRunning(ns, lowPod, normalPod) - - By("Kill high-priority pod") - err = kubeClient.DeletePod(highPod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for normal-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, normalPod.Name, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Ensure low priority pod is not running") - ensureNotRunning(ns, lowPod) - - By("Kill normal-priority pod") - err = kubeClient.DeletePod(normalPod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Wait for low-priority pod to begin running") - err = kubeClient.WaitForPodRunning(ns, lowPod.Name, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - By("Kill low-priority pod") - err = kubeClient.DeletePod(lowPod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) -} - -func ensureNotRunning(ns string, pods ...*v1.Pod) { - for _, pod := range pods { - podResult, err := kubeClient.GetPod(pod.Name, ns) - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(podResult.Status.Phase).ShouldNot(Equal(v1.PodRunning), pod.Name) - } -} - -func createPodConfWithTaskGroup(name, priorityClassName string, taskGroupMinResource map[string]resource.Quantity) k8s.TestPodConfig { - return k8s.TestPodConfig{ - Name: fmt.Sprintf("test-%s-priority-%s", name, common.RandSeq(5)), - Labels: map[string]string{ - constants.LabelQueueName: "root.default", - constants.LabelApplicationID: fmt.Sprintf("app-%s-%s", name, common.RandSeq(5))}, - Namespace: ns, - Annotations: &k8s.PodAnnotation{ - TaskGroupName: "group-" + name, - TaskGroups: []cache.TaskGroup{ - { - Name: "group-" + name, - MinMember: int32(1), - MinResource: taskGroupMinResource, - }, - }, - }, - Resources: rr, - PriorityClassName: priorityClassName, - RestartPolicy: v1.RestartPolicyNever, - } -} diff --git a/test/e2e/queue_quota_mgmt/queue_quota_mgmt_suite_test.go b/test/e2e/queue_quota_mgmt/queue_quota_mgmt_suite_test.go deleted file mode 100644 index e14229139..000000000 --- a/test/e2e/queue_quota_mgmt/queue_quota_mgmt_suite_test.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package queuequotamgmt_test - -import ( - "path/filepath" - "runtime" - "testing" - - v1 "k8s.io/api/core/v1" - - "github.com/onsi/ginkgo/v2/reporters" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -func TestQueueQuotaMgmt(t *testing.T) { - ginkgo.ReportAfterSuite("TestQueueQuotaMgmt", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-queue_quota_mgmt_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestQueueQuotaMgmt", ginkgo.Label("TestQueueQuotaMgmt")) -} - -// Declarations for Ginkgo DSL -var Fail = ginkgo.Fail -var Describe = ginkgo.Describe -var It = ginkgo.It -var PIt = ginkgo.PIt -var By = ginkgo.By -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach -var DescribeTable = ginkgo.DescribeTable -var Entry = ginkgo.Entry -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite - -// Declarations for Gomega DSL -var RegisterFailHandler = gomega.RegisterFailHandler - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred -var BeEmpty = gomega.BeEmpty -var BeEquivalentTo = gomega.BeEquivalentTo diff --git a/test/e2e/queue_quota_mgmt/queue_quota_mgmt_test.go b/test/e2e/queue_quota_mgmt/queue_quota_mgmt_test.go deleted file mode 100644 index 97ecda451..000000000 --- a/test/e2e/queue_quota_mgmt/queue_quota_mgmt_test.go +++ /dev/null @@ -1,260 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package queuequotamgmt_test - -import ( - "fmt" - "math" - "time" - - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" -) - -var _ = Describe("", func() { - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var sleepRespPod *v1.Pod - var maxCPUAnnotation = siCommon.DomainYuniKorn + "namespace.max.cpu" - var maxMemAnnotation = siCommon.DomainYuniKorn + "namespace.max.memory" - var quotaAnnotation = siCommon.DomainYuniKorn + "namespace.quota" - var oldAnnotations map[string]string - var expectedOldAnnotations map[string]int64 - var newAnnotations map[string]string - var expectedNewAnnotations map[string]int64 - var oldAndNewAnnotations map[string]string - var expectedOldAndNewAnnotations map[string]int64 - var emptyOldAnnotations map[string]string - var wrongOldAnnotations map[string]string - var wrongNewAnnotations map[string]string - var ns string - var queueInfo *dao.PartitionQueueDAOInfo - var maxResource yunikorn.ResourceUsage - var usedResource yunikorn.ResourceUsage - var usedPerctResource yunikorn.ResourceUsage - var pods []string - - BeforeEach(func() { - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - pods = []string{} - oldAnnotations = map[string]string{} - newAnnotations = map[string]string{} - oldAndNewAnnotations = map[string]string{} - emptyOldAnnotations = map[string]string{} - wrongOldAnnotations = map[string]string{} - wrongNewAnnotations = map[string]string{} - expectedOldAnnotations = map[string]int64{} - expectedNewAnnotations = map[string]int64{} - expectedOldAndNewAnnotations = map[string]int64{} - oldAnnotations[maxCPUAnnotation] = "300m" - oldAnnotations[maxMemAnnotation] = "300M" - newAnnotations[quotaAnnotation] = "{\"cpu\": \"400m\", \"memory\": \"400M\", \"nvidia.com/gpu\": \"1\"}" - oldAndNewAnnotations[maxCPUAnnotation] = "300m" - oldAndNewAnnotations[maxMemAnnotation] = "300M" - oldAndNewAnnotations[quotaAnnotation] = "{\"cpu\": \"600m\", \"memory\": \"600M\", \"nvidia.com/gpu\": \"2\"}" - emptyOldAnnotations[maxCPUAnnotation] = "nil" - emptyOldAnnotations[maxMemAnnotation] = "nil" - wrongOldAnnotations[maxCPUAnnotation] = "a" - wrongOldAnnotations[maxMemAnnotation] = "b" - wrongNewAnnotations[quotaAnnotation] = "{\"cpu\": \"error\", \"memory\": \"error\"}" - expectedOldAnnotations[siCommon.CPU] = 300 - expectedOldAnnotations[siCommon.Memory] = 300 - expectedNewAnnotations[siCommon.CPU] = 400 - expectedNewAnnotations[siCommon.Memory] = 400 - expectedNewAnnotations["nvidia.com/gpu"] = 1 - expectedOldAndNewAnnotations[siCommon.CPU] = 600 - expectedOldAndNewAnnotations[siCommon.Memory] = 600 - expectedOldAndNewAnnotations["nvidia.com/gpu"] = 2 - }) - - /* - 1. Create a namespace with different types of annotations set to max CPU, max Mem, max GPU etc - 2. Submit pod with CPU and Mem requests - 3. Verify the CPU and memory usage reported by Yunikorn is accurate. - 4. Submit another pod with resource request that exceeds the limits - 5. Verify the pod is in un-schedulable state - 6. Wait until pod-1 completes and verify that pod-2 is scheduled now. - */ - verifyQuotaAllocationUsingAnnotation := func(annotations map[string]string, resources map[string]int64, podCount int64) { - maxCPU := resources[siCommon.CPU] - maxMem := resources[siCommon.Memory] - maxGPU := int64(0) - ok := false - if maxGPU, ok = resources["nvidia.com/gpu"]; ok { - maxGPU = resources["nvidia.com/gpu"] - } - - ns = "ns-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace with maxCPU: %dm and maxMem: %dM", ns, maxCPU, maxMem)) - ns1, err1 := kClient.CreateNamespace(ns, annotations) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - - // Create sleep pod template - var reqCPU int64 = 100 - var reqMem int64 = 100 - - var iter int64 - nextPod := podCount + 1 - for iter = 1; iter <= podCount; iter++ { - sleepPodConfigs := k8s.SleepPodConfig{NS: ns, Time: 60, CPU: reqCPU, Mem: reqMem} - sleepObj, podErr := k8s.InitSleepPod(sleepPodConfigs) - Ω(podErr).NotTo(HaveOccurred()) - - pods = append(pods, sleepObj.Name) - By(fmt.Sprintf("App-%d: Deploy the sleep app:%s to %s namespace", iter, sleepObj.Name, ns)) - sleepRespPod, err = kClient.CreatePod(sleepObj, ns) - Ω(err).NotTo(HaveOccurred()) - - Ω(kClient.WaitForPodRunning(sleepRespPod.Namespace, sleepRespPod.Name, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - // Verify that the resources requested by above sleep pod is accounted for in the queues response - queueInfo, err = restClient.GetQueue("default", "root."+ns) - Ω(err).NotTo(HaveOccurred()) - Ω(queueInfo).NotTo(BeNil()) - Ω(queueInfo.QueueName).Should(Equal("root." + ns)) - Ω(queueInfo.Status).Should(Equal("Active")) - Ω(queueInfo.Properties).Should(BeEmpty()) - maxResource.ParseResourceUsage(queueInfo.MaxResource) - usedResource.ParseResourceUsage(queueInfo.AllocatedResource) - usedPerctResource.ParseResourceUsage(queueInfo.AbsUsedCapacity) - - By(fmt.Sprintf("App-%d: Verify max capacity on the queue is accurate", iter)) - Ω(maxResource.GetResourceValue(siCommon.CPU)).Should(Equal(maxCPU)) - Ω(maxResource.GetResourceValue(siCommon.Memory)).Should(Equal(maxMem * 1000 * 1000)) - Ω(maxResource.GetResourceValue("nvidia.com/gpu")).Should(Equal(maxGPU)) - - By(fmt.Sprintf("App-%d: Verify used capacity on the queue is accurate after 1st pod deployment", iter)) - Ω(usedResource.GetResourceValue(siCommon.CPU)).Should(Equal(reqCPU * iter)) - Ω(usedResource.GetResourceValue(siCommon.Memory)).Should(Equal(reqMem * iter * 1000 * 1000)) - - var perctCPU = int64(math.Floor((float64(reqCPU*iter) / float64(maxCPU)) * 100)) - var perctMem = int64(math.Floor((float64(reqMem*iter) / float64(maxMem)) * 100)) - By(fmt.Sprintf("App-%d: Verify used percentage capacity on the queue is accurate after 1st pod deployment", iter)) - Ω(usedPerctResource.GetResourceValue(siCommon.CPU)).Should(Equal(perctCPU)) - Ω(usedPerctResource.GetResourceValue(siCommon.Memory)).Should(Equal(perctMem)) - } - - By(fmt.Sprintf("App-%d: Submit another app which exceeds the queue quota limitation", nextPod)) - sleepPodConfigs := k8s.SleepPodConfig{NS: ns, Time: 60, CPU: reqCPU, Mem: reqMem} - sleepObj, app4PodErr := k8s.InitSleepPod(sleepPodConfigs) - Ω(app4PodErr).NotTo(HaveOccurred()) - - By(fmt.Sprintf("App-%d: Deploy the sleep app:%s to %s namespace", nextPod, sleepObj.Name, ns)) - sleepRespPod, err = kClient.CreatePod(sleepObj, ns) - Ω(err).NotTo(HaveOccurred()) - pods = append(pods, sleepObj.Name) - - By(fmt.Sprintf("App-%d: Verify app:%s in accepted state", nextPod, sleepObj.Name)) - // Wait for pod to move to accepted state - err = restClient.WaitForAppStateTransition("default", "root."+ns, sleepRespPod.ObjectMeta.Labels["applicationId"], - yunikorn.States().Application.Accepted, - 240) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Pod-%d: Verify pod:%s is in pending state", nextPod, sleepObj.Name)) - err = kClient.WaitForPodPending(ns, sleepObj.Name, time.Duration(60)*time.Second) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("App-1: Wait for 1st app:%s to complete, to make enough capacity to run the last app", pods[0])) - // Wait for pod to move to accepted state - err = kClient.WaitForPodSucceeded(ns, pods[0], time.Duration(360)*time.Second) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Pod-%d: Verify Pod:%s moved to running state", nextPod, sleepObj.Name)) - Ω(kClient.WaitForPodRunning(sleepRespPod.Namespace, sleepRespPod.Name, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - - } - - It("Verify_Queue_Quota_Allocation_Using_Old_Annotations", func() { - verifyQuotaAllocationUsingAnnotation(oldAnnotations, expectedOldAnnotations, 3) - }) - - It("Verify_Queue_Quota_Allocation_Using_New_Annotations", func() { - verifyQuotaAllocationUsingAnnotation(newAnnotations, expectedNewAnnotations, 4) - }) - - It("Verify_Queue_Quota_Allocation_Using_Old_New_Annotations", func() { - verifyQuotaAllocationUsingAnnotation(oldAndNewAnnotations, expectedOldAndNewAnnotations, 6) - }) - - It("Verify_NS_Without_Quota", func() { - ns = "ns-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace without quota information", ns)) - ns1, err1 := kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - DescribeTable("", func(annotations map[string]string) { - ns = "ns-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace with empty annotations", ns)) - ns1, err1 := kClient.CreateNamespace(ns, annotations) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - - // Create sleep pod - var reqCPU int64 = 300 - var reqMem int64 = 300 - sleepPodConfigs := k8s.SleepPodConfig{NS: ns, Time: 60, CPU: reqCPU, Mem: reqMem} - sleepObj, podErr := k8s.InitSleepPod(sleepPodConfigs) - Ω(podErr).NotTo(HaveOccurred()) - By(fmt.Sprintf("Deploy the sleep app:%s to %s namespace", sleepObj.Name, ns)) - sleepRespPod, err = kClient.CreatePod(sleepObj, ns) - Ω(err).NotTo(HaveOccurred()) - Ω(kClient.WaitForPodRunning(sleepRespPod.Namespace, sleepRespPod.Name, time.Duration(60)*time.Second)).NotTo(HaveOccurred()) - }, - Entry("Verify_NS_With_Empty_Quota_Annotations", emptyOldAnnotations), - Entry("Verify_NS_With_Wrong_Quota_Annotations", wrongOldAnnotations), - ) - - // Hierarchical Queues - Quota enforcement - // For now, these cases are skipped - PIt("Verify_Child_Max_Resource_Must_Be_Less_Than_Parent_Max", func() { - // P2 case - addressed later - }) - - PIt("Verify_Sum_Of_All_Child_Resource_Must_Be_Less_Than_Parent_Max", func() { - // P2 case - addressed later - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(HaveOccurred()) - Ω(checks).To(Equal(""), checks) - - By("Tearing down namespace: " + ns) - err = kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go deleted file mode 100644 index 8a8203636..000000000 --- a/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package recoveryandrestart_test - -import ( - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestRecoveryAndRestart(t *testing.T) { - ginkgo.ReportAfterSuite("TestRecoveryAndRestart", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-recovery_and_restart_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(gomega.HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "RecoveryAndRestart Suite") -} - -var Ω = gomega.Ω diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_test.go deleted file mode 100644 index ad46323bf..000000000 --- a/test/e2e/recovery_and_restart/recovery_and_restart_test.go +++ /dev/null @@ -1,389 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package recoveryandrestart_test - -import ( - "fmt" - "runtime" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -const ( - gangSleepJobPrefix = "gang-sleep-job" - normalSleepJobPrefix = "normal-sleep-job" - taskGroupA = "groupa" - taskGroupB = "groupb" - taskGroupE2E = "e2e-task-group" - taskGroupE2EPrefix = "tg-" + gangSleepJobPrefix + "-" + taskGroupE2E - parallelism = 3 - taintKey = "e2e_test" -) - -var suiteName string -var kClient k8s.KubeCtl -var restClient yunikorn.RClient -var oldConfigMap = new(v1.ConfigMap) -var sleepRespPod *v1.Pod -var dev = "dev" + common.RandSeq(5) -var annotation = "ann-" + common.RandSeq(10) - -// Define sleepPod -var sleepPodConfigs = k8s.SleepPodConfig{Name: "sleepjob", NS: dev} -var sleepPod2Configs = k8s.SleepPodConfig{Name: "sleepjob2", NS: dev} - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) - - ginkgo.By("create development namespace") - ns1, err := kClient.CreateNamespace(dev, nil) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns1.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - - ginkgo.By("Deploy the sleep pod to the development namespace") - sleepObj, podErr := k8s.InitSleepPod(sleepPodConfigs) - Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod, err = kClient.CreatePod(sleepObj, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("applicationId=%s", sleepRespPod.ObjectMeta.Labels["applicationId"]), - 60) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart the scheduler pod") - yunikorn.RestartYunikorn(&kClient) - - ginkgo.By("Port-forward scheduler pod after restart") - yunikorn.RestorePortForwarding(&kClient) - - ginkgo.By("Deploy 2nd sleep pod to the development namespace") - sleepObj2, podErr := k8s.InitSleepPod(sleepPod2Configs) - Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod2, err := kClient.CreatePod(sleepObj2, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("applicationId=%s", sleepRespPod2.ObjectMeta.Labels["applicationId"]), - 60) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) -}) - -var _ = ginkgo.AfterSuite(func() { - ginkgo.By("Tear down namespace: " + dev) - err := kClient.TearDownNamespace(dev) - Ω(err).NotTo(gomega.HaveOccurred()) - - // call the healthCheck api to check scheduler health - ginkgo.By("Check Yunikorn's health") - checks, err2 := yunikorn.GetFailedHealthChecks() - Ω(err2).NotTo(gomega.HaveOccurred()) - Ω(checks).To(gomega.Equal(""), checks) - - ginkgo.By("Restoring the old config maps") - var c, err1 = kClient.GetConfigMaps(configmanager.YuniKornTestConfig.YkNamespace, - configmanager.DefaultYuniKornConfigMap) - Ω(err1).NotTo(gomega.HaveOccurred()) - Ω(c).NotTo(gomega.BeNil()) - c.Data = oldConfigMap.Data - var e, err3 = kClient.UpdateConfigMap(c, configmanager.YuniKornTestConfig.YkNamespace) - Ω(err3).NotTo(gomega.HaveOccurred()) - Ω(e).NotTo(gomega.BeNil()) -}) - -var _ = ginkgo.Describe("", func() { - - ginkgo.It("Verify_Pod_Alloc_Props", func() { - err := restClient.WaitForAppStateTransition("default", "root."+dev, sleepRespPod.ObjectMeta.Labels["applicationId"], "Starting", 30) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - appsInfo, err := restClient.GetAppInfo("default", "root."+dev, sleepRespPod.ObjectMeta.Labels["applicationId"]) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(appsInfo).NotTo(gomega.BeNil()) - ginkgo.By("Verify the pod allocation properties") - gomega.Ω(appsInfo).NotTo(gomega.BeNil()) - gomega.Ω(len(appsInfo.Allocations)).NotTo(gomega.BeZero()) - allocations := appsInfo.Allocations[0] - gomega.Ω(allocations).NotTo(gomega.BeNil()) - gomega.Ω(allocations.AllocationKey).NotTo(gomega.BeNil()) - gomega.Ω(allocations.NodeID).NotTo(gomega.BeNil()) - gomega.Ω(allocations.Partition).NotTo(gomega.BeNil()) - gomega.Ω(allocations.AllocationID).NotTo(gomega.BeNil()) - gomega.Ω(allocations.ApplicationID).To(gomega.Equal(sleepRespPod.ObjectMeta.Labels["applicationId"])) - core := sleepRespPod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue() - mem := sleepRespPod.Spec.Containers[0].Resources.Requests.Memory().Value() - resMap := allocations.ResourcePerAlloc - gomega.Ω(len(resMap)).NotTo(gomega.BeZero()) - gomega.Ω(resMap["memory"]).To(gomega.Equal(mem)) - gomega.Ω(resMap["vcore"]).To(gomega.Equal(core)) - }) - - ginkgo.It("Verify_SleepJobs_Restart_YK", func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - defer yunikorn.RestorePortForwarding(&kClient) - - appID1 := normalSleepJobPrefix + "-" + common.RandSeq(5) - sleepPodConfig1 := k8s.SleepPodConfig{Name: "normal-sleep-job", NS: dev, Time: 300, AppID: appID1} - pod1, podErr := k8s.InitSleepPod(sleepPodConfig1) - Ω(podErr).NotTo(gomega.HaveOccurred()) - appID2 := normalSleepJobPrefix + "-" + common.RandSeq(5) - sleepPodConfig2 := k8s.SleepPodConfig{Name: "normal-sleep-job-2", NS: dev, Time: 300, AppID: appID2} - pod2, podErr2 := k8s.InitSleepPod(sleepPodConfig2) - Ω(podErr2).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Submitting two normal sleep jobs") - job1 := k8s.InitTestJob(appID1, parallelism, parallelism, pod1) - _, createErr := kClient.CreateJob(job1, dev) - Ω(createErr).NotTo(gomega.HaveOccurred()) - defer kClient.DeleteWorkloadAndPods(job1.Name, k8s.Job, dev) - job2 := k8s.InitTestJob(appID2, parallelism, parallelism, pod2) - _, createErr2 := kClient.CreateJob(job2, dev) - defer kClient.DeleteWorkloadAndPods(job2.Name, k8s.Job, dev) - Ω(createErr2).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart the scheduler pod immediately") - yunikorn.RestartYunikorn(&kClient) - - ginkgo.By("Listing pods") - pods, err := kClient.GetPods(dev) - Ω(err).NotTo(gomega.HaveOccurred()) - fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in namespace %s: %d\n", - dev, len(pods.Items)) - for _, pod := range pods.Items { - fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name: %-40s\tStatus: %s\n", pod.GetName(), pod.Status.Phase) - } - - ginkgo.By("Waiting for sleep pods to be running") - err = kClient.WaitForJobPodsRunning(dev, job1.Name, parallelism, 60*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - err = kClient.WaitForJobPodsRunning(dev, job2.Name, parallelism, 60*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("Verify_GangScheduling_TwoGangs_Restart_YK", func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - defer yunikorn.RestorePortForwarding(&kClient) - - appID := gangSleepJobPrefix + "-" + common.RandSeq(5) - sleepPodConfig := k8s.SleepPodConfig{Name: "gang-sleep-job", NS: dev, Time: 1, AppID: appID} - taskGroups := k8s.InitTaskGroups(sleepPodConfig, taskGroupA, taskGroupB, parallelism) - pod, podErr := k8s.InitSleepPod(sleepPodConfig) - Ω(podErr).NotTo(gomega.HaveOccurred()) - pod = k8s.DecoratePodForGangScheduling(30, "Soft", taskGroupA, - taskGroups, pod) - - ginkgo.By("Submitting gang sleep job") - job := k8s.InitTestJob(appID, parallelism, parallelism, pod) - _, err := kClient.CreateJob(job, dev) - Ω(err).NotTo(gomega.HaveOccurred()) - defer kClient.DeleteWorkloadAndPods(job.Name, k8s.Job, dev) - - ginkgo.By("Waiting job pods to be created") - createErr := kClient.WaitForJobPodsCreated(dev, job.Name, parallelism, 30*time.Second) - Ω(createErr).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Waiting for placeholders in task group A (expected state: Running)") - groupAPrefix := "tg-" + appID + "-" + taskGroupA + "-" - stateRunning := v1.PodRunning - err = kClient.WaitForPlaceholders(dev, groupAPrefix, parallelism, 30*time.Second, &stateRunning) - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Waiting for placeholders in task group B (expected state: Pending)") - groupBPrefix := "tg-" + appID + "-" + taskGroupB + "-" - statePending := v1.PodPending - err = kClient.WaitForPlaceholders(dev, groupBPrefix, parallelism+1, 30*time.Second, &statePending) - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart the scheduler pod") - yunikorn.RestartYunikorn(&kClient) - - // make sure that Yunikorn's internal state have been properly restored - ginkgo.By("Submit sleep job") - sleepJob3AppID := "sleepjob-" + common.RandSeq(5) - sleepPod3Configs := k8s.SleepPodConfig{Name: "sleepjob3", NS: dev, AppID: sleepJob3AppID} - sleepPod, podErr2 := k8s.InitSleepPod(sleepPod3Configs) - Ω(podErr2).NotTo(gomega.HaveOccurred()) - sleepRespPod, err = kClient.CreatePod(sleepPod, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("applicationId=%s", sleepJob3AppID), - 60) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - // After YK is up again, we expect to see the same pods in the same state - ginkgo.By("Verify the number of pods & their status") - pods, err2 := kClient.GetPods(dev) - Ω(err2).NotTo(gomega.HaveOccurred()) - var groupAPlaceholderCount int - var groupBPlaceholderCount int - var jobPodCount int - fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in namespace %s: %d\n", - dev, len(pods.Items)) - for _, pod := range pods.Items { - podPhase := pod.Status.Phase - podName := pod.GetName() - fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name: %-40s\tStatus: %s\n", podName, podPhase) - if strings.HasPrefix(podName, groupAPrefix) { - groupAPlaceholderCount++ - Ω(podPhase).To(gomega.Equal(v1.PodRunning)) - continue - } - if strings.HasPrefix(podName, groupBPrefix) { - groupBPlaceholderCount++ - Ω(podPhase).To(gomega.Equal(v1.PodPending)) - continue - } - if strings.HasPrefix(podName, gangSleepJobPrefix) { - jobPodCount++ - Ω(podPhase).To(gomega.Equal(v1.PodPending)) - continue - } - } - Ω(groupAPlaceholderCount).To(gomega.Equal(parallelism)) - Ω(groupBPlaceholderCount).To(gomega.Equal(parallelism + 1)) - - // Wait for placeholder timeout & replacement to real pod - ginkgo.By("Waiting for placeholder timeout & sleep pods to finish") - err = kClient.WaitForJobPodsSucceeded(dev, job.Name, parallelism, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("Verify_GangScheduling_PendingPlaceholders_Restart_YK", func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - defer yunikorn.RestorePortForwarding(&kClient) - - ginkgo.By("Trying to find an available worker node") - nodes, err := kClient.GetNodes() - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(len(nodes.Items) >= 2).Should(gomega.Equal(true), "Not enough nodes in the cluster, need at least 2") - - var workerResource *resource.Quantity - masterPresent := false - var selectedNode string - var nodesToTaint []string - for _, node := range nodes.Items { - // skip master if it's marked as such - node := node - if k8s.IsMasterNode(&node) { - masterPresent = true - continue - } - - if selectedNode == "" { - workerResource = node.Status.Allocatable.Memory() - selectedNode = node.Name - } else { - nodesToTaint = append(nodesToTaint, node.Name) - } - } - - memoryGiB := workerResource.ScaledValue(resource.Giga) - placeholderCount := int32(memoryGiB/3 + 2) // get a reasonable number to have both Running/Pending PH pods - fmt.Fprintf(ginkgo.GinkgoWriter, "%s allocatable memory in GiB = %d, number of placeholders to use = %d\n", - selectedNode, memoryGiB, placeholderCount) - - ginkgo.By("Tainting all nodes except " + selectedNode) - err = kClient.TaintNodes(nodesToTaint, taintKey, "value", v1.TaintEffectNoSchedule) - Ω(err).NotTo(gomega.HaveOccurred()) - - removeTaint := true - defer func() { - if removeTaint { - ginkgo.By("Untainting nodes (defer)") - err = kClient.UntaintNodes(nodesToTaint, taintKey) - Ω(err).NotTo(gomega.HaveOccurred(), "Could not remove taint from nodes "+strings.Join(nodesToTaint, ",")) - } - }() - - ginkgo.By("Submitting gang job") - appID := gangSleepJobPrefix + "-" + common.RandSeq(5) - sleepPodConfig := k8s.SleepPodConfig{Name: "gang-sleep-job", NS: dev, Time: 1, AppID: appID, Mem: 3000, CPU: 10} - taskGroups := k8s.InitTaskGroup(sleepPodConfig, taintKey, placeholderCount) - pod, podErr := k8s.InitSleepPod(sleepPodConfig) - Ω(podErr).NotTo(gomega.HaveOccurred()) - pod = k8s.DecoratePodForGangScheduling(900, "Soft", taskGroupE2E, - taskGroups, pod) - job := k8s.InitTestJob(appID, 1, 1, pod) - _, createErr := kClient.CreateJob(job, dev) - Ω(createErr).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Waiting for placeholders to be Running/Pending") - err = kClient.WaitForPlaceholdersStableState(dev, taskGroupE2EPrefix, 30*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart the scheduler pod and update tolerations if needed") - includeMaster := masterPresent && len(nodes.Items) == 2 - // YK can be scheduled on the master only if there are 2 nodes in the cluster - newTolerations := getSchedulerPodTolerations(includeMaster) - yunikorn.RestartYunikornAndAddTolerations(&kClient, true, newTolerations) - - ginkgo.By("Untainting nodes") - removeTaint = false - err = kClient.UntaintNodes(nodesToTaint, taintKey) - Ω(err).NotTo(gomega.HaveOccurred(), "Could not remove taint from nodes "+strings.Join(nodesToTaint, ",")) - - ginkgo.By("Waiting for placeholder replacement & sleep pods to finish") - err = kClient.WaitForJobPodsSucceeded(dev, job.Name, 1, 60*time.Second) - Ω(err).NotTo(gomega.HaveOccurred()) - }) -}) - -func getSchedulerPodTolerations(includeMaster bool) []v1.Toleration { - newTolerations := make([]v1.Toleration, 0) - if includeMaster { - for key := range common.MasterTaints { - t := v1.Toleration{ - Key: key, - Effect: v1.TaintEffectNoSchedule, - Operator: v1.TolerationOpEqual, - } - newTolerations = append(newTolerations, t) - } - } - - t := v1.Toleration{ - Key: taintKey, - Effect: v1.TaintEffectNoSchedule, - Operator: v1.TolerationOpEqual, - } - newTolerations = append(newTolerations, t) - return newTolerations -} diff --git a/test/e2e/resource_fairness/resource_fairness_suite_test.go b/test/e2e/resource_fairness/resource_fairness_suite_test.go deleted file mode 100644 index 7c5179781..000000000 --- a/test/e2e/resource_fairness/resource_fairness_suite_test.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package resourcefairness_test - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) -var kClient = k8s.KubeCtl{} //nolint -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - Ω(kClient.SetClient()).To(BeNil()) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -func TestResourceFairness(t *testing.T) { - ginkgo.ReportAfterSuite("TestResourceFairness", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-resource_fairness_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "Resource Fairness", ginkgo.Label("Resource Fairness")) -} - -// Declarations for Ginkgo DSL -var Describe = ginkgo.Describe -var PIt = ginkgo.PIt -var It = ginkgo.It -var By = ginkgo.By -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/resource_fairness/resource_fairness_test.go b/test/e2e/resource_fairness/resource_fairness_test.go deleted file mode 100644 index f003bc4fc..000000000 --- a/test/e2e/resource_fairness/resource_fairness_test.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package resourcefairness_test - -import ( - "context" - "fmt" - "math/rand" - "time" - - "github.com/onsi/ginkgo/v2" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/apache/yunikorn-core/pkg/common/configs" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var _ = Describe("FairScheduling:", func() { - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var ns string - var queuePath string - - var maxCPU int64 = 500 - var maxMem int64 = 500 - - BeforeEach(func() { - var namespace *v1.Namespace - ns = "test-" + common.RandSeq(10) - queuePath = "root." + ns - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - - By("Setting custom YuniKorn configuration") - annotation = "ann-" + common.RandSeq(10) - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "fair", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{ - Name: ns, - Parent: false, - Resources: configs.Resources{Max: map[string]string{"vcore": "500m", "memory": "500M"}, - Guaranteed: map[string]string{"vcore": "500m", "memory": "500M"}}, - Properties: map[string]string{"application.sort.policy": "fair"}, - }); err != nil { - return err - } - return nil - }) - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - - // Restart yunikorn and port-forward - // Required to change node sort policy. - ginkgo.By("Restart the scheduler pod") - yunikorn.RestartYunikorn(&kClient) - - ginkgo.By("Port-forward scheduler pod after restart") - yunikorn.RestorePortForwarding(&kClient) - - By(fmt.Sprintf("Creating test namespace %s", ns)) - namespace, err = kClient.CreateNamespace(ns, map[string]string{"vcore": "500m", "memory": "500M"}) - Ω(err).ShouldNot(HaveOccurred()) - Ω(namespace.Status.Phase).Should(Equal(v1.NamespaceActive)) - }) - - // Validates waitQueue order of requested app resources, according to fairAppScheduling. - // Step 1: Deploy 4 apps, which sum to 95% queueCPU / 35% queueMem - // -> Resource priority order: 1)CPU. 2)Mem - // Step 2: Deploy 1 more blocked pod each for apps0-2 - // Step 3: Kill an App3 pod. - // Step 4: App with least cpu use is allocated pod next. Break tie using mem. - It("Test_Wait_Queue_Order", func() { - // Create appIDs - var apps []string - for j := 0; j < 4; j++ { - id := fmt.Sprintf("app%d-%s", j, common.RandSeq(5)) - apps = append(apps, id) - } - - // Initial allocation to fill ns quota - appPods := map[string][]string{} - appAllocs := map[string]map[string]float64{ - apps[0]: {"cpu": 0.3, "mem": 0.1, "pods": 1}, - apps[1]: {"cpu": 0.2, "mem": 0.05, "pods": 1}, - apps[2]: {"cpu": 0.1, "mem": 0.15, "pods": 1}, - apps[3]: {"cpu": 0.4, "mem": 0.05, "pods": 4}, - } - for appID, req := range appAllocs { - appPods[appID] = []string{} - numPods := int(req["pods"]) - By(fmt.Sprintf("[%s] Deploy %d pods", appID, numPods)) - for i := 0; i < numPods; i++ { - // Calculate individual podRequest by dividing total appAlloc by numPods - reqCPU := int64(req["cpu"] / req["pods"] * float64(maxCPU)) - reqMem := int64(req["mem"] / req["pods"] * float64(maxMem)) - podName := fmt.Sprintf("%s-pod%d", appID, i) - appPods[appID] = append(appPods[appID], podName) - - // Deploy pod - sleepPodConf := k8s.SleepPodConfig{ - Name: podName, NS: ns, AppID: appID, CPU: reqCPU, Mem: reqMem, Labels: map[string]string{ - constants.LabelQueueName: queuePath}} - initPod, podErr := k8s.InitSleepPod(sleepPodConf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - err = kClient.WaitForPodRunning(ns, podName, 120*time.Second) - Ω(err).NotTo(HaveOccurred()) - } - } - - // Submit one blocked pod for each app in random order. Each requests 0.1 qCPU and 0.05 qMem. - By("Submitting additional blocked pod for each app") - randOrder := rand.Perm(3) - for _, i := range randOrder { - cpuPct, memPct, appID := 0.1, 0.05, apps[i] - reqCPU, reqMem := int64(cpuPct*float64(maxCPU)), int64(memPct*float64(maxMem)) - podNum := len(appPods[appID]) - podName := fmt.Sprintf("%s-pod%d", appID, podNum) - appPods[appID] = append(appPods[appID], podName) - - By(fmt.Sprintf("[%s] Submit %s", appID, podName)) - sleepPodConf := k8s.SleepPodConfig{ - Name: podName, NS: ns, AppID: appID, CPU: reqCPU, Mem: reqMem, Labels: map[string]string{ - constants.LabelQueueName: queuePath}} - initPod, podErr := k8s.InitSleepPod(sleepPodConf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - err = kClient.WaitForPodPending(ns, podName, 10*time.Second) - Ω(err).NotTo(HaveOccurred()) - - // Wait till requests has been added to application - err := wait.PollUntilContextTimeout(context.TODO(), 300*time.Millisecond, 60*time.Second, false, func(context.Context) (bool, error) { - app, err := restClient.GetAppInfo("default", queuePath, appID) - if err != nil { - return false, nil - } - if len(app.Requests) == 1 { - return true, nil - } - return false, nil - }) - Ω(err).NotTo(HaveOccurred()) - } - - // Log correct app priority and pod wait order - appOrder := []string{apps[2], apps[1], apps[0]} - var waitOrder []string - for _, appID := range appOrder { - l := len(appPods[appID]) - waitOrder = append(waitOrder, appPods[appID][l-1]) - } - - // Verify wait order by releasing app3 pod. Then check for correct pod running. - for _, podName := range waitOrder { - // Delete app3 pod to release resource - app3Pods, l := appPods[apps[3]], len(appPods[apps[3]]) - pod := app3Pods[l-1] - appPods[apps[3]] = app3Pods[:l-1] - - By(fmt.Sprintf("Delete %s", pod)) - err := kClient.DeletePod(pod, ns) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Verify %s is now running", podName)) - err = kClient.WaitForPodRunning(ns, podName, 120*time.Second) - Ω(err).NotTo(HaveOccurred()) - } - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By("Tear down namespace: " + ns) - err := kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/simple_preemptor/simple_preemptor_suite_test.go b/test/e2e/simple_preemptor/simple_preemptor_suite_test.go deleted file mode 100644 index 20dd27eba..000000000 --- a/test/e2e/simple_preemptor/simple_preemptor_suite_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package simple_preemptor_test - -import ( - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestSimplePreemptor(t *testing.T) { - ginkgo.ReportAfterSuite("TestSimplePreemptor", func(report ginkgo.Report) { - err := reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-simple_preemptor_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestSimplePreemptor", ginkgo.Label("TestSimplePreemptor")) -} - -var Ω = gomega.Ω -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/simple_preemptor/simple_preemptor_test.go b/test/e2e/simple_preemptor/simple_preemptor_test.go deleted file mode 100644 index 8f551f686..000000000 --- a/test/e2e/simple_preemptor/simple_preemptor_test.go +++ /dev/null @@ -1,247 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package simple_preemptor_test - -import ( - "fmt" - "runtime" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var suiteName string -var kClient k8s.KubeCtl -var restClient yunikorn.RClient -var ns *v1.Namespace -var dev = "dev" + common.RandSeq(5) -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) - -// Nodes -var Worker1 = "" -var Worker2 = "" -var Worker1Res = resource.NewQuantity(8*1000*1000, resource.DecimalSI) -var Worker2Res = resource.NewQuantity(8*1000*1000, resource.DecimalSI) -var sleepPodMemLimit1 int64 -var sleepPodMemLimit2 int64 -var taintKey = "e2e_test_simple_preemptor" -var nodesToTaint []string - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - Ω(restClient).NotTo(gomega.BeNil()) - - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "", annotation) - - ginkgo.By("Port-forward the scheduler pod") - var err = kClient.PortForwardYkSchedulerPod() - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("create development namespace") - ns, err = kClient.CreateNamespace(dev, nil) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - - var nodes *v1.NodeList - nodes, err = kClient.GetNodes() - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(len(nodes.Items)).NotTo(gomega.BeZero(), "Events cant be empty") - - // Extract node allocatable resources - for _, node := range nodes.Items { - // skip master if it's marked as such - node := node - if k8s.IsMasterNode(&node) || !k8s.IsComputeNode(&node) { - continue - } - if Worker1 == "" { - Worker1 = node.Name - Worker1Res = node.Status.Allocatable.Memory() - } else if Worker2 == "" { - Worker2 = node.Name - Worker2Res = node.Status.Allocatable.Memory() - } else { - nodesToTaint = append(nodesToTaint, node.Name) - } - } - ginkgo.By("Worker1:" + Worker1) - ginkgo.By("Worker2:" + Worker2) - - ginkgo.By("Tainting some nodes..") - err = kClient.TaintNodes(nodesToTaint, taintKey, "value", v1.TaintEffectNoSchedule) - Ω(err).NotTo(gomega.HaveOccurred()) - - var pods *v1.PodList - totalPodQuantity1 := *resource.NewQuantity(0, resource.DecimalSI) - totalPodQuantity2 := *resource.NewQuantity(0, resource.DecimalSI) - pods, err = kClient.GetPods("yunikorn") - if err == nil { - for _, pod := range pods.Items { - for _, c := range pod.Spec.Containers { - if pod.Spec.NodeName == Worker1 { - totalPodQuantity1.Add(*resource.NewQuantity(c.Resources.Requests.Memory().Value(), resource.DecimalSI)) - } else if pod.Spec.NodeName == Worker2 { - totalPodQuantity2.Add(*resource.NewQuantity(c.Resources.Requests.Memory().Value(), resource.DecimalSI)) - } - } - } - } - Worker1Res.Sub(totalPodQuantity1) - sleepPodMemLimit1 = int64(float64(Worker1Res.Value())/3.5) / (1000 * 1000) - Worker2Res.Sub(totalPodQuantity2) - sleepPodMemLimit2 = int64(float64(Worker2Res.Value())/3.5) / (1000 * 1000) -}) - -var _ = ginkgo.AfterSuite(func() { - - ginkgo.By("Untainting some nodes") - err := kClient.UntaintNodes(nodesToTaint, taintKey) - Ω(err).NotTo(gomega.HaveOccurred(), "Could not remove taint from nodes "+strings.Join(nodesToTaint, ",")) - - ginkgo.By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(checks).To(gomega.Equal(""), checks) - ginkgo.By("Tearing down namespace: " + ns.Name) - err = kClient.TearDownNamespace(ns.Name) - Ω(err).NotTo(gomega.HaveOccurred()) - - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -var _ = ginkgo.Describe("SimplePreemptor", func() { - ginkgo.It("Verify_basic_simple_preemption", func() { - // Use case: Only one pod is running and same pod has been selected as victim - // Define sleepPod - sleepPodConfigs := k8s.SleepPodConfig{Name: "sleepjob", NS: dev, Mem: sleepPodMemLimit1 * 2, Time: 600, RequiredNode: Worker1} - sleepPod2Configs := k8s.SleepPodConfig{Name: "sleepjob2", NS: dev, Mem: sleepPodMemLimit2 * 2, Time: 600} - sleepPod3Configs := k8s.SleepPodConfig{Name: "sleepjob3", NS: dev, RequiredNode: Worker2, Mem: sleepPodMemLimit2 * 2, Time: 600} - - for _, config := range []k8s.SleepPodConfig{sleepPodConfigs, sleepPod2Configs, sleepPod3Configs} { - ginkgo.By("Deploy the sleep pod " + config.Name + " to the development namespace") - sleepObj, podErr := k8s.InitSleepPod(config) - Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod, err := kClient.CreatePod(sleepObj, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), - 60) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - } - // assert sleeppod2 is killed - err := kClient.WaitForPodEvent(dev, "sleepjob2", "Killing", 60*time.Second) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("Verify_simple_preemption", func() { - // Use case: When 3 sleep pods (2 opted out, regular) are running, regular pod should be victim to free up resources for 4th sleep pod - // Define sleepPod - sleepPodConfigs := k8s.SleepPodConfig{Name: "sleepjob", NS: dev, Mem: sleepPodMemLimit1, Time: 600, RequiredNode: Worker1} - sleepPod2Configs := k8s.SleepPodConfig{Name: "sleepjob2", NS: dev, Mem: sleepPodMemLimit1, Time: 600, RequiredNode: Worker1} - sleepPod3Configs := k8s.SleepPodConfig{Name: "sleepjob3", NS: dev, Mem: sleepPodMemLimit1, Time: 600, RequiredNode: Worker1, AppID: "test01"} - - // add the same AppID to sleepPod3Configs so that sleepPod4Configs is not the originator pod - sleepPod4Configs := k8s.SleepPodConfig{Name: "sleepjob4", NS: dev, Mem: sleepPodMemLimit2, Time: 600, Optedout: k8s.Allow, AppID: "test01"} - sleepPod5Configs := k8s.SleepPodConfig{Name: "sleepjob5", NS: dev, Mem: sleepPodMemLimit2, Time: 600, Optedout: k8s.Deny} - sleepPod6Configs := k8s.SleepPodConfig{Name: "sleepjob6", NS: dev, Mem: sleepPodMemLimit2, Time: 600, Optedout: k8s.Deny} - sleepPod7Configs := k8s.SleepPodConfig{Name: "sleepjob7", NS: dev, Mem: sleepPodMemLimit2, Time: 600, RequiredNode: Worker2} - - for _, config := range []k8s.SleepPodConfig{sleepPodConfigs, sleepPod2Configs, - sleepPod3Configs, sleepPod4Configs, sleepPod5Configs, sleepPod6Configs} { - ginkgo.By("Deploy the sleep pod " + config.Name + " to the development namespace") - sleepObj, podErr := k8s.InitSleepPod(config) - - Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod, err := kClient.CreatePod(sleepObj, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), - 240) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Deploy the sleep pod " + sleepPod7Configs.Name + " to the development namespace") - sleepObj, podErr := k8s.InitSleepPod(sleepPod7Configs) - - Ω(podErr).NotTo(gomega.HaveOccurred()) - sleepRespPod, err := kClient.CreatePod(sleepObj, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), - 600) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - // assert sleeppod4 is killed - err = kClient.WaitForPodEvent(dev, "sleepjob4", "Killing", 60*time.Second) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - }) - - ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name}) - - // Delete all sleep pods - ginkgo.By("Delete all sleep pods") - pods, err := kClient.GetPodNamesFromNS(ns.Name) - if err == nil { - for _, each := range pods { - if strings.Contains(each, "sleep") { - ginkgo.By("Deleting sleep pod: " + each) - err = kClient.DeletePod(each, ns.Name) - if err != nil { - if statusErr, ok := err.(*k8serrors.StatusError); ok { - if statusErr.ErrStatus.Reason == metav1.StatusReasonNotFound { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pod %s - reason is %s, it "+ - "has been deleted in the meantime\n", each, statusErr.ErrStatus.Reason) - continue - } - } - } - } - } - } else { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to get pods from namespace %s - reason is %s\n", ns.Name, err.Error()) - } - }) -}) diff --git a/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_suite_test.go b/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_suite_test.go deleted file mode 100644 index a1968ffb7..000000000 --- a/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_suite_test.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package spark_jobs_scheduling - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation = "ann-" + common.RandSeq(10) -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "stateaware", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -func TestSparkJobs(t *testing.T) { - ginkgo.ReportAfterSuite("TestSparkJobs", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-spark_jobs_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestSparkJobs", ginkgo.Label("TestSparkJobs")) -} - -// Declarations for Ginkgo DSL -var Fail = ginkgo.Fail -var Describe = ginkgo.Describe -var It = ginkgo.It -var By = ginkgo.By -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite - -// Declarations for Gomega DSL -var RegisterFailHandler = gomega.RegisterFailHandler - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred -var BeEmpty = gomega.BeEmpty diff --git a/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_test.go b/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_test.go deleted file mode 100644 index 8dc4821a2..000000000 --- a/test/e2e/spark_jobs_scheduling/spark_jobs_scheduling_test.go +++ /dev/null @@ -1,176 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package spark_jobs_scheduling - -import ( - "context" - "fmt" - "net/url" - "os" - "os/exec" - "sort" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/rest" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var _ = Describe("", func() { - - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var sparkNS = "spark-" + common.RandSeq(10) - var svcAcc = "svc-acc-" + common.RandSeq(10) - var config *rest.Config - var masterURL string - var roleName = "spark-jobs-role-" + common.RandSeq(5) - var clusterEditRole = "edit" - var sparkImage = os.Getenv("SPARK_PYTHON_IMAGE") - var sparkHome = os.Getenv("SPARK_HOME") - if sparkHome == "" { - sparkHome = "/usr/local/" - } - var sparkExecutorCount = 3 - - BeforeEach(func() { - By(fmt.Sprintf("Spark image is: %s", sparkImage)) - Ω(sparkImage).NotTo(BeEmpty()) - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - Ω(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Creating namespace: %s for spark jobs", sparkNS)) - ns1, err := kClient.CreateNamespace(sparkNS, nil) - Ω(err).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - - By(fmt.Sprintf("Creating service account: %s under namespace: %s", svcAcc, sparkNS)) - _, err = kClient.CreateServiceAccount(svcAcc, sparkNS) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Creating cluster role binding: %s for spark jobs", roleName)) - _, err = kClient.CreateClusterRoleBinding(roleName, clusterEditRole, sparkNS, svcAcc) - Ω(err).NotTo(HaveOccurred()) - - config, err = kClient.GetKubeConfig() - Ω(err).NotTo(HaveOccurred()) - - u, err := url.Parse(config.Host) - Ω(err).NotTo(HaveOccurred()) - port := u.Port() - if port == "" { - port = "443" - if u.Scheme == "http" { - port = "80" - } - } - masterURL = u.Scheme + "://" + u.Hostname() + ":" + port - By(fmt.Sprintf("MasterURL info is %s ", masterURL)) - }) - - It("Test_With_Spark_Jobs", func() { - By("Submit the spark jobs") - err := exec.Command( - "bash", - "../testdata/spark_jobs.sh", - masterURL, - sparkImage, - sparkNS, - svcAcc, - string(rune(sparkExecutorCount))).Run() - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("[%s] wait for root queue to appear\n", time.Now().Format("01-02-2006 15:04:05"))) - err = restClient.WaitforQueueToAppear(configmanager.DefaultPartition, configmanager.RootQueue, 120) - Ω(err).NotTo(HaveOccurred()) - - sparkQueueName := configmanager.RootQueue + "." + sparkNS - By(fmt.Sprintf("[%s] wait for %s queue to appear\n", time.Now().Format("01-02-2006 15:04:05"), - sparkQueueName)) - err = restClient.WaitforQueueToAppear(configmanager.DefaultPartition, sparkQueueName, 120) - Ω(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Get apps from specific queue: %s", sparkNS)) - var appsFromQueue []*dao.ApplicationDAOInfo - // Poll for apps to appear in the queue - err = wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*100, time.Duration(120)*time.Second, false, func(context.Context) (done bool, err error) { - appsFromQueue, err = restClient.GetApps(configmanager.DefaultPartition, configmanager.RootQueue+"."+sparkNS) - if len(appsFromQueue) == 3 { - return true, nil - } - return false, err - }) - Ω(err).NotTo(HaveOccurred()) - Ω(appsFromQueue).NotTo(BeEmpty()) - - // Sort by submission time oldest first - sort.SliceStable(appsFromQueue, func(i, j int) bool { - l := appsFromQueue[i] - r := appsFromQueue[j] - return l.SubmissionTime < r.SubmissionTime - }) - Ω(appsFromQueue).NotTo(BeEmpty()) - - var appIds []string - for _, each := range appsFromQueue { - appIds = append(appIds, each.ApplicationID) - } - By(fmt.Sprintf("Apps submitted are: %s", appIds)) - - // Verify that all the spark jobs are scheduled and are in running state. - for _, id := range appIds { - By(fmt.Sprintf("Verify driver pod for application %s has been created.", id)) - err = kClient.WaitForPodBySelector(sparkNS, fmt.Sprintf("spark-app-selector=%s, spark-role=driver", id), 180*time.Second) - Ω(err).ShouldNot(HaveOccurred()) - - By(fmt.Sprintf("Verify driver pod for application %s was completed.", id)) - err = kClient.WaitForPodBySelectorSucceeded(sparkNS, fmt.Sprintf("spark-app-selector=%s, spark-role=driver", id), 360*time.Second) - Ω(err).NotTo(HaveOccurred()) - } - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{sparkNS}) - - By("Killing all spark jobs") - // delete the Spark pods one by one - err = kClient.DeletePods(sparkNS) - Ω(err).NotTo(HaveOccurred()) - - By("Deleting cluster role bindings ") - err = kClient.DeleteClusterRoleBindings(roleName) - Ω(err).NotTo(HaveOccurred()) - - By("Deleting service account") - err = kClient.DeleteServiceAccount(svcAcc, sparkNS) - Ω(err).NotTo(HaveOccurred()) - - By("Deleting test namespaces") - err = kClient.DeleteNamespace(sparkNS) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/state_aware_app_scheduling/drip_feed_schedule_test.go b/test/e2e/state_aware_app_scheduling/drip_feed_schedule_test.go deleted file mode 100644 index 4016b0eb5..000000000 --- a/test/e2e/state_aware_app_scheduling/drip_feed_schedule_test.go +++ /dev/null @@ -1,153 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package stateawareappscheduling_test - -import ( - "context" - "fmt" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var _ = Describe("DripFeedSchedule:", func() { - - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var app1 = "app01-" + common.RandSeq(5) - var app2 = "app02-" + common.RandSeq(5) - var app3 = "app03-" + common.RandSeq(5) - var ns = "sleep-" + common.RandSeq(10) - var testTimeout float64 = 360 - var running = yunikorn.States().Application.Running - var accepted = yunikorn.States().Application.Accepted - var starting = yunikorn.States().Application.Starting - - BeforeEach(func() { - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - By(fmt.Sprintf("Creating namespace: %s for sleep jobs", ns)) - var ns1, err1 = kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - It("Test_State_Aware_App_Sorting", func() { - By("Submit 3 apps(app01, app02, app03) with one pod each") - for _, appID := range []string{app1, app2, app3} { - podName := "pod1-" + common.RandSeq(5) - sleepPodConf := k8s.SleepPodConfig{Name: podName, NS: ns, AppID: appID} - initPod, podErr := k8s.InitSleepPod(sleepPodConf) - Ω(podErr).NotTo(HaveOccurred()) - Ω(err).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - time.Sleep(3 * time.Second) // Buffer time between pod submission - } - - By(fmt.Sprintf("Get apps from specific queue: %s", ns)) - var appsFromQueue []*dao.ApplicationDAOInfo - // Poll for apps to appear in the queue - err = wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Duration(60)*time.Second, false, func(context.Context) (done bool, err error) { - appsFromQueue, err = restClient.GetApps("default", "root."+ns) - if len(appsFromQueue) == 3 { - return true, nil - } - return false, err - }) - Ω(err).NotTo(HaveOccurred()) - Ω(appsFromQueue).NotTo(BeEmpty()) - - /* - At this point, the apps state will be - app01 - Starting - app02 - Accepted - app03 - Accepted - */ - By(fmt.Sprintf("Verify the app %s are in Starting state", app1)) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app1, starting, 60) - Ω(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Verify the app %s are in Accepted state", app2)) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app2, accepted, 60) - Ω(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Verify the app %s are in Accepted state", app3)) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app3, accepted, 60) - Ω(err).NotTo(HaveOccurred()) - - /* - Add another pod for app01, and once this pod is allocated, verify app states: - app01 - Running => pod1, pod2 - app02 - Starting => pod1 - app03 - Accepted => pod1 - - Add another pod for app02, and once this pod is allocated, verify app states: - app01 - Running => pod1, pod2 - app02 - Running => pod1, pod2 - app03 - starting => pod1 - - Add another pod for app03, and once this pod is allocated, verify app states: - app01 - Running => pod1, pod2 - app02 - Running => pod1, pod2 - app03 - Running => pod1, pod2 - */ - var appStates = map[string][]string{ - app1: {running, starting, accepted}, - app2: {running, running, starting}, - app3: {running, running, running}, - } - for _, appID := range []string{app1, app2, app3} { - By(fmt.Sprintf("Add one more pod to the app: %s", appID)) - podName := "pod2-" + common.RandSeq(5) - sleepPodConf := k8s.SleepPodConfig{Name: podName, NS: ns, AppID: appID} - initPod, podErr := k8s.InitSleepPod(sleepPodConf) - Ω(podErr).NotTo(HaveOccurred()) - _, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Verify that the app: %s is in running state", appID)) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app1, appStates[appID][0], 60) - Ω(err).NotTo(HaveOccurred()) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app2, appStates[appID][1], 60) - Ω(err).NotTo(HaveOccurred()) - err = restClient.WaitForAppStateTransition("default", "root."+ns, app3, appStates[appID][2], 60) - Ω(err).NotTo(HaveOccurred()) - } - - }, testTimeout) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(HaveOccurred()) - Ω(checks).To(Equal(""), checks) - - By("Tearing down namespace: " + ns) - err = kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/state_aware_app_scheduling/fallback_test.go b/test/e2e/state_aware_app_scheduling/fallback_test.go deleted file mode 100644 index bfa9a8f2f..000000000 --- a/test/e2e/state_aware_app_scheduling/fallback_test.go +++ /dev/null @@ -1,118 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package stateawareappscheduling_test - -import ( - "fmt" - - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-core/pkg/webservice/dao" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -var _ = Describe("FallbackTest:", func() { - var kClient k8s.KubeCtl - var restClient yunikorn.RClient - var err error - var sleepRespPod *v1.Pod - var ns string - var appsInfo *dao.ApplicationDAOInfo - - BeforeEach(func() { - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - ns = "test-" + common.RandSeq(10) - By(fmt.Sprintf("create %s namespace", ns)) - ns1, err1 := kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - - By(fmt.Sprintf("Deploy the sleep pod to %s namespace", ns)) - sleepPodConf := k8s.SleepPodConfig{Name: "sleepjob", NS: ns, Time: 600} - initPod, podErr := k8s.InitSleepPod(sleepPodConf) - Ω(podErr).NotTo(HaveOccurred()) - sleepRespPod, err = kClient.CreatePod(initPod, ns) - Ω(err).NotTo(HaveOccurred()) - // Wait for pod to move to running state - }) - - It("Verify_App_In_Starting_State", func() { - err = kClient.WaitForPodBySelectorRunning(ns, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), - 10) - Ω(err).NotTo(HaveOccurred()) - - appsInfo, err = restClient.GetAppInfo("default", "root."+ns, sleepRespPod.ObjectMeta.Labels["applicationId"]) - Ω(err).NotTo(HaveOccurred()) - Ω(appsInfo).NotTo(BeNil()) - By(fmt.Sprintf("Verify that the sleep pod is mapped to %s queue", ns)) - Ω(appsInfo.ApplicationID).To(Equal(sleepRespPod.ObjectMeta.Labels["applicationId"])) - Ω(appsInfo.QueueName).To(ContainSubstring(sleepRespPod.ObjectMeta.Namespace)) - By("Verify that the job is scheduled by YuniKorn & is in starting state") - Ω(appsInfo.State).To(Equal("Starting")) - Ω("yunikorn").To(Equal(sleepRespPod.Spec.SchedulerName)) - }) - - It("Verify_App_State_Transition_To_Running_Post_Timeout", func() { - By("Wait for fallback timeout of 5mins") - err = restClient.WaitForAppStateTransition("default", "root."+ns, sleepRespPod.ObjectMeta.Labels["applicationId"], - yunikorn.States().Application.Running, - 360) - Ω(err).NotTo(HaveOccurred()) - - // Get AppInfo again to check the allocations post running state. - appsInfo, err = restClient.GetAppInfo("default", "root."+ns, sleepRespPod.ObjectMeta.Labels["applicationId"]) - Ω(appsInfo.Allocations).NotTo(BeNil()) - Ω(len(appsInfo.Allocations)).NotTo(gomega.BeZero()) - allocation := appsInfo.Allocations[0] - Ω(allocation).NotTo(gomega.BeNil()) - Ω(allocation.AllocationKey).NotTo(BeNil()) - Ω(allocation.NodeID).NotTo(BeNil()) - Ω(allocation.Partition).NotTo(BeNil()) - Ω(allocation.AllocationID).NotTo(BeNil()) - Ω(allocation.ApplicationID).To(Equal(sleepRespPod.ObjectMeta.Labels["applicationId"])) - core := sleepRespPod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue() - mem := sleepRespPod.Spec.Containers[0].Resources.Requests.Memory().Value() - resMap := allocation.ResourcePerAlloc - Ω(len(resMap)).NotTo(gomega.BeZero()) - Ω(resMap["memory"]).To(gomega.Equal(mem)) - Ω(resMap["vcore"]).To(gomega.Equal(core)) - }) - - AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns}) - - By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(HaveOccurred()) - Ω(checks).To(Equal(""), checks) - - By("Tearing down namespace: " + ns) - err = kClient.TearDownNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) diff --git a/test/e2e/state_aware_app_scheduling/state_aware_app_scheduling_suite_test.go b/test/e2e/state_aware_app_scheduling/state_aware_app_scheduling_suite_test.go deleted file mode 100644 index c6d12a58c..000000000 --- a/test/e2e/state_aware_app_scheduling/state_aware_app_scheduling_suite_test.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package stateawareappscheduling_test - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - - v1 "k8s.io/api/core/v1" - - "github.com/onsi/ginkgo/v2/reporters" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -var suiteName string -var oldConfigMap = new(v1.ConfigMap) -var annotation string - -var _ = BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - annotation = "ann-" + common.RandSeq(10) - yunikorn.EnsureYuniKornConfigsPresent() - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "stateaware", annotation) -}) - -var _ = AfterSuite(func() { - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) -}) - -func TestStateAwareAppScheduling(t *testing.T) { - ginkgo.ReportAfterSuite("TestStateAwareAppScheduling", func(report ginkgo.Report) { - err := common.CreateJUnitReportDir() - Ω(err).NotTo(gomega.HaveOccurred()) - err = reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-stateaware_app_scheduling_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestStateAwareAppScheduling", ginkgo.Label("TestStateAwareAppScheduling")) -} - -// Declarations for Ginkgo DSL -var Fail = ginkgo.Fail -var Describe = ginkgo.Describe -var It = ginkgo.It -var PIt = ginkgo.PIt -var By = ginkgo.By -var BeforeEach = ginkgo.BeforeEach -var AfterEach = ginkgo.AfterEach -var BeforeSuite = ginkgo.BeforeSuite -var AfterSuite = ginkgo.AfterSuite - -// Declarations for Gomega DSL -var RegisterFailHandler = gomega.RegisterFailHandler - -// Declarations for Gomega Matchers -var Equal = gomega.Equal -var Ω = gomega.Expect -var BeNil = gomega.BeNil -var HaveOccurred = gomega.HaveOccurred -var BeEmpty = gomega.BeEmpty -var BeTrue = gomega.BeTrue -var ContainSubstring = gomega.ContainSubstring -var BeEquivalentTo = gomega.BeEquivalentTo diff --git a/test/e2e/user_group_limit/user_group_limit_suite_test.go b/test/e2e/user_group_limit/user_group_limit_suite_test.go deleted file mode 100644 index bf06fa0ec..000000000 --- a/test/e2e/user_group_limit/user_group_limit_suite_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package user_group_limit_test - -import ( - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - "github.com/onsi/gomega" - - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" -) - -func init() { - configmanager.YuniKornTestConfig.ParseFlags() -} - -func TestUserGroupLimit(t *testing.T) { - ginkgo.ReportAfterSuite("TestUserGroupLimit", func(report ginkgo.Report) { - err := reporters.GenerateJUnitReportWithConfig( - report, - filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-user_group_limit_junit.xml"), - reporters.JunitReportConfig{OmitSpecLabels: true}, - ) - Ω(err).NotTo(HaveOccurred()) - }) - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "TestUserGroupLimit", ginkgo.Label("TestUserGroupLimit")) -} - -var Ω = gomega.Ω -var HaveOccurred = gomega.HaveOccurred diff --git a/test/e2e/user_group_limit/user_group_limit_test.go b/test/e2e/user_group_limit/user_group_limit_test.go deleted file mode 100644 index 452cbc307..000000000 --- a/test/e2e/user_group_limit/user_group_limit_test.go +++ /dev/null @@ -1,658 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package user_group_limit_test - -import ( - "encoding/json" - "fmt" - "runtime" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - - "github.com/apache/yunikorn-core/pkg/common/configs" - "github.com/apache/yunikorn-core/pkg/common/resources" - "github.com/apache/yunikorn-core/pkg/webservice/dao" - amCommon "github.com/apache/yunikorn-k8shim/pkg/admission/common" - amconf "github.com/apache/yunikorn-k8shim/pkg/admission/conf" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" - "github.com/apache/yunikorn-scheduler-interface/lib/go/si" -) - -type TestType int - -const ( - largeMem = 100 - mediumMem = 50 - smallMem = 30 - sleepPodMem = 99 - user1 = "user1" - user2 = "user2" - group1 = "group1" - group2 = "group2" - sandboxQueue1 = "root.sandbox1" - sandboxQueue2 = "root.sandbox2" - - userTestType TestType = iota - groupTestType -) - -var ( - suiteName string - kClient k8s.KubeCtl - restClient yunikorn.RClient - ns *v1.Namespace - dev = "dev" + common.RandSeq(5) - oldConfigMap = new(v1.ConfigMap) - annotation = "ann-" + common.RandSeq(10) - admissionCustomConfig = map[string]string{ - "log.core.scheduler.ugm.level": "debug", - amconf.AMAccessControlBypassAuth: constants.True, - } -) - -var _ = ginkgo.BeforeSuite(func() { - _, filename, _, _ := runtime.Caller(0) - suiteName = common.GetSuiteName(filename) - // Initializing kubectl client - kClient = k8s.KubeCtl{} - Ω(kClient.SetClient()).To(gomega.BeNil()) - // Initializing rest client - restClient = yunikorn.RClient{} - Ω(restClient).NotTo(gomega.BeNil()) - - yunikorn.EnsureYuniKornConfigsPresent() - - ginkgo.By("Port-forward the scheduler pod") - var err = kClient.PortForwardYkSchedulerPod() - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("create development namespace") - ns, err = kClient.CreateNamespace(dev, nil) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) -}) - -var _ = ginkgo.AfterSuite(func() { - ginkgo.By("Check Yunikorn's health") - checks, err := yunikorn.GetFailedHealthChecks() - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(checks).To(gomega.Equal(""), checks) - ginkgo.By("Tearing down namespace: " + ns.Name) - err = kClient.TearDownNamespace(ns.Name) - Ω(err).NotTo(gomega.HaveOccurred()) -}) - -var _ = ginkgo.Describe("UserGroupLimit", func() { - ginkgo.It("Verify_maxresources_with_a_specific_user_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because memory usage is less than maxresources") - deploySleepPod(usergroup1, sandboxQueue1, false, "because final memory usage is more than maxresources") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - - // usergroup1 can deploy 2 sleep pods to root.sandbox2 - usergroup1Sandbox2Pod1 := deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - usergroup1Sandbox2Pod2 := deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - checkUsage(userTestType, user1, sandboxQueue2, []*v1.Pod{usergroup1Sandbox2Pod1, usergroup1Sandbox2Pod2}) - - // usergroup2 can deploy 2 sleep pods to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - usergroup2Sandbox1Pod2 := deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1, usergroup2Sandbox1Pod2}) - }) - - ginkgo.It("Verify_maxapplications_with_a_specific_user_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 1, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because application count is less than maxapplications") - deploySleepPod(usergroup1, sandboxQueue1, false, "because final application count is more than maxapplications") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - - // usergroup1 can deploy 2 sleep pods to root.sandbox2 - usergroup1Sandbox2Pod1 := deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - usergroup1Sandbox2Pod2 := deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - checkUsage(userTestType, user1, sandboxQueue2, []*v1.Pod{usergroup1Sandbox2Pod1, usergroup1Sandbox2Pod2}) - - // usergroup2 can deploy 2 sleep pods to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - usergroup2Sandbox1Pod2 := deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1, usergroup2Sandbox1Pod2}) - }) - - ginkgo.It("Verify_maxresources_with_a_specific_group_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because memory usage is less than maxresources") - _ = deploySleepPod(usergroup1, sandboxQueue1, false, "because final memory usage is more than maxresources") - checkUsage(groupTestType, group1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - - // usergroup1 can deploy 2 sleep pods to root.sandbox2 - deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - - // usergroup2 can deploy 2 sleep pods to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - }) - - ginkgo.It("Verify_maxapplications_with_a_specific_group_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 1, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because application count is less than maxapplications") - _ = deploySleepPod(usergroup1, sandboxQueue1, false, "because final application count is more than maxapplications") - checkUsage(groupTestType, group1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - - // usergroup1 can deploy 2 sleep pods to root.sandbox2 - deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - deploySleepPod(usergroup1, sandboxQueue2, true, "because there is no limit in root.sandbox2") - - // usergroup2 can deploy 2 sleep pods to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - deploySleepPod(usergroup2, sandboxQueue1, true, fmt.Sprintf("because there is no limit for %s", usergroup2)) - }) - - ginkgo.It("Verify_maxresources_with_user_limit_lower_than_group_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because memory usage is less than maxresources") - deploySleepPod(usergroup1, sandboxQueue1, false, "because final memory usage is more than maxresources in user limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - }) - - ginkgo.It("Verify_maxresources_with_group_limit_lower_than_user_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - if err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - }, - }); err != nil { - return err - } - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"}) - }) - }) - - // usergroup1 can deploy the first sleep pod to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - - // usergroup1 can't deploy the second sleep pod to root.sandbox1 - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because memory usage is less than maxresources") - _ = deploySleepPod(usergroup1, sandboxQueue1, false, "because final memory usage is more than maxresources in group limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1}) - }) - - ginkgo.It("Verify_maxresources_with_a_wildcard_user_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - { - Limit: "wildcard user entry", - Users: []string{"*"}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - }, - }) - }) - }) - - // usergroup1 can deploy 2 sleep pods to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is less than user entry limit") - usergroup1Sandbox1Pod2 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is equal to user entry limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1, usergroup1Sandbox1Pod2}) - - // usergroup2 can deploy 1 sleep pod to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, "because usage is less than wildcard user entry limit") - - // usergroup2 can't deploy the second sleep pod to root.sandbox1 - deploySleepPod(usergroup2, sandboxQueue1, false, "because final memory usage is more than wildcard maxresources") - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1}) - }) - - ginkgo.It("Verify_maxapplications_with_a_wildcard_user_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "user entry", - Users: []string{user1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - { - Limit: "wildcard user entry", - Users: []string{"*"}, - MaxApplications: 1, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - }, - }) - }) - }) - - // usergroup1 can deploy 2 sleep pods to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is less than user entry limit") - usergroup1Sandbox1Pod2 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is equal to user entry limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1, usergroup1Sandbox1Pod2}) - - // usergroup2 can deploy 1 sleep pod to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, "because usage is less than wildcard user entry limit") - - // usergroup2 can't deploy the second sleep pod to root.sandbox1 - deploySleepPod(usergroup2, sandboxQueue1, false, "because final application count is more than wildcard maxapplications") - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1}) - }) - - ginkgo.It("Verify_maxresources_with_a_wildcard_group_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - { - Limit: "wildcard group entry", - Groups: []string{"*"}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", mediumMem), - }, - }, - }, - }) - }) - }) - - // usergroup1 can deploy 2 sleep pods to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is less than user entry limit") - usergroup1Sandbox1Pod2 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is equal to user entry limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1, usergroup1Sandbox1Pod2}) - - // usergroup2 can deploy 1 sleep pod to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, "because usage is less than wildcard user entry limit") - - // usergroup2 can't deploy the second sleep pod to root.sandbox1 - deploySleepPod(usergroup2, sandboxQueue1, false, "because final memory usage is more than wildcard maxresources") - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1}) - }) - - ginkgo.It("Verify_maxapplications_with_a_wildcard_group_limit", func() { - ginkgo.By("Update config") - annotation = "ann-" + common.RandSeq(10) - // The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated. - yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() { - yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", annotation, admissionCustomConfig, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - - return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{ - Name: "sandbox1", - Limits: []configs.Limit{ - { - Limit: "group entry", - Groups: []string{group1}, - MaxApplications: 2, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - { - Limit: "wildcard group entry", - Groups: []string{"*"}, - MaxApplications: 1, - MaxResources: map[string]string{ - siCommon.Memory: fmt.Sprintf("%dM", largeMem), - }, - }, - }, - }) - }) - }) - - // usergroup1 can deploy 2 sleep pods to root.sandbox1 - usergroup1 := &si.UserGroupInformation{User: user1, Groups: []string{group1}} - usergroup1Sandbox1Pod1 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is less than group entry limit") - usergroup1Sandbox1Pod2 := deploySleepPod(usergroup1, sandboxQueue1, true, "because usage is equal to group entry limit") - checkUsage(userTestType, user1, sandboxQueue1, []*v1.Pod{usergroup1Sandbox1Pod1, usergroup1Sandbox1Pod2}) - - // usergroup2 can deploy 1 sleep pod to root.sandbox1 - usergroup2 := &si.UserGroupInformation{User: user2, Groups: []string{group2}} - usergroup2Sandbox1Pod1 := deploySleepPod(usergroup2, sandboxQueue1, true, "because usage is less than wildcard group entry limit") - - // usergroup2 can't deploy the second sleep pod to root.sandbox1 - deploySleepPod(usergroup2, sandboxQueue1, false, "because final application count is more than wildcard maxapplications") - checkUsage(userTestType, user2, sandboxQueue1, []*v1.Pod{usergroup2Sandbox1Pod1}) - }) - - ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name}) - - // Delete all sleep pods - ginkgo.By("Delete all sleep pods") - err := kClient.DeletePods(ns.Name) - if err != nil { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pods in namespace %s - reason is %s\n", ns.Name, err.Error()) - } - - // reset config - ginkgo.By("Restoring YuniKorn configuration") - yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) - }) -}) - -func deploySleepPod(usergroup *si.UserGroupInformation, queuePath string, expectedRunning bool, reason string) *v1.Pod { - usergroupJsonBytes, err := json.Marshal(usergroup) - Ω(err).NotTo(gomega.HaveOccurred()) - - sleepPodConfig := k8s.SleepPodConfig{NS: dev, Mem: smallMem, Labels: map[string]string{constants.LabelQueueName: queuePath}} - sleepPodObj, err := k8s.InitSleepPod(sleepPodConfig) - Ω(err).NotTo(gomega.HaveOccurred()) - sleepPodObj.Annotations[amCommon.UserInfoAnnotation] = string(usergroupJsonBytes) - - ginkgo.By(fmt.Sprintf("%s deploys the sleep pod %s to queue %s", usergroup, sleepPodObj.Name, queuePath)) - sleepPod, err := kClient.CreatePod(sleepPodObj, dev) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - if expectedRunning { - ginkgo.By(fmt.Sprintf("The sleep pod %s can be scheduled %s", sleepPod.Name, reason)) - err = kClient.WaitForPodRunning(dev, sleepPod.Name, 60*time.Second) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - } else { - ginkgo.By(fmt.Sprintf("The sleep pod %s can't be scheduled %s", sleepPod.Name, reason)) - // Since Pending is the initial state of PodPhase, sleep for 5 seconds, then check whether the pod is still in Pending state. - time.Sleep(5 * time.Second) - err = kClient.WaitForPodPending(sleepPod.Namespace, sleepPod.Name, 60*time.Second) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - } - return sleepPod -} - -func checkUsage(testType TestType, name string, queuePath string, expectedRunningPods []*v1.Pod) { - var rootQueueResourceUsageDAO *dao.ResourceUsageDAOInfo - if testType == userTestType { - ginkgo.By(fmt.Sprintf("Check user resource usage for %s in queue %s", name, queuePath)) - userUsageDAOInfo, err := restClient.GetUserUsage(constants.DefaultPartition, name) - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(userUsageDAOInfo).NotTo(gomega.BeNil()) - - rootQueueResourceUsageDAO = userUsageDAOInfo.Queues - } else if testType == groupTestType { - ginkgo.By(fmt.Sprintf("Check group resource usage for %s in queue %s", name, queuePath)) - groupUsageDAOInfo, err := restClient.GetGroupUsage(constants.DefaultPartition, name) - Ω(err).NotTo(gomega.HaveOccurred()) - Ω(groupUsageDAOInfo).NotTo(gomega.BeNil()) - - rootQueueResourceUsageDAO = groupUsageDAOInfo.Queues - } - Ω(rootQueueResourceUsageDAO).NotTo(gomega.BeNil()) - - var resourceUsageDAO *dao.ResourceUsageDAOInfo - for _, queue := range rootQueueResourceUsageDAO.Children { - if queue.QueuePath == queuePath { - resourceUsageDAO = queue - break - } - } - Ω(resourceUsageDAO).NotTo(gomega.BeNil()) - - appIDs := make([]interface{}, 0, len(expectedRunningPods)) - for _, pod := range expectedRunningPods { - appIDs = append(appIDs, pod.Labels[constants.LabelApplicationID]) - } - Ω(resourceUsageDAO.ResourceUsage).NotTo(gomega.BeNil()) - Ω(resourceUsageDAO.ResourceUsage.Resources["pods"]).To(gomega.Equal(resources.Quantity(len(expectedRunningPods)))) - Ω(resourceUsageDAO.RunningApplications).To(gomega.ConsistOf(appIDs...)) -}