[pull] master from apache:master

.golangci.yml

@@ -34,7 +34,7 @@ linters-settings:
   goimports:
     local-prefixes: github.com/apache/yunikorn
   govet:
-    check-shadowing: true
+    shadow: true
   goconst:
     min-occurrences: 5
   funlen:

Makefile

@@ -60,6 +60,11 @@ TEST_SERVER_BINARY=web-test-server
 TOOLS_DIR=tools
 REPO=github.com/apache/yunikorn-k8shim/pkg
 
+IMAGE_SOURCE?=https://github.com/apache/yunikorn-k8shim
+IMAGE_URL?=https://hub.docker.com/r/apache/yunikorn
+LICENSE=Apache-2.0
+DOCS_URL=https://yunikorn.apache.org
+
 # PATH
 export PATH := $(BASE_DIR)/$(TOOLS_DIR):$(GO_EXE_PATH):$(PATH)
 
@@ -132,7 +137,8 @@ endif
 
 # shellcheck
 SHELLCHECK_VERSION=v0.9.0
-SHELLCHECK_BIN=${TOOLS_DIR}/shellcheck
+SHELLCHECK_PATH=${TOOLS_DIR}/shellcheck-$(SHELLCHECK_VERSION)
+SHELLCHECK_BIN=${SHELLCHECK_PATH}/shellcheck
 SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).$(HOST_ARCH).tar.xz
 ifeq (darwin, $(OS))
 ifeq (arm64, $(HOST_ARCH))
@@ -143,41 +149,56 @@ ifeq (armv7l, $(HOST_ARCH))
 SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).armv6hf.tar.xz
 endif
 endif
+export PATH := $(BASE_DIR)/$(SHELLCHECK_PATH):$(PATH)
 
 # golangci-lint
 GOLANGCI_LINT_VERSION=1.57.2
-GOLANGCI_LINT_BIN=$(TOOLS_DIR)/golangci-lint
+GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-v$(GOLANGCI_LINT_VERSION)
+GOLANGCI_LINT_BIN=$(GOLANGCI_LINT_PATH)/golangci-lint
 GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz
 GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH)
+export PATH := $(BASE_DIR)/$(GOLANGCI_LINT_PATH):$(PATH)
 
 # kubectl
 KUBECTL_VERSION=v1.27.7
-KUBECTL_BIN=$(TOOLS_DIR)/kubectl
+KUBECTL_PATH=$(TOOLS_DIR)/kubectl-$(KUBECTL_VERSION)
+KUBECTL_BIN=$(KUBECTL_PATH)/kubectl
+export PATH := $(BASE_DIR)/$(KUBECTL_PATH):$(PATH)
 
 # kind
 KIND_VERSION=v0.23.0
-KIND_BIN=$(TOOLS_DIR)/kind
+KIND_PATH=$(TOOLS_DIR)/kind-$(KIND_VERSION)
+KIND_BIN=$(KIND_PATH)/kind
+export PATH := $(BASE_DIR)/$(KIND_PATH):$(PATH)
 
 # helm
 HELM_VERSION=v3.12.1
-HELM_BIN=$(TOOLS_DIR)/helm
+HELM_PATH=$(TOOLS_DIR)/helm-$(HELM_VERSION)
+HELM_BIN=$(HELM_PATH)/helm
 HELM_ARCHIVE=helm-$(HELM_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz
 HELM_ARCHIVE_BASE=$(OS)-$(EXEC_ARCH)
+export PATH := $(BASE_DIR)/$(HELM_PATH):$(PATH)
 
 # spark
 export SPARK_VERSION=3.3.3
 # sometimes the image is not avaiable with $SPARK_VERSION, the minor version must match
 export SPARK_PYTHON_VERSION=3.3.1
-export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark
+export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark-v$(SPARK_VERSION)
 export SPARK_SUBMIT_CMD=$(SPARK_HOME)/bin/spark-submit
 export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION)
+export PATH := $(SPARK_HOME):$(PATH)
 
 # go-licenses
 GO_LICENSES_VERSION=v1.6.0
-GO_LICENSES_BIN=$(TOOLS_DIR)/go-licenses
+GO_LICENSES_PATH=$(TOOLS_DIR)/go-licenses-$(GO_LICENSES_VERSION)
+GO_LICENSES_BIN=$(GO_LICENSES_PATH)/go-licenses
+export PATH := $(BASE_DIR)/$(GO_LICENSES_PATH):$(PATH)
 
 # ginkgo
-GINKGO_BIN=$(TOOLS_DIR)/ginkgo
+GINKGO_VERSION=v2.19.0
+GINKGO_PATH=$(TOOLS_DIR)/ginkgo-$(GINKGO_VERSION)
+GINKGO_BIN=$(GINKGO_PATH)/ginkgo
+export PATH := $(BASE_DIR)/$(GINKGO_PATH):$(PATH)
 
 FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf
 
@@ -222,50 +243,61 @@ init: conf/scheduler-config-local.yaml
 conf/scheduler-config-local.yaml: conf/scheduler-config.yaml
 	./scripts/plugin-conf-gen.sh $(KUBECONFIG) "conf/scheduler-config.yaml" "conf/scheduler-config-local.yaml"
 
+# Print tools version
+.PHONY: print_kubectl_version
+print_kubectl_version:
+	@echo $(KUBECTL_VERSION)
+.PHONY: print_kind_version
+print_kind_version:
+	@echo $(KIND_VERSION)
+.PHONY: print_helm_version
+print_helm_version:
+	@echo $(HELM_VERSION)
+
 # Install tools
 .PHONY: tools
 tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN) $(GINKGO_BIN)
 
 # Install shellcheck
 $(SHELLCHECK_BIN):
 	@echo "installing shellcheck $(SHELLCHECK_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
+	@mkdir -p "$(SHELLCHECK_PATH)"
 	@curl -sSfL "https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/$(SHELLCHECK_ARCHIVE)" \
-		| tar -x -J --strip-components=1 -C "$(TOOLS_DIR)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck"
+		| tar -x -J --strip-components=1 -C "$(SHELLCHECK_PATH)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck"
 
 # Install golangci-lint
 $(GOLANGCI_LINT_BIN):
 	@echo "installing golangci-lint v$(GOLANGCI_LINT_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
+	@mkdir -p "$(GOLANGCI_LINT_PATH)"
 	@curl -sSfL "https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/$(GOLANGCI_LINT_ARCHIVE)" \
-		| tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint"
+		| tar -x -z --strip-components=1 -C "$(GOLANGCI_LINT_PATH)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint"
 
 # Install kubectl
 $(KUBECTL_BIN):
 	@echo "installing kubectl $(KUBECTL_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
+	@mkdir -p "$(KUBECTL_PATH)"
 	@curl -sSfL -o "$(KUBECTL_BIN)" \
 		"https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(OS)/$(EXEC_ARCH)/kubectl" && \
 		chmod +x "$(KUBECTL_BIN)"
 
 # Install kind
 $(KIND_BIN):
 	@echo "installing kind $(KIND_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
+	@mkdir -p "$(KIND_PATH)"
 	@curl -sSfL -o "$(KIND_BIN)" \
 		"https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(OS)-$(EXEC_ARCH)" && \
 		chmod +x "$(KIND_BIN)"
 
 # Install helm
 $(HELM_BIN):
 	@echo "installing helm $(HELM_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
+	@mkdir -p "$(HELM_PATH)"
 	@curl -sSfL "https://get.helm.sh/$(HELM_ARCHIVE)" \
-		| tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(HELM_ARCHIVE_BASE)/helm"
+		| tar -x -z --strip-components=1 -C "$(HELM_PATH)" "$(HELM_ARCHIVE_BASE)/helm"
 
 # Install spark
 $(SPARK_SUBMIT_CMD):
-	@echo "installing spark $(SPARK_VERSION)"
+	@echo "installing spark v$(SPARK_VERSION)"
 	@rm -rf "$(SPARK_HOME)" "$(SPARK_HOME).tmp"
 	@mkdir -p "$(SPARK_HOME).tmp"
 	@curl -sSfL "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz" \
@@ -275,23 +307,20 @@ $(SPARK_SUBMIT_CMD):
 # Install go-licenses
 $(GO_LICENSES_BIN):
 	@echo "installing go-licenses $(GO_LICENSES_VERSION)"
-	@mkdir -p "$(TOOLS_DIR)"
-	@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)"
+	@mkdir -p "$(GO_LICENSES_PATH)"
+	@GOBIN="$(BASE_DIR)/$(GO_LICENSES_PATH)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)"
 
 $(GINKGO_BIN):
-	@echo "installing ginkgo"
-	@mkdir -p "$(TOOLS_DIR)"
-	@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo"
+	@echo "installing ginkgo $(GINKGO_VERSION)"
+	@mkdir -p "$(GINKGO_PATH)"
+	@GOBIN="$(BASE_DIR)/$(GINKGO_PATH)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)"
 
 # Run lint against the previous commit for PR and branch build
 # In dev setup look at all changes on top of master
 .PHONY: lint
 lint: $(GOLANGCI_LINT_BIN)
 	@echo "running golangci-lint"
-	@git symbolic-ref -q HEAD && REV="origin/HEAD" || REV="HEAD^" ; \
-	headSHA=$$(git rev-parse --short=12 $${REV}) ; \
-	echo "checking against commit sha $${headSHA}" ; \
-	"${GOLANGCI_LINT_BIN}" run
+	@"${GOLANGCI_LINT_BIN}" run
 
 # Check scripts
 .PHONY: check_scripts
@@ -477,6 +506,15 @@ sched_image: $(OUTPUT)/third-party-licenses.md scheduler docker/scheduler
 	--label "yunikorn-k8shim-revision=${SHIM_SHA}" \
 	--label "BuildTimeStamp=${DATE}" \
 	--label "Version=${VERSION}" \
+	--label "org.opencontainers.image.title=${SCHEDULER_BINARY}" \
+	--label "org.opencontainers.image.description=Apache YuniKorn Scheduler" \
+	--label "org.opencontainers.image.version=${VERSION}" \
+	--label "org.opencontainers.image.created=$(DATE)" \
+	--label "org.opencontainers.image.source=${IMAGE_SOURCE}" \
+	--label "org.opencontainers.image.url=${IMAGE_URL}" \
+	--label "org.opencontainers.image.revision=$(SHIM_SHA)" \
+	--label "org.opencontainers.image.license=${LICENSE}" \
+	--label "org.opencontainers.image.documentation=${DOCS_URL}" \
 	${QUIET}
 
 # Build a plugin image based on the production ready version
@@ -498,6 +536,15 @@ plugin_image: $(OUTPUT)/third-party-licenses.md plugin docker/plugin conf/schedu
 	--label "yunikorn-k8shim-revision=${SHIM_SHA}" \
 	--label "BuildTimeStamp=${DATE}" \
 	--label "Version=${VERSION}" \
+	--label "org.opencontainers.image.title=${PLUGIN_BINARY}" \
+	--label "org.opencontainers.image.description=Apache YuniKorn Scheduler (Plugin)" \
+	--label "org.opencontainers.image.version=${VERSION}" \
+	--label "org.opencontainers.image.created=$(DATE)" \
+	--label "org.opencontainers.image.source=${IMAGE_SOURCE}" \
+	--label "org.opencontainers.image.url=${IMAGE_URL}" \
+	--label "org.opencontainers.image.revision=$(SHIM_SHA)" \
+	--label "org.opencontainers.image.license=${LICENSE}" \
+	--label "org.opencontainers.image.documentation=${DOCS_URL}" \
 	${QUIET}
 
 # Build admission controller binary in a production ready version
@@ -546,6 +593,15 @@ adm_image: $(OUTPUT)/third-party-licenses.md admission docker/admission
 	--label "yunikorn-k8shim-revision=${SHIM_SHA}" \
 	--label "BuildTimeStamp=${DATE}" \
 	--label "Version=${VERSION}" \
+	--label "org.opencontainers.image.title=${ADMISSION_CONTROLLER_BINARY}" \
+	--label "org.opencontainers.image.description=Apache YuniKorn Admission Controller" \
+	--label "org.opencontainers.image.version=${VERSION}" \
+	--label "org.opencontainers.image.created=$(DATE)" \
+	--label "org.opencontainers.image.source=${IMAGE_SOURCE}" \
+	--label "org.opencontainers.image.url=${IMAGE_URL}" \
+	--label "org.opencontainers.image.revision=$(SHIM_SHA)" \
+	--label "org.opencontainers.image.license=${LICENSE}" \
+	--label "org.opencontainers.image.documentation=${DOCS_URL}" \
 	${QUIET}
 
 # Build all images based on the production ready version

go.mod

@@ -21,8 +21,8 @@ module github.com/apache/yunikorn-k8shim
 go 1.21
 
 require (
-	github.com/apache/yunikorn-core v0.0.0-20240711165824-d96cd583305b
-	github.com/apache/yunikorn-scheduler-interface v0.0.0-20240425182941-07f5695119a1
+	github.com/apache/yunikorn-core v0.0.0-20240802210614-4aec626c6bf9
+	github.com/apache/yunikorn-scheduler-interface v0.0.0-20240731203810-92032b13d586
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/looplab/fsm v1.0.1

go.sum

@@ -9,10 +9,10 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
-github.com/apache/yunikorn-core v0.0.0-20240711165824-d96cd583305b h1:GDizY3dcE+hkfik/+NY3Zdw71A/V4dWGp9Pl6k5Ii2M=
-github.com/apache/yunikorn-core v0.0.0-20240711165824-d96cd583305b/go.mod h1:pSi7AFBRiGCGQ7RwQffpD4m6dvA5lc1HuCrg7LpJJqs=
-github.com/apache/yunikorn-scheduler-interface v0.0.0-20240425182941-07f5695119a1 h1:v4J9L3MlW8BQfYnbq6FV2l3uyay3SqMS2Ffpo+SFat4=
-github.com/apache/yunikorn-scheduler-interface v0.0.0-20240425182941-07f5695119a1/go.mod h1:WuHJpVk34t8N5+1ErYGj/5Qq33/cRzL4YtuoAsbMtWc=
+github.com/apache/yunikorn-core v0.0.0-20240802210614-4aec626c6bf9 h1:s1Co/K+cR9Q/GW0e974dToW9eyLQZxYoCp0TCoEuEj0=
+github.com/apache/yunikorn-core v0.0.0-20240802210614-4aec626c6bf9/go.mod h1:S9yGBGA2i2hAajtEc2t4lmiPJDZz3Ek8eVxz5KhJqGI=
+github.com/apache/yunikorn-scheduler-interface v0.0.0-20240731203810-92032b13d586 h1:ZVpo9Qj2/gvwX6Rl44UxkZBm2pZWEJDYWTramc9hwF0=
+github.com/apache/yunikorn-scheduler-interface v0.0.0-20240731203810-92032b13d586/go.mod h1:WuHJpVk34t8N5+1ErYGj/5Qq33/cRzL4YtuoAsbMtWc=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=

pkg/admission/conf/am_conf.go

@@ -19,6 +19,7 @@
 package conf
 
 import (
+	"errors"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -117,7 +118,7 @@ func NewAdmissionControllerConf(configMaps []*v1.ConfigMap) *AdmissionController
 func (acc *AdmissionControllerConf) RegisterHandlers(configMaps informersv1.ConfigMapInformer) error {
 	_, err := configMaps.Informer().AddEventHandler(&configMapUpdateHandler{conf: acc})
 	if err != nil {
-		return fmt.Errorf("failed to create register handlers: %w", err)
+		return errors.Join(errors.New("failed to create register handlers: "), err)
 	}
 
 	return nil

pkg/admission/namespace_cache.go

@@ -19,7 +19,7 @@
 package admission
 
 import (
-	"fmt"
+	"errors"
 
 	v1 "k8s.io/api/core/v1"
 	informersv1 "k8s.io/client-go/informers/core/v1"
@@ -62,7 +62,7 @@ func NewNamespaceCache(namespaces informersv1.NamespaceInformer) (*NamespaceCach
 	if namespaces != nil {
 		_, err := namespaces.Informer().AddEventHandler(&namespaceUpdateHandler{cache: nsc})
 		if err != nil {
-			return nil, fmt.Errorf("failed to create namespace cache: %w", err)
+			return nil, errors.Join(errors.New("failed to create namespace cache: "), err)
 		}
 	}
 	return nsc, nil

pkg/admission/priority_class_cache.go

@@ -19,7 +19,7 @@
 package admission
 
 import (
-	"fmt"
+	"errors"
 
 	schedulingv1 "k8s.io/api/scheduling/v1"
 	informersv1 "k8s.io/client-go/informers/scheduling/v1"
@@ -45,7 +45,7 @@ func NewPriorityClassCache(priorityClasses informersv1.PriorityClassInformer) (*
 	if priorityClasses != nil {
 		_, err := priorityClasses.Informer().AddEventHandler(&priorityClassUpdateHandler{cache: pcc})
 		if err != nil {
-			return nil, fmt.Errorf("failed to create a new cache and register the handler: %w", err)
+			return nil, errors.Join(errors.New("failed to create a new cache and register the handler: "), err)
 		}
 	}
 	return pcc, nil

pkg/admission/util_test.go

@@ -19,6 +19,7 @@
 package admission
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 
@@ -189,3 +190,18 @@ func TestDefaultQueueName(t *testing.T) {
 		t.Fatal("UpdatePodLabelForAdmissionController is not as expected")
 	}
 }
+
+func TestConvert2Namespace(t *testing.T) {
+	namespace := &v1.Namespace{}
+	result := convert2Namespace(namespace)
+	assert.Check(t, result != nil)
+	assert.Check(t, reflect.DeepEqual(result, namespace))
+
+	obj := struct{}{}
+	result = convert2Namespace(obj)
+	assert.Check(t, result == nil)
+
+	pod := &v1.Pod{}
+	result = convert2Namespace(pod)
+	assert.Check(t, result == nil)
+}

pkg/cache/application.go

@@ -115,14 +115,10 @@ func (app *Application) canHandle(ev events.ApplicationEvent) bool {
 	return app.sm.Can(ev.GetEvent())
 }
 
-func (app *Application) GetTask(taskID string) (*Task, error) {
+func (app *Application) GetTask(taskID string) *Task {
 	app.lock.RLock()
 	defer app.lock.RUnlock()
-	if task, ok := app.taskMap[taskID]; ok {
-		return task, nil
-	}
-	return nil, fmt.Errorf("task %s doesn't exist in application %s",
-		taskID, app.applicationID)
+	return app.taskMap[taskID]
 }
 
 func (app *Application) GetApplicationID() string {

pkg/cache/application_test.go

@@ -1184,9 +1184,7 @@ func TestPlaceholderTimeoutEvents(t *testing.T) {
 	})
 	assert.Assert(t, task1 != nil)
 	assert.Equal(t, task1.GetTaskID(), "task02")
-
-	_, taskErr := app.GetTask("task02")
-	assert.NilError(t, taskErr, "Task should exist")
+	assert.Assert(t, app.GetTask("task02") != nil, "Task should exist")
 
 	task1.allocationKey = allocationKey