diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml index 61361295..13e2e09e 100644 --- a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml @@ -1,4 +1,4 @@ -- name: ci-kubernetes-e2e-kubeadm-kinder-cp-kubelet-local-{{ dashVer .KubernetesVersion }} +- name: ci-kubernetes-e2e-kubeadm-kinder-control-plane-kubelet-local-{{ dashVer .KubernetesVersion }} cluster: eks-prow-build-cluster interval: {{ .JobInterval }} decorate: true @@ -7,9 +7,9 @@ preset-kind-volume-mounts: "true" annotations: testgrid-dashboards: sig-cluster-lifecycle-kubeadm - testgrid-tab-name: kubeadm-kinder-cp-kubelet-local-{{ dashVer .KubernetesVersion }} + testgrid-tab-name: kubeadm-kinder-control-plane-kubelet-local-{{ dashVer .KubernetesVersion }} testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io - description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using ControlPlaneKubeletLocalMode to join control-plane and run kubeadm-e2e and the conformance suite" + description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using the ControlPlaneKubeletLocalMode feature gate and run the kubeadm-e2e and the conformance suites" testgrid-num-columns-recent: "20" {{ .AlertAnnotations }} decoration_config: diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml index 917b17ec..bc4311d5 100644 --- a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml +++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml @@ -2,7 +2,7 @@ version: 1 summary: | This workflow implements a sequence of tasks used test the proper functioning - of kubeadm not in control plane local kubelet mode during joining control plane. + of the ControlPlaneKubeletLocalMode feature gate. vars: # vars defines default values for variable used by tasks in this workflow; # those values might be overridden when importing this files. @@ -62,6 +62,24 @@ tasks: - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true" - --copy-certs=auto timeout: 5m +- name: post-init + description: | + Run commands after kubeadm init is called on a primary CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: join description: | Join the other nodes to the Kubernetes cluster @@ -74,6 +92,52 @@ tasks: - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} - --copy-certs=auto timeout: 10m +- name: post-join + description: | + Run commands after kubeadm join is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane3" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: pre-upgrade + description: | + Run commands before kubeadm upgrade is called on all joined CP node to replace + the server in the kubelet's kubeconfig to point to the load balancer. + cmd: /bin/bash + args: + - -c + - | + set -x + + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: upgrade description: | upgrades the cluster to Kubernetes "upgradeVersion" @@ -86,6 +150,31 @@ tasks: - --loglevel=debug - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} timeout: 15m +- name: post-upgrade + description: | + Run commands after kubeadm upgrade is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane3" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: cluster-info description: | Runs cluster-info diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml index dbfec517..a9d1038c 100644 --- a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml +++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml @@ -1,7 +1,7 @@ version: 1 summary: | This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using - control plane local kubelet mode to join control plane. + the ControlPlaneKubeletLocalMode feature gate. test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }} config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }} vars: diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml index 7367b915..4ecc6dfb 100644 --- a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml +++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml @@ -2,7 +2,7 @@ version: 1 summary: | This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using - control plane local kubelet mode to join control plane. + the ControlPlaneKubeletLocalMode feature gate. test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml vars: diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml index 03068913..4138f2d9 100644 --- a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml +++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml @@ -3,7 +3,7 @@ version: 1 summary: | This workflow implements a sequence of tasks used test the proper functioning - of kubeadm not in control plane local kubelet mode during joining control plane. + of the ControlPlaneKubeletLocalMode feature gate. vars: # vars defines default values for variable used by tasks in this workflow; # those values might be overridden when importing this files. @@ -63,6 +63,24 @@ tasks: - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true" - --copy-certs=auto timeout: 5m +- name: post-init + description: | + Run commands after kubeadm init is called on a primary CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: join description: | Join the other nodes to the Kubernetes cluster @@ -75,6 +93,52 @@ tasks: - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} - --copy-certs=auto timeout: 10m +- name: post-join + description: | + Run commands after kubeadm join is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane3" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: pre-upgrade + description: | + Run commands before kubeadm upgrade is called on all joined CP node to replace + the server in the kubelet's kubeconfig to point to the load balancer. + cmd: /bin/bash + args: + - -c + - | + set -x + + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: upgrade description: | upgrades the cluster to Kubernetes "upgradeVersion" @@ -87,6 +151,31 @@ tasks: - --loglevel=debug - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} timeout: 15m +- name: post-upgrade + description: | + Run commands after kubeadm upgrade is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane3" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m - name: cluster-info description: | Runs cluster-info