review fixes

kubernetes · Jun 28, 2024 · ca843b9 · ca843b9
1 parent 9a3d7ce
commit ca843b9
Show file tree

Hide file tree

Showing 5 changed files with 185 additions and 7 deletions.
diff --git a/...update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml b/...update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
@@ -1,4 +1,4 @@
-- name: ci-kubernetes-e2e-kubeadm-kinder-cp-kubelet-local-{{ dashVer .KubernetesVersion }}
+- name: ci-kubernetes-e2e-kubeadm-kinder-control-plane-kubelet-local-{{ dashVer .KubernetesVersion }}
   cluster: eks-prow-build-cluster
   interval: {{ .JobInterval }}
   decorate: true
@@ -7,9 +7,9 @@
     preset-kind-volume-mounts: "true"
   annotations:
     testgrid-dashboards: sig-cluster-lifecycle-kubeadm
-    testgrid-tab-name: kubeadm-kinder-cp-kubelet-local-{{ dashVer .KubernetesVersion }}
+    testgrid-tab-name: kubeadm-kinder-control-plane-kubelet-local-{{ dashVer .KubernetesVersion }}
     testgrid-alert-email: [email protected]
-    description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using ControlPlaneKubeletLocalMode to join control-plane and run kubeadm-e2e and the conformance suite"
+    description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using the ControlPlaneKubeletLocalMode feature gate and run the kubeadm-e2e and the conformance suites"  
     testgrid-num-columns-recent: "20"
 {{ .AlertAnnotations }}
   decoration_config:

diff --git a/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -2,7 +2,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of kubeadm not in control plane local kubelet mode during joining control plane.
+  of the ControlPlaneKubeletLocalMode feature gate.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -62,6 +62,24 @@ tasks:
   - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
   - --copy-certs=auto
   timeout: 5m
+- name: post-init
+  description: |
+    Run commands after kubeadm init is called on a primary CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: join
   description: |
     Join the other nodes to the Kubernetes cluster
@@ -74,6 +92,52 @@ tasks:
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
   - --copy-certs=auto
   timeout: 10m
+- name: post-join
+  description: |
+    Run commands after kubeadm join is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane3"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: pre-upgrade
+  description: |
+    Run commands before kubeadm upgrade is called on all joined CP node to replace
+    the server in the kubelet's kubeconfig to point to the load balancer.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: upgrade
   description: |
     upgrades the cluster to Kubernetes "upgradeVersion"
@@ -86,6 +150,31 @@ tasks:
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
   timeout: 15m
+- name: post-upgrade
+  description: |
+    Run commands after kubeadm upgrade is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane3"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: cluster-info
   description: |
     Runs cluster-info

diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
@@ -1,7 +1,7 @@
 version: 1
 summary: |
   This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using
-  control plane local kubelet mode to join control plane.
+  the ControlPlaneKubeletLocalMode feature gate.  
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }}
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
 vars:

diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
@@ -2,7 +2,7 @@
 version: 1
 summary: |
   This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using
-  control plane local kubelet mode to join control plane.
+  the ControlPlaneKubeletLocalMode feature gate.  
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
 vars:

diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -3,7 +3,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of kubeadm not in control plane local kubelet mode during joining control plane.
+  of the ControlPlaneKubeletLocalMode feature gate.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -63,6 +63,24 @@ tasks:
   - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
   - --copy-certs=auto
   timeout: 5m
+- name: post-init
+  description: |
+    Run commands after kubeadm init is called on a primary CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: join
   description: |
     Join the other nodes to the Kubernetes cluster
@@ -75,6 +93,52 @@ tasks:
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
   - --copy-certs=auto
   timeout: 10m
+- name: post-join
+  description: |
+    Run commands after kubeadm join is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane3"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: pre-upgrade
+  description: |
+    Run commands before kubeadm upgrade is called on all joined CP node to replace
+    the server in the kubelet's kubeconfig to point to the load balancer.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
+      ${CMD} sed 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: upgrade
   description: |
     upgrades the cluster to Kubernetes "upgradeVersion"
@@ -87,6 +151,31 @@ tasks:
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
   timeout: 15m
+- name: post-upgrade
+  description: |
+    Run commands after kubeadm upgrade is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='\{\{ .NetworkSettings.IPAddress \}\}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane3"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
 - name: cluster-info
   description: |
     Runs cluster-info