Merge pull request #3080 from chrischdi/pr-cp-kubelet-local-tests

kinder: add test workflow for testing ControlPlaneKubeletLocalMode feature gate
kubernetes · Jul 12, 2024 · 2095c16 · 2095c16
2 parents 1445271 + 8f7cbc8
commit 2095c16
Show file tree

Hide file tree

Showing 6 changed files with 574 additions and 0 deletions.
diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml
@@ -248,3 +248,15 @@ jobGroups:
     - ./templates/workflows/encryption-algorithm-tasks.yaml
   jobs:
   - kubernetesVersion: latest
+
+- name: control-plane-local-kubelet-mode
+  testInfraJobSpec:
+    targetFile: kubeadm-kinder-control-plane-local-kubelet-mode.yaml
+    template: ./templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
+  kinderWorkflowSpec:
+    targetFile: control-plane-local-kubelet-mode-{{ .KubernetesVersion }}.yaml
+    template: ./templates/workflows/control-plane-local-kubelet-mode.yaml
+    additionalFiles:
+    - ./templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
+  jobs:
+  - kubernetesVersion: latest
diff --git a/...update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml b/...update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
@@ -0,0 +1,42 @@
+- name: ci-kubernetes-e2e-kubeadm-kinder-control-plane-kubelet-local-mode-{{ dashVer .KubernetesVersion }}
+  cluster: eks-prow-build-cluster
+  interval: {{ .JobInterval }}
+  decorate: true
+  labels:
+    preset-dind-enabled: "true"
+    preset-kind-volume-mounts: "true"
+  annotations:
+    testgrid-dashboards: sig-cluster-lifecycle-kubeadm
+    testgrid-tab-name: kubeadm-kinder-control-plane-kubelet-local-mode-{{ dashVer .KubernetesVersion }}
+    testgrid-alert-email: [email protected]
+    description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using the ControlPlaneKubeletLocalMode feature gate and run the kubeadm-e2e and the conformance suites"
+    testgrid-num-columns-recent: "20"
+{{ .AlertAnnotations }}
+  decoration_config:
+    timeout: 60m
+  extra_refs:
+  - org: kubernetes
+    repo: kubernetes
+    base_ref: {{ branchFor .KubernetesVersion }}
+    path_alias: k8s.io/kubernetes
+  - org: kubernetes
+    repo: kubeadm
+    base_ref: main
+    path_alias: k8s.io/kubeadm
+  spec:
+    containers:
+    - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }}
+      command:
+      - runner.sh
+      - "../kubeadm/kinder/ci/kinder-run.sh"
+      args:
+      - {{ .WorkflowFile }}
+      securityContext:
+        privileged: true
+      resources:
+        limits:
+          memory: "9000Mi"
+          cpu: 2000m
+        requests:
+          memory: "9000Mi"
+          cpu: 2000m
diff --git a/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -0,0 +1,248 @@
+# IMPORTANT! this workflow is imported by control-plane-local-kubelet-mode-* workflows.
+version: 1
+summary: |
+  This workflow implements a sequence of tasks used test the proper functioning
+  of the ControlPlaneKubeletLocalMode feature gate.
+vars:
+  # vars defines default values for variable used by tasks in this workflow;
+  # those values might be overridden when importing this files.
+  kubernetesVersion: v1.31.0
+  upgradeVersion: v1.31.0
+  controlPlaneNodes: 3
+  workerNodes: 2
+  baseImage: kindest/base:v20221102-76f15095 # has containerd
+  image: kindest/node:test
+  clusterName: kinder-control-plane-local-kubelet-mode
+  kubeadmVerbosity: 6
+tasks:
+- name: pull-base-image
+  description: |
+    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
+  cmd: docker
+  args:
+  - pull
+  - "{{ .vars.baseImage }}"
+- name: add-kubernetes-versions
+  description: |
+    creates a node-image-variant by adding a Kubernetes version
+  cmd: kinder
+  args:
+  - build
+  - node-image-variant
+  - --base-image={{ .vars.baseImage }}
+  - --image={{ .vars.image }}
+  - --with-init-artifacts={{ .vars.kubernetesVersion }}
+  - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
+  - --loglevel=debug
+  timeout: 15m
+- name: create-cluster
+  description: |
+    create a set of nodes ready for hosting the Kubernetes cluster
+  cmd: kinder
+  args:
+  - create
+  - cluster
+  - --name={{ .vars.clusterName }}
+  - --image={{ .vars.image }}
+  - --control-plane-nodes={{ .vars.controlPlaneNodes }}
+  - --worker-nodes={{ .vars.workerNodes }}
+  - --loglevel=debug
+  timeout: 5m
+- name: init
+  description: |
+    Initializes the Kubernetes cluster with version "initVersion"
+    by starting the boostrap control-plane nodes
+  cmd: kinder
+  args:
+  - do
+  - kubeadm-init
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
+  - --copy-certs=auto
+  timeout: 5m
+- name: post-init
+  description: |
+    Run commands after kubeadm init is called on a primary CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: join
+  description: |
+    Join the other nodes to the Kubernetes cluster
+  cmd: kinder
+  args:
+  - do
+  - kubeadm-join
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  - --copy-certs=auto
+  timeout: 10m
+- name: post-join
+  description: |
+    Run commands after kubeadm join is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
+
+      # Ensure kubelet.conf points to the local IP.
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: pre-upgrade
+  description: |
+    Run commands before kubeadm upgrade is called on all joined CP node to replace
+    the server in the kubelet's kubeconfig to point to the load balancer.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: upgrade
+  description: |
+    upgrades the cluster to Kubernetes "upgradeVersion"
+  cmd: kinder
+  args:
+  - do
+  - kubeadm-upgrade
+  - --upgrade-version={{ .vars.upgradeVersion }}
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 15m
+- name: post-upgrade
+  description: |
+    Run commands after kubeadm upgrade is called on all joined CP node to checks if
+    the kubelet's kubeconfig file points to the local apiserver.
+  cmd: /bin/bash
+  args:
+    - -c
+    - |
+      set -x
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
+      CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
+      ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
+
+      # Ensure exit status of 0
+      exit 0
+  timeout: 5m
+- name: cluster-info
+  description: |
+    Runs cluster-info
+  cmd: kinder
+  args:
+  - do
+  - cluster-info
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+- name: e2e-kubeadm
+  description: |
+    Runs kubeadm e2e tests
+  cmd: kinder
+  args:
+  - test
+  - e2e-kubeadm
+  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  timeout: 10m
+- name: e2e
+  description: |
+    Runs Kubernetes e2e test (conformance)
+  cmd: kinder
+  args:
+  - test
+  - e2e
+  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e
+  - --parallel
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  timeout: 35m
+- name: get-logs
+  description: |
+    Collects all the test logs
+  cmd: kinder
+  args:
+  - export
+  - logs
+  - --loglevel=debug
+  - --name={{ .vars.clusterName }}
+  - "{{ .env.ARTIFACTS }}"
+  force: true
+  timeout: 5m
+  # kind export log is know to be flaky, so we are temporary ignoring errors in order
+  # to make the test pass in case everything else passed
+  # see https://github.com/kubernetes-sigs/kind/issues/456
+  ignoreError: true
+- name: reset
+  description: |
+    Exec kubeadm reset
+  cmd: kinder
+  args:
+  - do
+  - kubeadm-reset
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  force: true
+- name: delete
+  description: |
+    Deletes the cluster
+  cmd: kinder
+  args:
+  - delete
+  - cluster
+  - --name={{ .vars.clusterName }}
+  - --loglevel=debug
+  force: true
diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
@@ -0,0 +1,11 @@
+version: 1
+summary: |
+  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using
+  the ControlPlaneKubeletLocalMode feature gate.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }}
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
+vars:
+  kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
+  upgradeVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
+tasks:
+- import: control-plane-local-kubelet-mode-tasks.yaml
diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
@@ -0,0 +1,12 @@
+# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
+version: 1
+summary: |
+  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using
+  the ControlPlaneKubeletLocalMode feature gate.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
+vars:
+  kubernetesVersion: "{{ resolve `ci/latest` }}"
+  upgradeVersion: "{{ resolve `ci/latest` }}"
+tasks:
+- import: control-plane-local-kubelet-mode-tasks.yaml