feat(probe): add startup probe (#1420)

Signed-off-by: Wassim DHIF <[email protected]>
DataDog · Jun 28, 2024 · 7f4da79 · 7f4da79
1 parent 4af0ec6
commit 7f4da79
Show file tree

Hide file tree

Showing 14 changed files with 175 additions and 2 deletions.
diff --git a/charts/datadog/CHANGELOG.md b/charts/datadog/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Datadog changelog
 
+## 3.67.0
+
+* Add startup probe for `Agent`, `Cluster-Agent` and `Cluster-Check-Runner`.
+
 ## 3.66.1
 
 * Add 'datadog.namespaceAnnotationsAsTags' to assign namespace annotations as tags on pod entities in the tagger.

diff --git a/charts/datadog/Chart.yaml b/charts/datadog/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: datadog
-version: 3.66.1
+version: 3.67.0
 appVersion: "7"
 description: Datadog Agent
 keywords:

diff --git a/charts/datadog/README.md b/charts/datadog/README.md
@@ -1,6 +1,6 @@
 # Datadog
 
-![Version: 3.66.1](https://img.shields.io/badge/Version-3.66.1-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
+![Version: 3.67.0](https://img.shields.io/badge/Version-3.67.0-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
 
 [Datadog](https://www.datadoghq.com/) is a hosted infrastructure monitoring platform. This chart adds the Datadog Agent to all nodes in your cluster via a DaemonSet. It also optionally depends on the [kube-state-metrics chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). For more information about monitoring Kubernetes with Datadog, please refer to the [Datadog documentation website](https://docs.datadoghq.com/agent/basic_agent_usage/kubernetes/).
 
@@ -467,6 +467,7 @@ helm install <RELEASE_NAME> \
 | agents.containers.agent.readinessProbe | object | Every 15s / 6 KO / 1 OK | Override default agent readiness probe settings |
 | agents.containers.agent.resources | object | `{}` | Resource requests and limits for the agent container. |
 | agents.containers.agent.securityContext | object | `{}` | Allows you to overwrite the default container SecurityContext for the agent container. |
+| agents.containers.agent.startupProbe | object | Every 15s / 6 KO / 1 OK | Override default agent startup probe settings |
 | agents.containers.initContainers.resources | object | `{}` | Resource requests and limits for the init containers |
 | agents.containers.initContainers.securityContext | object | `{}` | Allows you to overwrite the default container SecurityContext for the init containers. |
 | agents.containers.initContainers.volumeMounts | list | `[]` | Specify additional volumes to mount for the init containers |
@@ -611,6 +612,7 @@ helm install <RELEASE_NAME> \
 | clusterAgent.revisionHistoryLimit | int | `10` | The number of old ReplicaSets to keep in this Deployment. |
 | clusterAgent.securityContext | object | `{}` | Allows you to overwrite the default PodSecurityContext on the cluster-agent pods. |
 | clusterAgent.shareProcessNamespace | bool | `false` | Set the process namespace sharing on the Datadog Cluster Agent |
+| clusterAgent.startupProbe | object | Every 15s / 6 KO / 1 OK | Override default Cluster Agent startup probe settings |
 | clusterAgent.strategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Allow the Cluster Agent deployment to perform a rolling update on helm update |
 | clusterAgent.token | string | `""` | Cluster Agent token is a preshared key between node agents and cluster agent (autogenerated if empty, needs to be at least 32 characters a-zA-z) |
 | clusterAgent.tokenExistingSecret | string | `""` | Existing secret name to use for Cluster Agent token. Put the Cluster Agent token in a key named `token` inside the Secret |
@@ -652,6 +654,7 @@ helm install <RELEASE_NAME> \
 | clusterChecksRunner.resources | object | `{}` | Datadog clusterchecks-agent resource requests and limits. |
 | clusterChecksRunner.revisionHistoryLimit | int | `10` | The number of old ReplicaSets to keep in this Deployment. |
 | clusterChecksRunner.securityContext | object | `{}` | Allows you to overwrite the default PodSecurityContext on the clusterchecks pods. |
+| clusterChecksRunner.startupProbe | object | Every 15s / 6 KO / 1 OK | Override default agent startup probe settings |
 | clusterChecksRunner.strategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Allow the ClusterChecks deployment to perform a rolling update on helm update |
 | clusterChecksRunner.tolerations | list | `[]` | Tolerations for pod assignment |
 | clusterChecksRunner.topologySpreadConstraints | list | `[]` | Allow the ClusterChecks Deployment to schedule using pod topology spreading |

diff --git a/charts/datadog/templates/NOTES.txt b/charts/datadog/templates/NOTES.txt
@@ -27,6 +27,7 @@ Then run:
 {{- end }}
 
 {{- $healthPort := .Values.agents.containers.agent.healthPort }}
+
 {{- with $liveness := .Values.agents.containers.agent.livenessProbe.httpGet }}
 {{- if and $liveness.port (ne $healthPort $liveness.port) }}
 
@@ -37,6 +38,7 @@ Then run:
 Node Agent liveness probe port ({{ $liveness.port }}) is different from the configured health port ({{ $healthPort }}).
 {{- end }}
 {{- end }}
+
 {{- with $readiness := .Values.agents.containers.agent.readinessProbe.httpGet }}
 {{- if and $readiness.port (ne $healthPort $readiness.port) }}
 
@@ -47,6 +49,18 @@ Node Agent liveness probe port ({{ $liveness.port }}) is different from the conf
 Node Agent readiness probe port ({{ $readiness.port }}) is different from the configured health port ({{ $healthPort }}).
 {{- end }}
 {{- end }}
+
+{{- with $startup := .Values.agents.containers.agent.startupProbe.httpGet }}
+{{- if and $startup.port (ne $healthPort $startup.port) }}
+
+##############################################################################
+####               ERROR: Node Agent startup probe misconfiguration       ####
+##############################################################################
+
+Node Agent readiness probe port ({{ $startup.port }}) is different from the configured health port ({{ $healthPort }}).
+{{- end }}
+{{- end }}
+
 {{- if eq (include "should-deploy-cluster-agent" .) "true" }}
 
   {{- if .Values.clusterAgent.metricsProvider.enabled }}
@@ -65,6 +79,7 @@ Create an application key at https://app.datadoghq.com/account/settings#api
     {{- end }}
   {{- end }}
   {{- $healthPort := .Values.clusterAgent.healthPort }}
+
   {{- with $liveness := .Values.clusterAgent.livenessProbe.httpGet }}
   {{- if and $liveness.port (ne $healthPort $liveness.port) }}
 
@@ -75,6 +90,7 @@ Create an application key at https://app.datadoghq.com/account/settings#api
 Cluster Agent liveness probe port ({{ $liveness.port }}) is different from the configured health port ({{ $healthPort }}).
   {{- end }}
   {{- end }}
+
   {{- with $readiness := .Values.clusterAgent.readinessProbe.httpGet }}
   {{- if and $readiness.port (ne $healthPort $readiness.port) }}
 
@@ -85,8 +101,21 @@ Cluster Agent liveness probe port ({{ $liveness.port }}) is different from the c
 Cluster Agent readiness probe port ({{ $readiness.port }}) is different from the configured health port ({{ $healthPort }}).
   {{- end }}
   {{- end }}
+
+  {{- with $startup := .Values.clusterAgent.startupProbe.httpGet }}
+  {{- if and $startup.port (ne $healthPort $startup.port) }}
+
+##############################################################################
+####               ERROR: Cluster Agent startup probe misconfiguration    ####
+##############################################################################
+
+Cluster Agent readiness probe port ({{ $startup.port }}) is different from the configured health port ({{ $healthPort }}).
+  {{- end }}
+  {{- end }}
+
   {{- if (eq (include "should-enable-cluster-check-workers" .) "true") }}
     {{- $healthPort := .Values.clusterChecksRunner.healthPort }}
+
     {{- with $liveness := .Values.clusterChecksRunner.livenessProbe.httpGet }}
     {{- if and $liveness.port (ne $healthPort $liveness.port) }}
 
@@ -97,6 +126,7 @@ Cluster Agent readiness probe port ({{ $readiness.port }}) is different from the
 Cluster Checks Runner liveness probe port ({{ $liveness.port }}) is different from the configured health port ({{ $healthPort }}).
     {{- end }}
     {{- end }}
+
     {{- with $readiness := .Values.clusterChecksRunner.readinessProbe.httpGet }}
     {{- if and $readiness.port (ne $healthPort $readiness.port) }}
 
@@ -107,6 +137,18 @@ Cluster Checks Runner liveness probe port ({{ $liveness.port }}) is different fr
 Cluster Checks Runner readiness probe port ({{ $readiness.port }}) is different from the configured health port ({{ $healthPort }}).
     {{- end }}
     {{- end }}
+
+    {{- with $startup := .Values.clusterChecksRunner.startupProbe.httpGet }}
+    {{- if and $startup.port (ne $healthPort $startup.port) }}
+
+#####################################################################################
+####               ERROR: Cluster Checks Runner startup probe misconfiguration   ####
+#####################################################################################
+
+Cluster Checks Runner readiness probe port ({{ $startup.port }}) is different from the configured health port ({{ $healthPort }}).
+    {{- end }}
+    {{- end }}
+
   {{- end }}
 {{- end }}
 {{- if or .Values.datadog.apm.enabled .Values.datadog.apm.portEnabled }}

diff --git a/charts/datadog/templates/_container-agent.yaml b/charts/datadog/templates/_container-agent.yaml
@@ -330,4 +330,7 @@
   readinessProbe:
 {{- $ready := .Values.agents.containers.agent.readinessProbe }}
 {{ include "probe.http" (dict "path" "/ready" "port" $healthPort "settings" $ready) | indent 4 }}
+  startupProbe:
+{{- $startup := .Values.agents.containers.agent.startupProbe }}
+{{ include "probe.http" (dict "path" "/startup" "port" $healthPort "settings" $startup) | indent 4 }}
 {{- end -}}
diff --git a/charts/datadog/templates/agent-clusterchecks-deployment.yaml b/charts/datadog/templates/agent-clusterchecks-deployment.yaml
@@ -227,6 +227,9 @@ spec:
         readinessProbe:
 {{- $ready := .Values.clusterChecksRunner.readinessProbe }}
 {{ include "probe.http" (dict "settings" $ready "path" "/ready" "port" $healthPort) | indent 10 }}
+        startupProbe:
+{{- $startup := .Values.clusterChecksRunner.startupProbe }}
+{{ include "probe.http" (dict "settings" $startup "path" "/startup" "port" $healthPort) | indent 10 }}
       volumes:
         - name: installinfo
           configMap:

diff --git a/charts/datadog/templates/cluster-agent-deployment.yaml b/charts/datadog/templates/cluster-agent-deployment.yaml
@@ -368,6 +368,9 @@ spec:
         readinessProbe:
 {{- $ready := .Values.clusterAgent.readinessProbe }}
 {{ include "probe.http" (dict "path" "/ready" "port" $healthPort "settings" $ready) | indent 10 }}
+        startupProbe:
+{{- $startup := .Values.clusterAgent.startupProbe }}
+{{ include "probe.http" (dict "path" "/startup" "port" $healthPort "settings" $startup) | indent 10 }}
 {{- if .Values.clusterAgent.containers.clusterAgent.securityContext }}
         securityContext:
 {{ toYaml .Values.clusterAgent.containers.clusterAgent.securityContext | indent 10 }}

diff --git a/charts/datadog/values.yaml b/charts/datadog/values.yaml
@@ -1249,6 +1249,15 @@ clusterAgent:
     successThreshold: 1
     failureThreshold: 6
 
+  # clusterAgent.startupProbe -- Override default Cluster Agent startup probe settings
+  # @default -- Every 15s / 6 KO / 1 OK
+  startupProbe:
+    initialDelaySeconds: 15
+    periodSeconds: 15
+    timeoutSeconds: 5
+    successThreshold: 1
+    failureThreshold: 6
+
   # clusterAgent.strategy -- Allow the Cluster Agent deployment to perform a rolling update on helm update
 
   ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
@@ -1585,6 +1594,15 @@ agents:
         successThreshold: 1
         failureThreshold: 6
 
+      # agents.containers.agent.startupProbe -- Override default agent startup probe settings
+      # @default -- Every 15s / 6 KO / 1 OK
+      startupProbe:
+        initialDelaySeconds: 15
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 6
+
       # agents.containers.agent.securityContext -- Allows you to overwrite the default container SecurityContext for the agent container.
       securityContext: {}
 
@@ -2028,6 +2046,23 @@ clusterChecksRunner:
     successThreshold: 1
     failureThreshold: 6
 
+  # clusterChecksRunner.startupProbe -- Override default agent startup probe settings
+  # @default -- Every 15s / 6 KO / 1 OK
+
+  ## In case of issues with the probe, you can disable it with the
+  ## following values, to allow easier investigating:
+  #
+  # startupProbe:
+  #   exec:
+  #     command: ["/bin/true"]
+  #
+  startupProbe:
+    initialDelaySeconds: 15
+    periodSeconds: 15
+    timeoutSeconds: 5
+    successThreshold: 1
+    failureThreshold: 6
+
   # clusterChecksRunner.deploymentAnnotations -- Annotations to add to the cluster-checks-runner's Deployment
   deploymentAnnotations: {}
   #   key: "value"

diff --git a/test/datadog/baseline/agent-clusterchecks-deployment_default.yaml b/test/datadog/baseline/agent-clusterchecks-deployment_default.yaml
@@ -160,6 +160,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /startup
+            port: 5557
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
       volumes:
         - name: installinfo
           configMap:

diff --git a/test/datadog/baseline/cluster-agent-deployment_default.yaml b/test/datadog/baseline/cluster-agent-deployment_default.yaml
@@ -189,6 +189,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /live
+            port: 5556
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

diff --git a/test/datadog/baseline/cluster-agent-deployment_default_advanced_AC_injection.yaml b/test/datadog/baseline/cluster-agent-deployment_default_advanced_AC_injection.yaml
@@ -203,6 +203,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /startup
+            port: 5556
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

diff --git a/test/datadog/baseline/cluster-agent-deployment_default_minimal_AC_injection.yaml b/test/datadog/baseline/cluster-agent-deployment_default_minimal_AC_injection.yaml
@@ -199,6 +199,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /live
+            port: 5556
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

diff --git a/test/datadog/baseline/daemonset_default.yaml b/test/datadog/baseline/daemonset_default.yaml
@@ -181,6 +181,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /startup
+            port: 5555
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
       - name: trace-agent
         image: "gcr.io/datadoghq/agent:7.54.0"
         imagePullPolicy: IfNotPresent

diff --git a/test/datadog/baseline/other_default.yaml b/test/datadog/baseline/other_default.yaml
@@ -969,6 +969,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /startup
+            port: 5555
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
       - name: trace-agent
         image: "gcr.io/datadoghq/agent:7.54.0"
         imagePullPolicy: IfNotPresent
@@ -1437,6 +1447,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /live
+            port: 5557
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
       volumes:
         - name: installinfo
           configMap:
@@ -1647,6 +1667,16 @@ spec:
           periodSeconds: 15
           successThreshold: 1
           timeoutSeconds: 5
+        startupProbe:
+          failureThreshold: 6
+          httpGet:
+            path: /startup
+            port: 5556
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          successThreshold: 1
+          timeoutSeconds: 5
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true