Merge branch 'main' into clamoriniere/kind-config

.gitlab-ci.yml

@@ -1,7 +1,3 @@
-variables:
-  # Do not modify this - must be the repository name for Kubernetes gitlab runners to run
-  KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: helm-charts
-
 stages:
   - e2e
 
@@ -14,22 +10,32 @@ e2e:
           - charts/**/*.yaml
           - test/**/*
         compare_to: "refs/heads/main"
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:986d192b
+      when: always
+    - if: $CI_COMMIT_BRANCH != "main"
+      when: manual
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:95dca87f269a
   tags: ["arch:amd64"]
+  variables:
+    AWS_KEYPAIR_NAME: datadog-agent-ci
+    AWS_PRIVATE_KEY_FILE: $CI_PROJECT_DIR/ssh_key
+    KUBERNETES_CPU_REQUEST: 2
+    KUBERNETES_MEMORY_REQUEST: 4Gi
+    KUBERNETES_MEMORY_LIMIT: 12Gi
   before_script:
-    # Update awscli v2
-    - aws --version
-    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip awscliv2.zip
-    - ./aws/install --update
-    - aws --version
-    # Setup AWS Credentials
     - mkdir -p ~/.aws
     - set +x
+    # Set GITHUB_TOKEN to avoid getting rate-limited when pulumi sdk downloads the kubernetes provider
+    - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.github_token --with-decryption --query "Parameter.Value" --out text)
+    # Configure AWS EC2 ssh key needed for create pulumi EKS environment
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_key --with-decryption --query "Parameter.Value" --out text > $AWS_PRIVATE_KEY_FILE
+    - set -x
+    # Without the newline ssh silently fails and moves on to try other auth methods
+    - echo "" >> $AWS_PRIVATE_KEY_FILE
+    - chmod 600 $AWS_PRIVATE_KEY_FILE
+    # Configure AWS profile
     - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.e2e-agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
     - set -x
     - export AWS_PROFILE=agent-qa-ci
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   script:
-    # TODO: enable running E2E tests when upstream pulumi-eks fix is merged https://github.com/pulumi/pulumi-eks/pull/886
-    - echo "Success!"
+    - make test-e2e

Makefile

@@ -46,15 +46,15 @@ vet:
 
 .PHONY: unit-test
 unit-test:
-	go test -C test ./... -count=1 -skip=E2E
+	go test -C test ./... -count=1
 
 .PHONY: update-test-baselines
 update-test-baselines:
-	go test -C test ./... -count=1 -skip=E2E -args -updateBaselines=true
+	go test -C test ./... -count=1 -args -updateBaselines=true
 
 .PHONY: integration-test
 integration-test:
-	go test -C test/integ --tags=integration -skip=E2E -count=1 -v
+	go test -C test/integ --tags=integration -count=1 -v
 
 # Running E2E tests locally:
 ## Must be connected to appgate
@@ -67,14 +67,14 @@ test-e2e: fmt vet e2e-test
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test
 .PHONY: e2e-test
 e2e-test:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-preserve-stacks
 .PHONY: e2e-test-preserve-stacks
 e2e-test-preserve-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -preserveStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -preserveStacks=true
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-cleanup-stacks
 .PHONY: e2e-test-cleanup-stacks
 e2e-test-cleanup-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -destroyStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -destroyStacks=true

charts/datadog/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Datadog changelog
 
+## 3.57.2
+
+* Add `networkpolicies` default permission for the cluster agent.
+
+## 3.57.1
+
+* Allow configuring CWS security profile based auto suppression feature and enable it by default.
+
+## 3.57.0
+
+* Set default `Agent` and `Cluster-Agent` version to `7.51.0`.
+
 ## 3.56.0
 
 * Allow templating of `datadog.clusterName`.

charts/datadog/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: datadog
-version: 3.56.0
+version: 3.57.2
 appVersion: "7"
 description: Datadog Agent
 keywords:

charts/datadog/README.md

@@ -1,6 +1,6 @@
 # Datadog
 
-![Version: 3.56.0](https://img.shields.io/badge/Version-3.56.0-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
+![Version: 3.57.2](https://img.shields.io/badge/Version-3.57.2-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
 
 [Datadog](https://www.datadoghq.com/) is a hosted infrastructure monitoring platform. This chart adds the Datadog Agent to all nodes in your cluster via a DaemonSet. It also optionally depends on the [kube-state-metrics chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). For more information about monitoring Kubernetes with Datadog, please refer to the [Datadog documentation website](https://docs.datadoghq.com/agent/basic_agent_usage/kubernetes/).
 
@@ -508,7 +508,7 @@ helm install <RELEASE_NAME> \
 | agents.image.pullPolicy | string | `"IfNotPresent"` | Datadog Agent image pull policy |
 | agents.image.pullSecrets | list | `[]` | Datadog Agent repository pullSecret (ex: specify docker registry credentials) |
 | agents.image.repository | string | `nil` | Override default registry + image.name for Agent |
-| agents.image.tag | string | `"7.50.3"` | Define the Agent version to use |
+| agents.image.tag | string | `"7.51.0"` | Define the Agent version to use |
 | agents.image.tagSuffix | string | `""` | Suffix to append to Agent tag |
 | agents.localService.forceLocalServiceEnabled | bool | `false` | Force the creation of the internal traffic policy service to target the agent running on the local node. By default, the internal traffic service is created only on Kubernetes 1.22+ where the feature became beta and enabled by default. This option allows to force the creation of the internal traffic service on kubernetes 1.21 where the feature was alpha and required a feature gate to be explicitly enabled. |
 | agents.localService.overrideName | string | `""` | Name of the internal traffic service to target the agent running on the local node |
@@ -574,7 +574,7 @@ helm install <RELEASE_NAME> \
 | clusterAgent.image.pullPolicy | string | `"IfNotPresent"` | Cluster Agent image pullPolicy |
 | clusterAgent.image.pullSecrets | list | `[]` | Cluster Agent repository pullSecret (ex: specify docker registry credentials) |
 | clusterAgent.image.repository | string | `nil` | Override default registry + image.name for Cluster Agent |
-| clusterAgent.image.tag | string | `"7.50.3"` | Cluster Agent image tag to use |
+| clusterAgent.image.tag | string | `"7.51.0"` | Cluster Agent image tag to use |
 | clusterAgent.livenessProbe | object | Every 15s / 6 KO / 1 OK | Override default Cluster Agent liveness probe settings |
 | clusterAgent.metricsProvider.aggregator | string | `"avg"` | Define the aggregator the cluster agent will use to process the metrics. The options are (avg, min, max, sum) |
 | clusterAgent.metricsProvider.createReaderRbac | bool | `true` | Create `external-metrics-reader` RBAC automatically (to allow HPA to read data from Cluster Agent) |
@@ -625,7 +625,7 @@ helm install <RELEASE_NAME> \
 | clusterChecksRunner.image.pullPolicy | string | `"IfNotPresent"` | Datadog Agent image pull policy |
 | clusterChecksRunner.image.pullSecrets | list | `[]` | Datadog Agent repository pullSecret (ex: specify docker registry credentials) |
 | clusterChecksRunner.image.repository | string | `nil` | Override default registry + image.name for Cluster Check Runners |
-| clusterChecksRunner.image.tag | string | `"7.50.3"` | Define the Agent version to use |
+| clusterChecksRunner.image.tag | string | `"7.51.0"` | Define the Agent version to use |
 | clusterChecksRunner.image.tagSuffix | string | `""` | Suffix to append to Agent tag |
 | clusterChecksRunner.livenessProbe | object | Every 15s / 6 KO / 1 OK | Override default agent liveness probe settings |
 | clusterChecksRunner.networkPolicy.create | bool | `false` | If true, create a NetworkPolicy for the cluster checks runners. DEPRECATED. Use datadog.networkPolicy.create instead |
@@ -785,6 +785,7 @@ helm install <RELEASE_NAME> \
 | datadog.securityAgent.runtime.network.enabled | bool | `true` | Set to true to enable the collection of CWS network events |
 | datadog.securityAgent.runtime.policies.configMap | string | `nil` | Contains CWS policies that will be used |
 | datadog.securityAgent.runtime.securityProfile.anomalyDetection.enabled | bool | `true` | Set to true to enable CWS runtime drift events |
+| datadog.securityAgent.runtime.securityProfile.autoSuppression.enabled | bool | `true` | Set to true to enable CWS runtime auto suppression |
 | datadog.securityAgent.runtime.securityProfile.enabled | bool | `true` | Set to true to enable CWS runtime security profiles |
 | datadog.securityAgent.runtime.syscallMonitor.enabled | bool | `false` | Set to true to enable the Syscall monitoring (recommended for troubleshooting only) |
 | datadog.securityContext | object | `{"runAsUser":0}` | Allows you to overwrite the default PodSecurityContext on the Daemonset or Deployment |

charts/datadog/templates/cluster-agent-rbac.yaml

@@ -171,6 +171,7 @@ rules:
   - networking.k8s.io
   resources:
   - ingresses
+  - networkpolicies
   verbs:
   - list
   - get

charts/datadog/templates/system-probe-configmap.yaml

@@ -73,7 +73,7 @@ data:
         anomaly_detection:
           enabled: {{ $.Values.datadog.securityAgent.runtime.securityProfile.anomalyDetection.enabled }}
         auto_suppression:
-          enabled: false
+          enabled: {{ $.Values.datadog.securityAgent.runtime.securityProfile.autoSuppression.enabled }}
 
 {{- if eq .Values.datadog.systemProbe.seccomp "localhost/system-probe" }}
 ---

charts/datadog/values.yaml

@@ -783,6 +783,10 @@ datadog:
           # datadog.securityAgent.runtime.securityProfile.anomalyDetection.enabled -- Set to true to enable CWS runtime drift events
           enabled: true
 
+        autoSuppression:
+          # datadog.securityAgent.runtime.securityProfile.autoSuppression.enabled -- Set to true to enable CWS runtime auto suppression
+          enabled: true
+
   ## Manage NetworkPolicy
   networkPolicy:
     # datadog.networkPolicy.create -- If true, create NetworkPolicy for all the components
@@ -887,7 +891,7 @@ clusterAgent:
     name: cluster-agent
 
     # clusterAgent.image.tag -- Cluster Agent image tag to use
-    tag: 7.50.3
+    tag: 7.51.0
 
     # clusterAgent.image.digest -- Cluster Agent image digest to use, takes precedence over tag if specified
     digest: ""
@@ -1315,7 +1319,7 @@ agents:
     name: agent
 
     # agents.image.tag -- Define the Agent version to use
-    tag: 7.50.3
+    tag: 7.51.0
 
     # agents.image.digest -- Define Agent image digest to use, takes precedence over tag if specified
     digest: ""
@@ -1783,7 +1787,7 @@ clusterChecksRunner:
     name: agent
 
     # clusterChecksRunner.image.tag -- Define the Agent version to use
-    tag: 7.50.3
+    tag: 7.51.0
 
     # clusterChecksRunner.image.digest -- Define Agent image digest to use, takes precedence over tag if specified
     digest: ""

test/common/common_e2e.go

@@ -9,10 +9,10 @@ import (
 	"os"
 	"strings"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/utils/infra"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/eks"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -23,12 +23,13 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
-var defaultLocalPulumiConfigs = runner.ConfigMap{
+var defaultPulumiConfigs = runner.ConfigMap{
 	"ddinfra:aws/defaultKeyPairName": auto.ConfigValue{Value: os.Getenv("AWS_KEYPAIR_NAME")},
 }
 var defaultCIPulumiConfigs = runner.ConfigMap{
-	"aws:skipCredentialsValidation": auto.ConfigValue{Value: "true"},
-	"aws:skipMetadataApiCheck":      auto.ConfigValue{Value: "false"},
+	"aws:skipCredentialsValidation":     auto.ConfigValue{Value: "true"},
+	"aws:skipMetadataApiCheck":          auto.ConfigValue{Value: "false"},
+	"ddinfra:aws/defaultPrivateKeyPath": auto.ConfigValue{Value: os.Getenv("AWS_PRIVATE_KEY_FILE")},
 }
 
 type E2EEnv struct {
@@ -101,6 +102,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 	res := runner.ConfigMap{}
 	configs := parseE2EConfigParams()
 	if os.Getenv("E2E_PROFILE") == "ci" {
+		res.Merge(defaultPulumiConfigs)
 		res.Merge(defaultCIPulumiConfigs)
 	} else {
 		// use "local" E2E profile for local testing
@@ -111,7 +113,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 		if !e2eApiKeyBool || !e2eAppKeyBool || !e2eAwsKeypairNameBool {
 			return nil, fmt.Errorf("missing required environment variables. Must set `E2E_API_KEY`, `E2E_APP_KEY`, and `AWS_KEYPAIR_NAME` for the local E2E profile")
 		} else {
-			res.Merge(defaultLocalPulumiConfigs)
+			res.Merge(defaultPulumiConfigs)
 		}
 	}
 

test/datadog/e2e_eks_test.go → test/e2e/datadog/e2e_eks_test.go

@@ -1,17 +1,21 @@
+//go:build e2e
+// +build e2e
+
 package datadog
 
 import (
 	"fmt"
 	"testing"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/runner"
-	"github.com/DataDog/helm-charts/test/common"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+
+	"github.com/DataDog/helm-charts/test/common"
 )
 
 const namespace = "datadog"
@@ -30,8 +34,6 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 		"ddtestworkload:deploy":                      auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/linuxBottlerocketNodeGroup": auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/windowsNodeGroup":           auto.ConfigValue{Value: "false"},
-		// TODO: remove when upstream eks-pulumi bug is fixed https://github.com/pulumi/pulumi-eks/pull/886
-		"pulumi:disable-default-providers": auto.ConfigValue{Value: "[]"},
 	}
 	stackConfig.Merge(config)
 
@@ -48,14 +50,14 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 			kc := kubeconfig.Value.(map[string]interface{})
 			_, restConfig, k8sClient, err = common.NewClientFromKubeconfig(kc)
 			if err == nil {
-				verifyPods(t)
+				t.Run("Agent pods should be created", verifyPods)
 			}
 		} else {
 			err = fmt.Errorf("could not create Kubernetes client, cluster kubeconfig is nil")
 		}
 	}
 	if err != nil {
-		t.Skipf("Skipping test. Encountered problem creating or updating E2E stack: %s", err)
+		t.Errorf("Skipping tests. Encountered problem creating or updating E2E stack: %s", err)
 	}
 }
 
@@ -79,10 +81,15 @@ func verifyPods(t *testing.T) {
 		RestConfig: restConfig,
 	}
 
-	assertPodStatus(t, podExec, ddaPodList, "agent")
-	assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
-	assertPodStatus(t, podExec, ccPodList, "agent")
-
+	t.Run("exec `agent status` for `agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ddaPodList, "agent")
+	})
+	t.Run("`exec `agent status` for `cluster-agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
+	})
+	t.Run("exec `agent status` for `cluster-check-runner` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ccPodList, "agent")
+	})
 }
 
 func assertPodStatus(t *testing.T, podExec common.K8sExec, podList *v1.PodList, containerName string) {