feat(testing): automate PR checks with trace based tests

.github/workflows/pr-checks.yaml

@@ -0,0 +1,68 @@
+name: Create Cluster
+
+on: pull_request
+
+jobs:
+  tracetests:
+    name: Run tracetests on a KinD cluster
+    runs-on: ubuntu-latest
+    env:
+      TESTS_REPO_DIR: "/tmp/tests"
+      TRACETEST_LICENSE: ${{ secrets.TRACETEST_ONPREM_TEST_LICENSE }}
+      AGENT_API_KEY: ${{ secrets.AGENT_API_KEY }}
+      AGENT_ENV_ID: ${{ secrets.AGENT_ENV_ID }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup KinD
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: tracetest
+          install_only: true
+
+      - name: Test public images
+        run: |
+          ./scripts/setup_kind_cluster.sh --ci --build-deps  --install-demo --force-setup
+          ./scripts/setup_ci_host.sh
+
+      - name: Configure intial org and env
+        env:
+          TRACETEST_TEST_REPO_CLONE_PAT: ${{ secrets.TRACETEST_TEST_REPO_CLONE_PAT }}
+          TRACETEST_TEST_REPO: ${{ secrets.TRACETEST_TEST_REPO }}
+          GH_USERNAME: ${{ secrets.TRACETEST_USER_GH_USERNAME }}
+          GH_PASSWORD: ${{ secrets.TRACETEST_USER_GH_PASSWORD }}
+          GH_SECRET: ${{ secrets.TRACETEST_USER_GH_SECRET }}
+          GH_MEMBER_EMAIL: ${{ secrets.TRACETEST_MEMBER_GH_EMAIL }}
+          GH_MEMBER_USERNAME: ${{ secrets.TRACETEST_MEMBER_GH_USERNAME }}
+          GH_MEMBER_PASSWORD: ${{ secrets.TRACETEST_MEMBER_GH_PASSWORD }}
+          GH_MEMBER_SECRET: ${{ secrets.TRACETEST_MEMBER_GH_SECRET }}
+          TRACETEST_ENDPOINT: https://tracetest.localdev:30000/
+          OUTPUT_TOKEN_FILE: "/tmp/token.json"
+        run: |
+          BASE_DIR=$(pwd)
+
+          # sparse checkout of the testing directory
+          git clone --depth 1 --filter=blob:none --sparse https://[email protected]/$TRACETEST_TEST_REPO.git $TESTS_REPO_DIR
+          cd $TESTS_REPO_DIR
+          git sparse-checkout set testing/
+
+          $BASE_DIR/scripts/prepare_ci_env.sh
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: images
+          path: /tmp/tests/testing/e2e-tracetesting/test-results
+          retention-days: 90
+
+      - name: Configure Tracetest CLI
+        uses: kubeshop/tracetest-github-action@v1
+        with:
+          token: ${{secrets.TRACETEST_CLI_TOKEN}}
+
+      - name: run tracetests
+        run: |
+          tracetest run testsuite -f $TESTS_REPO_DIR/testing/server-tracetesting/features --vars ./tests/vars/ci.yaml

.gitignore

@@ -1,6 +1,6 @@
 **/*/Chart.lock
 **/*/charts/*
 cluster.env
-tracetest.kubeconfig
-.env
+*.env
+*tracetest.kubeconfig
 *.bkp

charts/tracetest-agent/templates/deployment.yaml

@@ -22,6 +22,9 @@ spec:
           imagePullPolicy: IfNotPresent
 
           args: [
+            "--mode", "ci",
+            "-v",
+
             {{- if .Values.agent.environmentId }}
             "--environment",
             "{{ .Values.agent.environmentId }}",

charts/tracetest-core/values.yaml

@@ -34,6 +34,7 @@ config:
     sampling: 50
 
   analytics: false
+  # collectorEndpoint: "some-collector.namespace:4317"
 
 server:
   httpPort: *httpPort

kind-config.yaml

@@ -6,6 +6,4 @@ nodes:
   - containerPort: 30000
     hostPort: 30000
     protocol: TCP
-- role: worker
-- role: worker
 - role: worker

scripts/prepare_ci_env.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$*" == *"--debug"* ]]; then
+  set -x
+fi
+
+BASE_DIR=$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")
+
+if [ -z "$TESTS_REPO_DIR" ]; then
+  echo "Error: TESTS_REPO_DIR is not set."
+  exit 1
+fi
+
+if [ -z "$OUTPUT_TOKEN_FILE" ]; then
+  echo "Error: OUTPUT_TOKEN_FILE is not set."
+  exit 1
+fi
+
+
+
+cd $TESTS_REPO_DIR/testing/e2e-tracetesting
+
+if [[ "$*" != *"--skip-playwright"* ]]; then
+  npm install
+  npx playwright test tests/setup/extract-default-token.manual.ts --grep '@manual'
+fi
+
+############################################
+############################################
+
+cd $BASE_DIR
+
+# run script for initial configuration
+
+export VARS_TOKEN=$(cat $OUTPUT_TOKEN_FILE | jq -r '.token')
+sed -i.bkp 's/%TOKEN_PLACEHOLDER%/'"$VARS_TOKEN"'/g' tests/vars/ci.yaml
+
+export VARS_ORG_ID=$(cat $OUTPUT_TOKEN_FILE | jq -r '.orgId')
+sed -i.bkp 's/%ORG_ID_PLACEHOLDER%/'"$VARS_ORG_ID"'/g' tests/vars/ci.yaml
+
+export VARS_ENV_ID=$(cat $OUTPUT_TOKEN_FILE | jq -r '.envId')
+sed -i.bkp 's/%ENV_ID_PLACEHOLDER%/'"$VARS_ENV_ID"'/g' tests/vars/ci.yaml
+
+export VARS_COOKIE=$(cat $OUTPUT_TOKEN_FILE | jq -r '.cookieHeader' | sed 's/[&/\]/\\&/g')
+sed -i.bkp 's/%COOKIE_PLACEHOLDER%/'"$VARS_COOKIE"'/g' tests/vars/ci.yaml
+
+export VARS_INVITE_EMAIL=$(cat $OUTPUT_TOKEN_FILE | jq -r '.inviteEmail')
+sed -i.bkp 's/%INVITE_EMAIL_PLACEHOLDER%/'"$VARS_INVITE_EMAIL"'/g' tests/vars/ci.yaml

scripts/setup_ci_host.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+printf "\n\e[42m\e[1mPersiste kubeconfig file\e[0m\e[0m\n"
+kind get kubeconfig --name tracetest > ~/.kube/config
+
+printf "\n\e[42m\e[1mConfigure host DNS\e[0m\e[0m\n"
+# setup custom DNS resolve
+sudo echo "127.0.0.1 tracetest.localdev" | sudo tee -a /etc/hosts
+sudo echo "127.0.0.1 pokeshop.localdev" | sudo tee -a /etc/hosts
+
+kubectl wait --for=condition=available --timeout=60s deployment/cloudagent-tracetest-agent
+printf "\n\e[42m\e[1mCloudAgent deployed\e[0m\e[0m\n"

scripts/setup_kind_cluster.sh

@@ -5,6 +5,7 @@ set -e
 function show_help() {
   echo "Usage: setup_kind_cluster.sh [OPTIONS]"
   echo "Options:"
+  echo "  --ci                 Setup cluster for a CI environment"
   echo "  --reset              Reset the existing kind cluster"
   echo "  --private            Use private repositories. Requires a PAT with read:packages scope."
   echo "  --build-deps         Build dependencies for all charts"
@@ -15,6 +16,8 @@ function show_help() {
   echo ""
   echo "Environment variables that might be read:"
   echo "  TRACETEST_LICENSE    OnPrem license key (if not provided, the script will prompt for it)"
+  echo "  AGENT_API_KEY        Cloud Agent API key"
+  echo "  AGENT_ENV_ID         Cloud Agent environment ID"
 }
 
 if [[ "$@" == *"--help"* ]]; then
@@ -26,6 +29,11 @@ PROJECT_ROOT=$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")
 KUBECONFIG_FILE=$PROJECT_ROOT/tracetest.kubeconfig
 ENV_FILE=$PROJECT_ROOT/cluster.env
 HELM_EXTRA_FLAGS=()
+VALUES_FILE=values-kind.yaml
+
+if [[ "$@" == *"--ci"* ]]; then
+  VALUES_FILE=values-kind-ci.yaml
+fi
 
 if [[ "$@" == *"--debug"* ]]; then
   set -x
@@ -68,12 +76,23 @@ else
   printf "\n\e[1mCluster already exists\e[0m\n"
 fi
 
-cat <<EOF > $ENV_FILE
+# the kind cluster might have been created from outside this script, as in CI pipelines.
+# if that's the case, leave kubeconfig alone
+if [[ -f $KUBECONFIG_FILE ]]; then
+  cat <<EOF > $ENV_FILE
 export KUBECONFIG=$KUBECONFIG:$KUBECONFIG_FILE
 kubectl config use-context kind-tracetest
 EOF
 
-source $ENV_FILE
+  source $ENV_FILE
+else
+  printf "\n\e[43m\e[1mCustom Kubeconfig file not detected, use default kubectl config\e[0m\e[0m\n"
+fi
+
+if [[ "$@" == *"--force-setup"* ]]; then
+  printf "\n\e[43m\e[1mForce setup flag detected\e[0m\e[0m\n"
+  SETUP_CLUSTER=true
+fi
 
 if [[ "$SETUP_CLUSTER" == true ]]; then
 
@@ -116,10 +135,10 @@ fi
 printf "\n\e[42m\e[1mStarting Tracetest OnPrem components installation\e[0m\e[0m\n"
 
 printf "\n\e[42m\e[1mInstalling Tracetest dependencies\e[0m\e[0m\n"
-helm upgrade --install ttdeps $PROJECT_ROOT/charts/tracetest-dependencies -f $PROJECT_ROOT/values-kind.yaml "${HELM_EXTRA_FLAGS[@]}"
+helm upgrade --install ttdeps $PROJECT_ROOT/charts/tracetest-dependencies -f $PROJECT_ROOT/$VALUES_FILE "${HELM_EXTRA_FLAGS[@]}"
 
 printf "\n\e[42m\e[1mInstalling Tracetest on-prem\e[0m\e[0m\n"
-helm upgrade --install tt $PROJECT_ROOT/charts/tracetest-onprem -f $PROJECT_ROOT/values-kind.yaml "${HELM_EXTRA_FLAGS[@]}"
+helm upgrade --install tt $PROJECT_ROOT/charts/tracetest-onprem -f $PROJECT_ROOT/$VALUES_FILE "${HELM_EXTRA_FLAGS[@]}"
 
 if [[ "$@" == *"--install-demo"* ]]; then
   printf "\n\e[42m\e[1mInstalling Pokeshop demo\e[0m\e[0m\n"
@@ -137,6 +156,27 @@ if [[ "$SETUP_CLUSTER" == true || "$@" == *"--config-coredns"* ]]; then
   printf "\n"
 fi
 
+if [[ "$@" == *"--ci"* ]]; then
+  printf "\n\e[42m\e[1mInstalling CI Agent\e[0m\e[0m\n"
+  # Validate required environment variables
+  if [[ -z "$AGENT_API_KEY" ]]; then
+    printf "\e[41mError: AGENT_API_KEY environment variable is not set\e[0m\n"
+    exit 1
+  fi
+
+  if [[ -z "$AGENT_ENV_ID" ]]; then
+    printf "\e[41mError: AGENT_ENV_ID environment variable is not set\e[0m\n"
+    exit 1
+  fi
+
+  helm upgrade --install cloudagent ./charts/tracetest-agent \
+    -n default \
+    --set agent.apiKey="$AGENT_API_KEY" \
+    --set agent.environmentId="$AGENT_ENV_ID"
+
+  printf "\n"
+fi
+
 # Capturing the end time
 end_time=$(date +%s)
 

tests/vars/ci.yaml

@@ -0,0 +1,30 @@
+type: VariableSet
+spec:
+  id: helm-ci
+  name: Helm CI
+  values:
+  - key: BASE_URL
+    value: https://tracetest.localdev:30000
+    type: raw
+  - key: TOKEN
+    value: "%TOKEN_PLACEHOLDER%"
+    type: raw
+  - key: INVITE_EMAIL
+    value: "%INVITE_EMAIL_PLACEHOLDER%"
+  - key: INVITED_COOKIE
+    value: "%COOKIE_PLACEHOLDER%"
+    type: raw
+  - key: ORG_ID
+    value: "%ORG_ID_PLACEHOLDER%"
+    type: raw
+  - key: ENV_ID
+    value: "%ENV_ID_PLACEHOLDER%"
+    type: raw
+  - key: POKESHOP_URL
+    value: https://pokeshop.localdev:30000
+    type: raw
+  # This placeholder exists because when we try to run the tests testsuite,
+  # the cli will ask for the
+  - key: CREATED_TEST_ID
+    value: PLACEHOLDER
+    type: raw

values-kind-ci.yaml

@@ -0,0 +1,63 @@
+global:
+  validCertificate: false
+  urls:
+    rootDomain: &rootDomain "tracetest.localdev"
+
+  postgresql:
+    auth:
+      host: "ttdeps-postgresql"
+      username: "postgres"
+      password: "postgres"
+      database: "tracetest"
+
+  mongodb:
+    auth:
+      protocol: "mongodb"
+      host: "ttdeps-tracetest-dependencies-mongodb"
+      username: "mongodb"
+      password: "mongodb"
+      database: "tracetest"
+      options:
+        retryWrites: "true"
+        authSource: admin
+
+
+traefik:
+  service:
+    type: NodePort
+
+  dnsNames:
+    - *rootDomain
+    - "pokeshop.localdev"
+
+  tls:
+    issuerRef:
+      name: issuer-selfsigned
+      kind: ClusterIssuer
+      group: cert-manager.io
+
+tracetest-core:
+  config:
+    telemetry:
+      enabled: true
+      collector:
+        endpoint: "cloudagent-tracetest-agent.default:4318"
+
+tracetest-cloud:
+  config:
+    telemetry:
+      enabled: true
+      collector:
+        endpoint: "cloudagent-tracetest-agent.default:4317"
+
+tracetest-agent-operator:
+  config:
+    telemetry:
+      collector:
+        endpoint: "cloudagent-tracetest-agent.default:4317"
+
+tracetest-monitor-operator:
+  config:
+    telemetry:
+      collector:
+        endpoint: "cloudagent-tracetest-agent.default:4317"