Fix clusters

.github/custom-tests/comparison/all/cluster.tf

@@ -0,0 +1,19 @@
+project                   = "ce-team-zaid"
+cluster_location          = "us-west1-a"
+gke_version               = ""
+cluster_machine_type      = "c2-standard-4"
+service_machine_type      = ""
+upstream_machine_type     = "c2-standard-8"
+tests_machine_type        = "c2-standard-8"
+resources_machine_type    = ""
+dependencies_machine_type = "c2-standard-8"
+
+services_nodes_count     = 1
+upstream_nodes_count     = 1
+tests_nodes_count        = 1
+resource_nodes_count     = 1
+dependencies_nodes_count = 2
+
+tyk_enabled      = true
+kong_enabled     = true
+gravitee_enabled = true

.github/custom-tests/comparison/all/deployments.tf

@@ -0,0 +1,41 @@
+kubernetes_config_context = "performance-testing"
+
+analytics_database_enabled    = true
+analytics_prometheus_enabled  = true
+auth_enabled                  = true
+auth_type                     = "authToken"
+quota_enabled                 = false
+quota_rate                    = 999999
+quota_per                     = 3600
+rate_limit_enabled            = true
+rate_limit_rate               = 999999
+rate_limit_per                = 60
+open_telemetry_enabled        = false
+open_telemetry_sampling_ratio = "0.5"
+
+hpa_enabled                 = false
+hpa_max_replica_count       = 10
+replica_count               = 4
+hpa_avg_cpu_util_percentage = 80
+external_traffic_policy     = "local"
+resources_requests_cpu      = "750m"
+resources_requests_memory   = "2048Mi"
+resources_limits_cpu        = "750m"
+resources_limits_memory     = "2048Mi"
+
+tyk_enabled         = true
+tyk_version         = "v5.6"
+tyk_deployment_type = "Deployment"
+tyk_go_gc           = 1600
+tyk_go_max_procs    = 8
+
+kong_enabled         = true
+kong_version         = "3.6"
+kong_deployment_type = "Deployment"
+
+gravitee_enabled         = true
+gravitee_version         = "4.4"
+gravitee_deployment_type = "Deployment"
+gravitee_nginx_enabled   = false
+
+grafana_service_type = "ClusterIP"

.github/custom-tests/comparison/all/tests.tf

@@ -0,0 +1,12 @@
+tyk_enabled      = true
+kong_enabled     = true
+gravitee_enabled = true
+
+tests_fortio_options = "size=20"
+tests_executor       = "constant-arrival-rate"
+tests_auth_key_count = 10
+tests_ramping_steps  = 10
+tests_duration       = 120
+tests_rate           = 5000
+tests_virtual_users  = 500
+tests_parallelism    = 4

.github/custom-tests/tyk-open-telemetry/cluster.tf

@@ -1,5 +1,5 @@
 cluster_location          = "westus"
-aks_version               = "1.29"
+aks_version               = null
 cluster_machine_type      = "Standard_F8s_v2"
 service_machine_type      = ""
 upstream_machine_type     = ""

.github/custom-tests/tyk-open-telemetry/deployments.tf

@@ -23,7 +23,7 @@ resources_limits_cpu        = "0"
 resources_limits_memory     = "0"
 
 tyk_enabled         = true
-tyk_version         = "v5.4"
+tyk_version         = "v5.6"
 tyk_deployment_type = "Deployment"
 tyk_go_gc           = 1600
 tyk_go_max_procs    = 8

.github/workflows/cluster.yml

@@ -85,11 +85,11 @@ jobs:
           cd aks
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="cluster_location=${{ vars.AZURE_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.AZURE_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.AZURE_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: EKS cluster ${{ inputs.action }}
@@ -98,11 +98,11 @@ jobs:
           cd eks
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="cluster_location=${{ vars.AWS_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.AWS_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.AWS_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: GKE cluster ${{ inputs.action }}
@@ -111,10 +111,10 @@ jobs:
           cd gke
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="project=${{ secrets.GCP_PROJECT }}" \
             --var="cluster_location=${{ vars.GCP_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.GCP_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.GCP_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve

.github/workflows/custom_performance_test.yml

@@ -0,0 +1,202 @@
+name: Custom Performance Test
+on:
+  workflow_dispatch:
+    inputs:
+      cloud:
+        description: 'Choose Cloud Provider'
+        required: true
+        type: choice
+        default: Azure
+        options:
+          - Azure
+          - AWS
+          - GCP
+      custom_cluster_descriptor:
+        description: 'Custom Cluster Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/cluster.tf'
+      custom_deployments_descriptor:
+        description: 'Custom Deployments Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/deployments.tf'
+      custom_tests_descriptor:
+        description: 'Custom Tests Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/tests.tf'
+
+env:
+  provider: ${{ inputs.cloud == 'Azure' && 'aks' || (inputs.cloud == 'AWS' && 'eks' || 'gke') }}
+
+concurrency:
+  group: ${{ inputs.cloud }}
+
+jobs:
+  performance_test:
+    name: "${{ inputs.cloud }} custom full performance test run"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Configure AKS credentials
+        if: ${{ inputs.cloud == 'Azure' }}
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Configure AWS credentials
+        if: ${{ inputs.cloud == 'AWS' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_CLUSTER_LOCATION }}
+
+      - name: Authenticate into gcloud
+        if: ${{ inputs.cloud == 'GCP' }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+
+      - name: Install gcloud CLI
+        if: ${{ inputs.cloud == 'GCP' }}
+        uses: google-github-actions/[email protected]
+
+      - name: Install gcloud k8s auth component
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: gcloud components install gke-gcloud-auth-plugin
+
+      - name: Install Terraform
+        uses: hashicorp/[email protected]
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: "1.8.2"
+
+      - name: Create Terraform Cloud descriptors
+        run: |
+          cp ${{ env.provider }}/terraform.cloud.tf.example ${{ env.provider }}/terraform.cloud.tf
+          cp .github/${{ env.provider }}/deployments.tf deployments/terraform.cloud.tf
+          cp .github/${{ env.provider }}/tests.tf tests/terraform.cloud.tf
+
+      - name: Copy Terraform descriptor
+        run: |
+          cp ${{ inputs.custom_cluster_descriptor }} ${{ env.provider }}/main.tfvars
+          cp ${{ inputs.custom_deployments_descriptor }} deployments/main.tfvars
+          cp ${{ inputs.custom_tests_descriptor }} tests/main.tfvars
+
+      - name: AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: |
+          cd aks
+          terraform init
+          terraform apply \
+            --var="cluster_location=${{ vars.AZURE_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.AZURE_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.AZURE_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: |
+          cd eks
+          terraform init
+          terraform apply \
+            --var="cluster_location=${{ vars.AWS_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.AWS_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.AWS_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: |
+          cd gke
+          terraform init
+          terraform apply \
+            --var="project=${{ secrets.GCP_PROJECT }}" \
+            --var="cluster_location=${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.GCP_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.GCP_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: Connect to AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: |
+          az aks get-credentials \
+            --resource-group "pt-${{ vars.AZURE_CLUSTER_LOCATION }}" \
+            --name "pt-${{ vars.AZURE_CLUSTER_LOCATION }}"
+
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Connect to EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: |
+          aws eks --region "${{ vars.AWS_CLUSTER_LOCATION }}" update-kubeconfig --name "pt-${{ vars.AWS_CLUSTER_LOCATION }}"
+
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Connect to GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: |
+          gcloud container clusters get-credentials "pt-${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --zone "${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --project "${{ secrets.GCP_PROJECT }}"
+
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Run Terraform action on deployments
+        run: |
+          cd deployments
+          terraform init
+          terraform apply \
+            --var="tyk_license=${{ secrets.DASH_LICENSE }}" \
+            --var-file=main.tfvars \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Run Tests
+        run: |
+          cd tests
+          terraform init
+          terraform apply \
+            --var-file=main.tfvars \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Test Grafana Snapshot
+        run: |
+          kubectl logs -n dependencies $(kubectl get pods -n dependencies --selector=app=snapshot-job -o jsonpath='{.items[-1].metadata.name}') --tail=1
+
+      - name: Destroy Tests
+        run: |
+          cd tests
+          terraform destroy \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Destroy testing resources
+        run: |
+          cd deployments
+          terraform destroy \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Destroy AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: cd aks && terraform destroy --auto-approve
+
+      - name: Destroy EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: cd eks && terraform destroy --auto-approve
+
+      - name: Destroy GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: cd gke && terraform destroy --auto-approve

.github/workflows/deployments.yml

@@ -48,7 +48,7 @@ on:
         description: 'Tyk Gateway version'
         required: true
         type: string
-        default: 'v5.4'
+        default: 'v5.6'
       custom_deployment_descriptor:
         description: 'Custom Deployment Descriptor'
         required: true
@@ -139,7 +139,6 @@ jobs:
           cd deployments
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="kubernetes_config_context=performance-testing" \
             --var="analytics_database_enabled=${{ inputs.analytics_database_enabled }}" \
             --var="analytics_prometheus_enabled=${{ inputs.analytics_prometheus_enabled }}" \
@@ -149,6 +148,7 @@ jobs:
             --var="tyk_version=${{ inputs.tyk_version }}" \
             --var="tyk_license=${{ secrets.DASH_LICENSE }}" \
             --var="grafana_service_type=LoadBalancer" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: Access to Grafana Dashboard

.github/workflows/full_performance_test.yml

@@ -15,7 +15,7 @@ on:
         description: 'Tyk Gateway version'
         required: true
         type: string
-        default: 'v5.4'
+        default: 'v5.6'
 
 env:
   provider: ${{ inputs.cloud == 'Azure' && 'aks' || (inputs.cloud == 'AWS' && 'eks' || 'gke') }}

.github/workflows/tests.yml

@@ -119,10 +119,10 @@ jobs:
           cd tests
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="kubernetes_config_context=performance-testing" \
             --var="tests_duration=${{ inputs.tests_duration }}" \
             --var="tests_virtual_users=${{ inputs.tests_virtual_users }}" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: Access to Grafana Dashboard

README.md

@@ -92,7 +92,7 @@ resources_limits_cpu        = "0"
 resources_limits_memory     = "0"
 
 tyk_enabled         = true
-tyk_version         = "v5.4"
+tyk_version         = "v5.6"
 tyk_license         = ""
 tyk_deployment_type = "Deployment"
 tyk_go_gc           = 1600

aks/README.md

@@ -26,8 +26,8 @@ az aks get-credentials \
 ## Inputs
 
 | Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_aks_version"></a> [aks\_version](#input\_aks\_version) | AKS cluster version. | `string` | `"1.29"` | no |
+|------|-------------|------|------|:--------:|
+| <a name="input_aks_version"></a> [aks\_version](#input\_aks\_version) | AKS cluster version. | `string` | `""` | no |
 | <a name="input_cluster_location"></a> [cluster\_location](#input\_cluster\_location) | AKS cluster location. | `string` | `"westus"` | no |
 | <a name="input_cluster_machine_type"></a> [cluster\_machine\_type](#input\_cluster\_machine\_type) | Default machine type for cluster. | `string` | `"Standard_F4s_v2"` | no |
 | <a name="input_dependencies_machine_type"></a> [dependencies\_machine\_type](#input\_dependencies\_machine\_type) | Machine type for dependencies, overrides cluster\_machine\_type. | `string` | `""` | no |