diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 29b4239d9..2c7240df7 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -109,8 +109,13 @@ This table contains the configuration parameters of the chart and their default | `ingress.tls` | `[]` | TLS backend configuration for ingress resource | | `autoscaling.enableWithExistingKEDA` | `false` | Enable autoscaling of browser nodes. | | `autoscaling.enabled` | `false` | Same as above plus installation of KEDA | -| `autoscaling.scalingType` | `job` | Which typ of KEDA scaling to use: job or deployment | -| `autoscaling.scaledJobOptions` | See `values.yaml` | Options for KEDA ScaledJobs | +| `autoscaling.scalingType` | `job` | Which typ of KEDA scaling to use: `job` or `deployment` | +| `autoscaling.scaledOptions` | See `values.yaml` | Common options for KEDA scaled resources (both ScaledJobs and ScaledObjects) | +| `autoscaling.scaledOptions.minReplicaCount` | `0` | Min number of replicas that each browser nodes has when autoscaling | +| `autoscaling.scaledOptions.maxReplicaCount` | `8` | Max number of replicas that each browser nodes can auto scale up to | +| `autoscaling.scaledOptions.pollingInterval` | `10` | The interval to check each trigger on | +| `autoscaling.scaledJobOptions` | See `values.yaml` | Options for KEDA ScaledJobs (when `scalingType` is set to `job`) | +| `autoscaling.scaledObjectOptions` | See `values.yaml` | Options for KEDA ScaledObjects (when `scalingType` is set to `deployment`) | | `autoscaling.deregisterLifecycle` | See `values.yaml` | Lifecycle applied to pods of deployments controlled by KEDA. Makes the node deregister from selenium hub | | `chromeNode.enabled` | `true` | Enable chrome nodes | | `chromeNode.deploymentEnabled` | `true` | Enable creation of Deployment for chrome nodes | @@ -146,8 +151,9 @@ This table contains the configuration parameters of the chart and their default | `chromeNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `chromeNode.hpa.browserName` | `chrome` | BrowserName from the capability | | `chromeNode.hpa.browserVersion` | `` | BrowserVersion from the capability | -| `chromeNode.maxReplicaCount` | `8` | Max number of replicas that this browsernode can auto scale up to | -| `chromeNode.minReplicaCount` | `1` | Min number of replicas that this browsernode has when jobs are running | +| `chromeNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for chrome nodes | +| `chromeNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for chrome nodes | +| `chromeNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for chrome nodes | | `firefoxNode.enabled` | `true` | Enable firefox nodes | | `firefoxNode.deploymentEnabled` | `true` | Enable creation of Deployment for firefox nodes | | `firefoxNode.replicas` | `1` | Number of firefox nodes. Disabled if autoscaling is enabled. | @@ -182,8 +188,9 @@ This table contains the configuration parameters of the chart and their default | `firefoxNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `firefoxNode.hpa.browserName` | `firefox` | BrowserName from the capability | | `firefoxNode.hpa.browserVersion` | `` | BrowserVersion from the capability | -| `firefoxNode.maxReplicaCount` | `8` | Max number of replicas that this browsernode can auto scale up to | -| `firefoxNode.minReplicaCount` | `1` | Min number of replicas that this browsernode has when jobs are running | +| `firefoxNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for firefox nodes | +| `firefoxNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for firefox nodes | +| `firefoxNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for firefox nodes | | `edgeNode.enabled` | `true` | Enable edge nodes | | `edgeNode.deploymentEnabled` | `true` | Enable creation of Deployment for edge nodes | | `edgeNode.replicas` | `1` | Number of edge nodes. Disabled if autoscaling is enabled. | @@ -218,8 +225,9 @@ This table contains the configuration parameters of the chart and their default | `edgeNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `edgeNode.hpa.browserName` | `edge` | BrowserName from the capability | | `edgeNode.hpa.browserVersion` | `` | BrowserVersion from the capability | -| `edgeNode.maxReplicaCount` | `8` | Max number of replicas that this browsernode can auto scale up to | -| `edgeNode.minReplicaCount` | `1` | Min number of replicas that this browsernode has when jobs are running | +| `edgeNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for edge nodes | +| `edgeNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for edge nodes | +| `edgeNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for edge nodes | | `videoRecorder.enabled` | `false` | Enable video recorder for node | | `videoRecorder.imageName` | `selenium/video` | Selenium video recoder image name | | `videoRecorder.imageTag` | `ffmpeg-6.0-20231102` | Image tag of video recorder | diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 4d816fa11..5fa362094 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -101,6 +101,47 @@ Is autoscaling using KEDA enabled {{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}} {{- end -}} +{{/* +Common autoscaling spec template +*/}} +{{- define "seleniumGrid.autoscalingTemplate" -}} +{{- $spec := toYaml (dict) -}} +{{/* Merge with precedence from right to left */}} +{{- with .Values.autoscaling.scaledOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} +{{- end -}} +{{- with .node.scaledOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} +{{- end -}} +{{- if eq .Values.autoscaling.scalingType "deployment" -}} + {{- with .Values.autoscaling.scaledObjectOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} + {{- end -}} + {{- with .node.scaledObjectOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} + {{- end -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) (dict "scaleTargetRef" (dict "name" .name)) | toYaml -}} +{{- else if eq .Values.autoscaling.scalingType "job" -}} + {{- with .Values.autoscaling.scaledJobOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} + {{- end -}} + {{- with .node.scaledJobOptions -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} + {{- end -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) (dict "jobTargetRef" .podTemplate) | toYaml -}} +{{- end -}} +{{- if and $spec (ne $spec "{}") -}} + {{ tpl $spec $ }} +{{- end -}} +{{- if not .Values.autoscaling.scaledOptions.triggers }} +triggers: + - type: selenium-grid + {{- with .node.hpa }} + metadata: {{- tpl (toYaml .) $ | nindent 6 }} + {{- end }} +{{- end }} +{{- end -}} + {{/* Common pod template */}} diff --git a/charts/selenium-grid/templates/chrome-node-deployment.yaml b/charts/selenium-grid/templates/chrome-node-deployment.yaml index 4d9a30ac0..2ea8c010c 100644 --- a/charts/selenium-grid/templates/chrome-node-deployment.yaml +++ b/charts/selenium-grid/templates/chrome-node-deployment.yaml @@ -23,8 +23,8 @@ spec: app: selenium-chrome-node app.kubernetes.io/instance: {{ .Release.Name }} {{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-chrome-node" -}} -{{- $_ = set $podScope "node" .Values.chromeNode -}} +{{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} +{{- $_ = set $podScope "node" .Values.chromeNode -}} {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml index 7cb78043c..74c588198 100644 --- a/charts/selenium-grid/templates/chrome-node-hpa.yaml +++ b/charts/selenium-grid/templates/chrome-node-hpa.yaml @@ -11,13 +11,8 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }} spec: - maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }} - minReplicaCount: {{ .Values.chromeNode.minReplicaCount }} - scaleTargetRef: - name: {{ template "seleniumGrid.chromeNode.fullname" . }} - triggers: - - type: selenium-grid - {{- with .Values.chromeNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.chromeNode -}} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml index c72818b05..22a9fcef2 100644 --- a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml @@ -19,22 +19,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }} - {{- with .Values.autoscaling.scaledJobOptions -}} - {{ toYaml . | nindent 2 }} - {{- end }} - triggers: - - type: selenium-grid - {{- with .Values.chromeNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - jobTargetRef: - parallelism: 1 - completions: 1 - backoffLimit: 0 -{{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-chrome-node" -}} -{{- $_ = set $podScope "node" .Values.chromeNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} -{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.chromeNode -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-deployment.yaml b/charts/selenium-grid/templates/edge-node-deployment.yaml index 6216dc101..33fc360aa 100644 --- a/charts/selenium-grid/templates/edge-node-deployment.yaml +++ b/charts/selenium-grid/templates/edge-node-deployment.yaml @@ -23,8 +23,8 @@ spec: app: selenium-edge-node app.kubernetes.io/instance: {{ .Release.Name }} {{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-edge-node" -}} -{{- $_ = set $podScope "node" .Values.edgeNode -}} +{{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} +{{- $_ = set $podScope "node" .Values.edgeNode -}} {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml index b115de353..0839a3bdd 100644 --- a/charts/selenium-grid/templates/edge-node-hpa.yaml +++ b/charts/selenium-grid/templates/edge-node-hpa.yaml @@ -11,13 +11,8 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.edgeNode.fullname" . }} spec: - maxReplicaCount: {{ .Values.edgeNode.maxReplicaCount }} - minReplicaCount: {{ .Values.edgeNode.minReplicaCount }} - scaleTargetRef: - name: {{ template "seleniumGrid.edgeNode.fullname" . }} - triggers: - - type: selenium-grid - {{- with .Values.edgeNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.edgeNode -}} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-scaledjob.yaml b/charts/selenium-grid/templates/edge-node-scaledjob.yaml index eae9ba79a..a9f235c31 100644 --- a/charts/selenium-grid/templates/edge-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/edge-node-scaledjob.yaml @@ -19,22 +19,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - maxReplicaCount: {{ .Values.edgeNode.maxReplicaCount }} - {{- with .Values.autoscaling.scaledJobOptions -}} - {{ toYaml . | nindent 2 }} - {{- end }} - triggers: - - type: selenium-grid - {{- with .Values.edgeNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - jobTargetRef: - parallelism: 1 - completions: 1 - backoffLimit: 0 -{{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-edge-node" -}} -{{- $_ = set $podScope "node" .Values.edgeNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} -{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.edgeNode -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/event-bus-configmap.yaml b/charts/selenium-grid/templates/event-bus-configmap.yaml index efe33fa7d..05f279bba 100644 --- a/charts/selenium-grid/templates/event-bus-configmap.yaml +++ b/charts/selenium-grid/templates/event-bus-configmap.yaml @@ -15,6 +15,6 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - SE_EVENT_BUS_HOST: {{ $eventBusHost }} + SE_EVENT_BUS_HOST: {{ $eventBusHost | quote }} SE_EVENT_BUS_PUBLISH_PORT: {{ $eventBusPublishPort | quote }} SE_EVENT_BUS_SUBSCRIBE_PORT: {{ $eventBusSubscribePort | quote }} diff --git a/charts/selenium-grid/templates/firefox-node-deployment.yaml b/charts/selenium-grid/templates/firefox-node-deployment.yaml index 13dcf904f..f5dc6d962 100644 --- a/charts/selenium-grid/templates/firefox-node-deployment.yaml +++ b/charts/selenium-grid/templates/firefox-node-deployment.yaml @@ -23,8 +23,8 @@ spec: app: selenium-firefox-node app.kubernetes.io/instance: {{ .Release.Name }} {{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-firefox-node" -}} -{{- $_ = set $podScope "node" .Values.firefoxNode -}} +{{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} +{{- $_ = set $podScope "node" .Values.firefoxNode -}} {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml index 70a463e79..ad18269a5 100644 --- a/charts/selenium-grid/templates/firefox-node-hpa.yaml +++ b/charts/selenium-grid/templates/firefox-node-hpa.yaml @@ -11,13 +11,8 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.firefoxNode.fullname" . }} spec: - maxReplicaCount: {{ .Values.firefoxNode.maxReplicaCount }} - minReplicaCount: {{ .Values.firefoxNode.minReplicaCount }} - scaleTargetRef: - name: {{ template "seleniumGrid.firefoxNode.fullname" . }} - triggers: - - type: selenium-grid - {{- with .Values.firefoxNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.firefoxNode -}} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml index 3fceee4f5..9105bc358 100644 --- a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml @@ -19,22 +19,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - maxReplicaCount: {{ .Values.firefoxNode.maxReplicaCount }} - {{- with .Values.autoscaling.scaledJobOptions -}} - {{ toYaml . | nindent 2 }} - {{- end }} - triggers: - - type: selenium-grid - {{- with .Values.firefoxNode.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - jobTargetRef: - parallelism: 1 - completions: 1 - backoffLimit: 0 -{{- $podScope := deepCopy . -}} -{{- $_ := set $podScope "name" "selenium-firefox-node" -}} -{{- $_ = set $podScope "node" .Values.firefoxNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} -{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.firefoxNode -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 2d4625b6f..6fdb961ef 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -348,11 +348,31 @@ autoscaling: # Annotations for KEDA resources: ScaledObject and ScaledJob annotations: helm.sh/hook: post-install,post-upgrade - # Options for KEDA ScaledJobs - scaledJobOptions: + # Options for KEDA scaled resources (keep only common options used for both ScaledJob and ScaledObject) + scaledOptions: + minReplicaCount: 0 + maxReplicaCount: 8 pollingInterval: 10 + # List of triggers. Be careful, the default trigger of selenium-grid will be overwritten if you specify this + #triggers: + # Options for KEDA ScaledJobs (only used when scalingType is set to "job") + # see https://keda.sh/docs/latest/concepts/scaling-jobs/#scaledjob-spec + scaledJobOptions: scalingStrategy: strategy: accurate + # Number of Completed jobs should be kept + successfulJobsHistoryLimit: 0 + # Number of Failed jobs should be kept (for troubleshooting purposes) + failedJobsHistoryLimit: 100 + jobTargetRef: + parallelism: 1 + completions: 1 + backoffLimit: 0 + # Options for KEDA ScaledObjects (only used when scalingType is set to "deployment") + # see https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec + scaledObjectOptions: + scaleTargetRef: + kind: Deployment deregisterLifecycle: preStop: exec: @@ -472,8 +492,10 @@ chromeNode: # persistentVolumeClaim: # claimName: my-pv-claim - maxReplicaCount: 8 - minReplicaCount: 1 + # Override the scaled options for chrome nodes + #scaledOptions: + #scaledJobOptions: + #scaledObjectOptions: hpa: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: chrome @@ -594,8 +616,11 @@ firefoxNode: # - name: my-extra-volume-from-pvc # persistentVolumeClaim: # claimName: my-pv-claim - maxReplicaCount: 8 - minReplicaCount: 1 + + # Override the scaled options for firefox nodes + #scaledOptions: + #scaledJobOptions: + #scaledObjectOptions: hpa: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: firefox @@ -713,8 +738,11 @@ edgeNode: # - name: my-extra-volume-from-pvc # persistentVolumeClaim: # claimName: my-pv-claim - maxReplicaCount: 8 - minReplicaCount: 1 + + # Override the scaled options for edge nodes + #scaledOptions: + #scaledJobOptions: + #scaledObjectOptions: hpa: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: MicrosoftEdge