diff --git a/Video/Dockerfile b/Video/Dockerfile index d6408bef6..4edc5a64a 100644 --- a/Video/Dockerfile +++ b/Video/Dockerfile @@ -18,10 +18,10 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get -qqy update \ && apt-get upgrade -yq \ && apt-get -qqy --no-install-recommends install \ - supervisor x11-xserver-utils python3-pip \ + supervisor x11-xserver-utils x11-utils curl jq python3-pip \ && python3 -m pip install --upgrade pip \ && python3 -m pip install --upgrade setuptools \ - && python3 -m pip install --upgrade wheel \ + && python3 -m pip install --upgrade wheel \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* #====================================== diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 28eb06224..affe63635 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -87,6 +87,13 @@ Service Account fullname {{- .Values.serviceAccount.name | default "selenium-serviceaccount" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Video ConfigMap fullname +*/}} +{{- define "seleniumGrid.video.fullname" -}} +{{- default "selenium-video" .Values.videoRecorder.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Is autoscaling using KEDA enabled */}} @@ -168,6 +175,74 @@ template: {{- if .node.sidecars }} {{- toYaml .node.sidecars | nindent 6 }} {{- end }} + {{- if .Values.videoRecorder.enabled }} + - name: video + image: {{ printf "%s:%s" .Values.videoRecorder.imageName .Values.videoRecorder.imageTag }} + imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }} + env: + - name: UPLOAD_DESTINATION_PREFIX + value: {{ .Values.videoRecorder.uploadDestinationPrefix }} + {{- with .Values.videoRecorder.extraEnvironmentVariables }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Values.busConfigMap.name }} + {{- with .Values.videoRecorder.extraEnvFrom }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if gt (len .Values.videoRecorder.ports) 0 }} + ports: + {{- range .Values.videoRecorder.ports }} + - containerPort: {{ . }} + protocol: TCP + {{- end }} + {{- end }} + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: video-scripts + mountPath: /opt/bin/video.sh + subPath: video.sh + - name: video + mountPath: /videos + {{- if .Values.videoRecorder.extraVolumeMounts }} + {{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- with .Values.videoRecorder.resources }} + resources: {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .uploader }} + - name: uploader + image: {{ printf "%s:%s" .uploader.imageName .uploader.imageTag }} + imagePullPolicy: {{ .uploader.imagePullPolicy }} + {{- with .uploader.command }} + command: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .uploader.args }} + args: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .uploader.extraEnvironmentVariables }} + env: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .uploader.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: video + mountPath: /videos + {{- if .uploader.extraVolumeMounts }} + {{- toYaml .uploader.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- with .uploader.resources }} + resources: {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .uploader.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- end }} {{- if or .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }} imagePullSecrets: - name: {{ default .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }} @@ -194,6 +269,14 @@ template: {{- if .node.extraVolumes }} {{ toYaml .node.extraVolumes | nindent 6 }} {{- end }} + {{- if .Values.videoRecorder.enabled }} + - name: video-scripts + configMap: + name: {{ template "seleniumGrid.video.fullname" . }} + defaultMode: 0500 + - name: video + {{- toYaml .Values.videoRecorder.volume | nindent 8 }} + {{- end }} {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/chrome-node-deployment.yaml b/charts/selenium-grid/templates/chrome-node-deployment.yaml index 0b3d2a9b5..a89800fdf 100644 --- a/charts/selenium-grid/templates/chrome-node-deployment.yaml +++ b/charts/selenium-grid/templates/chrome-node-deployment.yaml @@ -25,5 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-chrome-node" -}} {{- $_ = set $podScope "node" .Values.chromeNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml index 4971aec74..b5770e8a2 100644 --- a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml @@ -35,5 +35,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-chrome-node" -}} {{- $_ = set $podScope "node" .Values.chromeNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-deployment.yaml b/charts/selenium-grid/templates/edge-node-deployment.yaml index 883a45a76..d81b7ef93 100644 --- a/charts/selenium-grid/templates/edge-node-deployment.yaml +++ b/charts/selenium-grid/templates/edge-node-deployment.yaml @@ -25,5 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-edge-node" -}} {{- $_ = set $podScope "node" .Values.edgeNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-scaledjob.yaml b/charts/selenium-grid/templates/edge-node-scaledjob.yaml index 6d113d186..6083226e5 100644 --- a/charts/selenium-grid/templates/edge-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/edge-node-scaledjob.yaml @@ -35,5 +35,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-edge-node" -}} {{- $_ = set $podScope "node" .Values.edgeNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-deployment.yaml b/charts/selenium-grid/templates/firefox-node-deployment.yaml index d0b250470..645986c11 100644 --- a/charts/selenium-grid/templates/firefox-node-deployment.yaml +++ b/charts/selenium-grid/templates/firefox-node-deployment.yaml @@ -25,5 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-firefox-node" -}} {{- $_ = set $podScope "node" .Values.firefoxNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml index de90024c7..4ec26225a 100644 --- a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml @@ -35,5 +35,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" "selenium-firefox-node" -}} {{- $_ = set $podScope "node" .Values.firefoxNode -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/video-cm.yaml b/charts/selenium-grid/templates/video-cm.yaml new file mode 100644 index 000000000..41a13b027 --- /dev/null +++ b/charts/selenium-grid/templates/video-cm.yaml @@ -0,0 +1,92 @@ +{{- if .Values.videoRecorder.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seleniumGrid.video.fullname" . }} +data: + video.sh: | + #!/usr/bin/env bash + set -em + function finish { + echo exit > /videos/uploadpipe + kill -s SIGINT `cat /var/run/supervisor/supervisord.pid` + } + trap finish EXIT + FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} + CODEC=${CODEC:-$SE_CODEC} + PRESET=${PRESET:-$SE_PRESET} + export DISPLAY=localhost:${DISPLAY_NUM}.0 + + return_code=1 + max_attempts=600 + attempts=0 + mkfifo /videos/uploadpipe + if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]] + then + echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder. + exit + fi + echo Checking if the display is open + until xset b off || [[ $attempts = $max_attempts ]] + do + echo Waiting before next display check + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = $max_attempts ]] + then + echo Can not open display, exiting. + exit + fi + VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') + + recording_started="false" + video_file_name="" + video_file="" + prev_session_id="" + attempts=0 + echo Checking if node API responds + until curl -s --request GET http://localhost:5555/status || [[ $attempts = $max_attempts ]] + do + echo Waiting before next API check + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = $max_attempts ]] + then + echo Can not reach node API, exiting. + exit + fi + while curl -s --request GET http://localhost:5555/status > /tmp/status.json + do + session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) + echo $session_id + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] + then + video_file_name="$session_id.mp4" + video_file="${VIDEO_LOCATION:-/videos}/$video_file_name" + echo "Starting to record video" + ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & + recording_started="true" + echo "Video recording started" + elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] + then + echo "Stopping to record video" + kill -INT %1 + fg || echo ffmpeg exited with code $? + upload_destination=${UPLOAD_DESTINATION_PREFIX}${video_file_name} + echo "Uploading video to $upload_destination" + echo $video_file $upload_destination > /videos/uploadpipe & + recording_started="false" + elif [[ $recording_started = "true" ]] + then + echo "Video recording in progress" + sleep 1 + else + echo "No session in progress" + sleep 1 + fi + prev_session_id=$session_id + done + echo +{{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index dce24f9a3..193e5499c 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -24,6 +24,7 @@ serviceAccount: create: true name: "" annotations: {} + # eks.amazonaws.com/role-arn: "arn:aws:iam::12345678:role/video-bucket-permissions" # Configure the ingress resource to access the Grid installation. ingress: @@ -724,5 +725,79 @@ edgeNode: # It should be set using the --set-json option sidecars: [] +videoRecorder: + # Image of video recorder + imageName: selenium/video + enabled: false + # Where to upload the video file. Should be set to something like 's3://myvideobucket/' + uploadDestinationPrefix: "" + # What uploader to use. See .videRecorder.s3 for how to create a new one. + uploader: s3 + + # Image of video recorder + imageTag: latest + # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) + imagePullPolicy: IfNotPresent + + ports: + - 5666 + resources: + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "1Gi" + cpu: "1" + extraEnvironmentVariables: [] + # - name: VIDEO_LOCATION + # value: /videos + # Custom environment variables by sourcing entire configMap, Secret, etc. for video recorder. + extraEnvFrom: + # - configMapRef: + # name: proxy-settings + # - secretRef: + # name: mysecret + # Wait for pod startup + terminationGracePeriodSeconds: 30 + volume: + emptyDir: {} + s3: + imageName: public.ecr.aws/bitnami/aws-cli + imageTag: "2" + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + command: + - /bin/sh + args: + - -c + - | + while ! [ -p /videos/uploadpipe ] + do + echo Waiting for /videos/uploadpipe to be created + sleep 1 + done + echo Waiting for files to upload + while read FILE DESTINATION < /videos/uploadpipe + do + if [ "$FILE" = "exit" ] + then + break + else + aws s3 cp --no-progress $FILE $DESTINATION + fi + done + # extraEnvironmentVariables: + # - name: AWS_ACCESS_KEY_ID + # value: aws_access_key_id + # - name: AWS_SECRET_ACCESS_KEY + # value: aws_secret_access_key + # - name: + # valueFrom: + # secretKeyRef: + # name: secret-name + # key: secret-key + + # Custom labels for k8s resources customLabels: {}