Skip to content

Commit

Permalink
feat: video recording with pluggable upload container (#1881)
Browse files Browse the repository at this point in the history
  • Loading branch information
msvticket authored Oct 25, 2023
1 parent 3a03f97 commit 01648f3
Show file tree
Hide file tree
Showing 10 changed files with 258 additions and 2 deletions.
4 changes: 2 additions & 2 deletions Video/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ ENV DEBIAN_FRONTEND=noninteractive \
RUN apt-get -qqy update \
&& apt-get upgrade -yq \
&& apt-get -qqy --no-install-recommends install \
supervisor x11-xserver-utils python3-pip \
supervisor x11-xserver-utils x11-utils curl jq python3-pip \
&& python3 -m pip install --upgrade pip \
&& python3 -m pip install --upgrade setuptools \
&& python3 -m pip install --upgrade wheel \
&& python3 -m pip install --upgrade wheel \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*

#======================================
Expand Down
83 changes: 83 additions & 0 deletions charts/selenium-grid/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,13 @@ Service Account fullname
{{- .Values.serviceAccount.name | default "selenium-serviceaccount" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Video ConfigMap fullname
*/}}
{{- define "seleniumGrid.video.fullname" -}}
{{- default "selenium-video" .Values.videoRecorder.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Is autoscaling using KEDA enabled
*/}}
Expand Down Expand Up @@ -168,6 +175,74 @@ template:
{{- if .node.sidecars }}
{{- toYaml .node.sidecars | nindent 6 }}
{{- end }}
{{- if .Values.videoRecorder.enabled }}
- name: video
image: {{ printf "%s:%s" .Values.videoRecorder.imageName .Values.videoRecorder.imageTag }}
imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }}
env:
- name: UPLOAD_DESTINATION_PREFIX
value: {{ .Values.videoRecorder.uploadDestinationPrefix }}
{{- with .Values.videoRecorder.extraEnvironmentVariables }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Values.busConfigMap.name }}
{{- with .Values.videoRecorder.extraEnvFrom }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if gt (len .Values.videoRecorder.ports) 0 }}
ports:
{{- range .Values.videoRecorder.ports }}
- containerPort: {{ . }}
protocol: TCP
{{- end }}
{{- end }}
volumeMounts:
- name: dshm
mountPath: /dev/shm
- name: video-scripts
mountPath: /opt/bin/video.sh
subPath: video.sh
- name: video
mountPath: /videos
{{- if .Values.videoRecorder.extraVolumeMounts }}
{{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 8 }}
{{- end }}
{{- with .Values.videoRecorder.resources }}
resources: {{- toYaml . | nindent 10 }}
{{- end }}
{{- if .uploader }}
- name: uploader
image: {{ printf "%s:%s" .uploader.imageName .uploader.imageTag }}
imagePullPolicy: {{ .uploader.imagePullPolicy }}
{{- with .uploader.command }}
command: {{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .uploader.args }}
args: {{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .uploader.extraEnvironmentVariables }}
env: {{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .uploader.extraEnvFrom }}
envFrom:
{{- toYaml . | nindent 10 }}
{{- end }}
volumeMounts:
- name: video
mountPath: /videos
{{- if .uploader.extraVolumeMounts }}
{{- toYaml .uploader.extraVolumeMounts | nindent 8 }}
{{- end }}
{{- with .uploader.resources }}
resources: {{- toYaml . | nindent 10 }}
{{- end }}
{{- with .uploader.securityContext }}
securityContext: {{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
{{- end }}
{{- if or .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
imagePullSecrets:
- name: {{ default .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
Expand All @@ -194,6 +269,14 @@ template:
{{- if .node.extraVolumes }}
{{ toYaml .node.extraVolumes | nindent 6 }}
{{- end }}
{{- if .Values.videoRecorder.enabled }}
- name: video-scripts
configMap:
name: {{ template "seleniumGrid.video.fullname" . }}
defaultMode: 0500
- name: video
{{- toYaml .Values.videoRecorder.volume | nindent 8 }}
{{- end }}
{{- end -}}

{{/*
Expand Down
1 change: 1 addition & 0 deletions charts/selenium-grid/templates/chrome-node-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
{{- $_ = set $podScope "node" .Values.chromeNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
{{- end }}
1 change: 1 addition & 0 deletions charts/selenium-grid/templates/chrome-node-scaledjobs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
{{- $_ = set $podScope "node" .Values.chromeNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
{{- end }}
1 change: 1 addition & 0 deletions charts/selenium-grid/templates/edge-node-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-edge-node" -}}
{{- $_ = set $podScope "node" .Values.edgeNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
{{- end }}
1 change: 1 addition & 0 deletions charts/selenium-grid/templates/edge-node-scaledjob.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-edge-node" -}}
{{- $_ = set $podScope "node" .Values.edgeNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-firefox-node" -}}
{{- $_ = set $podScope "node" .Values.firefoxNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
{{- end }}
1 change: 1 addition & 0 deletions charts/selenium-grid/templates/firefox-node-scaledjob.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,6 @@ spec:
{{- $podScope := deepCopy . -}}
{{- $_ := set $podScope "name" "selenium-firefox-node" -}}
{{- $_ = set $podScope "node" .Values.firefoxNode -}}
{{- $_ = set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
{{- end }}
92 changes: 92 additions & 0 deletions charts/selenium-grid/templates/video-cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
{{- if .Values.videoRecorder.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "seleniumGrid.video.fullname" . }}
data:
video.sh: |
#!/usr/bin/env bash
set -em
function finish {
echo exit > /videos/uploadpipe
kill -s SIGINT `cat /var/run/supervisor/supervisord.pid`
}
trap finish EXIT
FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE}
CODEC=${CODEC:-$SE_CODEC}
PRESET=${PRESET:-$SE_PRESET}
export DISPLAY=localhost:${DISPLAY_NUM}.0
return_code=1
max_attempts=600
attempts=0
mkfifo /videos/uploadpipe
if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]]
then
echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder.
exit
fi
echo Checking if the display is open
until xset b off || [[ $attempts = $max_attempts ]]
do
echo Waiting before next display check
sleep 0.5
attempts=$((attempts+1))
done
if [[ $attempts = $max_attempts ]]
then
echo Can not open display, exiting.
exit
fi
VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}')
recording_started="false"
video_file_name=""
video_file=""
prev_session_id=""
attempts=0
echo Checking if node API responds
until curl -s --request GET http://localhost:5555/status || [[ $attempts = $max_attempts ]]
do
echo Waiting before next API check
sleep 0.5
attempts=$((attempts+1))
done
if [[ $attempts = $max_attempts ]]
then
echo Can not reach node API, exiting.
exit
fi
while curl -s --request GET http://localhost:5555/status > /tmp/status.json
do
session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json)
echo $session_id
if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]]
then
video_file_name="$session_id.mp4"
video_file="${VIDEO_LOCATION:-/videos}/$video_file_name"
echo "Starting to record video"
ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file &
recording_started="true"
echo "Video recording started"
elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]]
then
echo "Stopping to record video"
kill -INT %1
fg || echo ffmpeg exited with code $?
upload_destination=${UPLOAD_DESTINATION_PREFIX}${video_file_name}
echo "Uploading video to $upload_destination"
echo $video_file $upload_destination > /videos/uploadpipe &
recording_started="false"
elif [[ $recording_started = "true" ]]
then
echo "Video recording in progress"
sleep 1
else
echo "No session in progress"
sleep 1
fi
prev_session_id=$session_id
done
echo
{{- end }}
75 changes: 75 additions & 0 deletions charts/selenium-grid/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ serviceAccount:
create: true
name: ""
annotations: {}
# eks.amazonaws.com/role-arn: "arn:aws:iam::12345678:role/video-bucket-permissions"

# Configure the ingress resource to access the Grid installation.
ingress:
Expand Down Expand Up @@ -724,5 +725,79 @@ edgeNode:
# It should be set using the --set-json option
sidecars: []

videoRecorder:
# Image of video recorder
imageName: selenium/video
enabled: false
# Where to upload the video file. Should be set to something like 's3://myvideobucket/'
uploadDestinationPrefix: ""
# What uploader to use. See .videRecorder.s3 for how to create a new one.
uploader: s3

# Image of video recorder
imageTag: latest
# Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images)
imagePullPolicy: IfNotPresent

ports:
- 5666
resources:
requests:
memory: "1Gi"
cpu: "1"
limits:
memory: "1Gi"
cpu: "1"
extraEnvironmentVariables: []
# - name: VIDEO_LOCATION
# value: /videos
# Custom environment variables by sourcing entire configMap, Secret, etc. for video recorder.
extraEnvFrom:
# - configMapRef:
# name: proxy-settings
# - secretRef:
# name: mysecret
# Wait for pod startup
terminationGracePeriodSeconds: 30
volume:
emptyDir: {}
s3:
imageName: public.ecr.aws/bitnami/aws-cli
imageTag: "2"
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 0
command:
- /bin/sh
args:
- -c
- |
while ! [ -p /videos/uploadpipe ]
do
echo Waiting for /videos/uploadpipe to be created
sleep 1
done
echo Waiting for files to upload
while read FILE DESTINATION < /videos/uploadpipe
do
if [ "$FILE" = "exit" ]
then
break
else
aws s3 cp --no-progress $FILE $DESTINATION
fi
done
# extraEnvironmentVariables:
# - name: AWS_ACCESS_KEY_ID
# value: aws_access_key_id
# - name: AWS_SECRET_ACCESS_KEY
# value: aws_secret_access_key
# - name:
# valueFrom:
# secretKeyRef:
# name: secret-name
# key: secret-key


# Custom labels for k8s resources
customLabels: {}

0 comments on commit 01648f3

Please sign in to comment.