From 27a5456ca897b60b753bd082f7987921047bcc37 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Mon, 18 Mar 2024 16:01:01 +1000 Subject: [PATCH] fix: use same docker image for celery worker (#3710) --- build/app/Dockerfile | 27 +++++++++++++++++++ build/app/celery-start.sh | 8 ++++++ build/app/mailarchive-start.sh | 13 +++++++++ build/app/start.sh | 20 ++++++++++++++ .../templates/celery-worker/deployment.yaml | 4 ++- charts/mailarchive/templates/deployment.yaml | 2 ++ docker/base.Dockerfile | 9 +++++++ 7 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 build/app/Dockerfile create mode 100644 build/app/celery-start.sh create mode 100644 build/app/mailarchive-start.sh create mode 100644 build/app/start.sh diff --git a/build/app/Dockerfile b/build/app/Dockerfile new file mode 100644 index 00000000..d88f979c --- /dev/null +++ b/build/app/Dockerfile @@ -0,0 +1,27 @@ +FROM ghcr.io/ietf-tools/mailarchive-app-base:latest +LABEL maintainer="IETF Tools Team " + +# install dependencies first for image layer reuse +COPY requirements.txt . +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt + +# Switch to local dev user +USER dev:dev + +COPY . . +COPY ./build/app/start.sh ./start.sh +COPY ./build/app/mailarchive-start.sh ./mailarchive-start.sh +COPY ./build/app/celery-start.sh ./celery-start.sh + +RUN chmod +x start.sh && \ + chmod +x mailarchive-start.sh && \ + chmod +x celery-start.sh && \ + chmod +x docker/scripts/app-create-dirs.sh && \ + sh ./docker/scripts/app-create-dirs.sh + +# VOLUME + +# document the port the container listens on +# EXPOSE 8000 + +CMD ["./start.sh"] diff --git a/build/app/celery-start.sh b/build/app/celery-start.sh new file mode 100644 index 00000000..bc38a64f --- /dev/null +++ b/build/app/celery-start.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Run a celery worker +# +# echo "Running Mailarchive checks..." +# ./backend/manage.py check + +celery "$@" diff --git a/build/app/mailarchive-start.sh b/build/app/mailarchive-start.sh new file mode 100644 index 00000000..aaa9b8e8 --- /dev/null +++ b/build/app/mailarchive-start.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "Running Mailarchive checks..." +./backend/manage.py check + +echo "Running Mailarchive migrations..." +./backend/manage.py migrate + +echo "Running Initializing index..." +./backend/manage.py init_index + +echo "Starting Mailarchive..." +./backend/manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/build/app/start.sh b/build/app/start.sh new file mode 100644 index 00000000..86e92a14 --- /dev/null +++ b/build/app/start.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Environment config: +# +# CONTAINER_ROLE - mailarchive, celery, or beat (defaults to mailarchive) +# +case "${CONTAINER_ROLE:-mailarchive}" in + mailarchive) + exec ./mailarchive-start.sh + ;; + celery) + exec ./celery-start.sh --app="${CELERY_APP:-mlarchive.celeryapp:app}" worker + ;; + beat) + exec ./celery-start.sh --app="${CELERY_APP:-mlarchive.celeryapp:app}" beat + ;; + *) + echo "Unknown role '${CONTAINER_ROLE}'" + exit 255 +esac \ No newline at end of file diff --git a/charts/mailarchive/templates/celery-worker/deployment.yaml b/charts/mailarchive/templates/celery-worker/deployment.yaml index 528455ed..53cb1abb 100644 --- a/charts/mailarchive/templates/celery-worker/deployment.yaml +++ b/charts/mailarchive/templates/celery-worker/deployment.yaml @@ -35,7 +35,7 @@ spec: {{- end }} securityContext: {{- toYaml .securityContext | nindent 12 }} - image: "{{ $.Values.celeryWorker.image.repository }}:{{ $.Values.celeryWorker.image.tag }}" + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" imagePullPolicy: {{ $.Values.image.pullPolicy }} {{- with .command }} command: @@ -43,6 +43,8 @@ spec: {{- end }} env: {{- include "mailarchive.db.env" . | nindent 12 }} + - name: CONTAINER_ROLE + value: celery - name: ELASTICSEARCH_HOST value: elasticsearch-master - name: SERVER_ROLE diff --git a/charts/mailarchive/templates/deployment.yaml b/charts/mailarchive/templates/deployment.yaml index 6902479c..d7be6099 100644 --- a/charts/mailarchive/templates/deployment.yaml +++ b/charts/mailarchive/templates/deployment.yaml @@ -38,6 +38,8 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} env: {{- include "mailarchive.db.env" . | nindent 12 }} + - name: CONTAINER_ROLE + value: mailarchive - name: ELASTICSEARCH_HOST value: elasticsearch-master - name: IMPORT_MESSAGE_APIKEY diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index f325fbc4..a18a0f47 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,6 +1,7 @@ FROM python:3.9-bullseye LABEL maintainer="Ryan Cross " +# Ensure apt is in non-interactive to avoid prompts ENV DEBIAN_FRONTEND=noninteractive # Update system packages @@ -36,7 +37,11 @@ RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \ chmod +x /tmp/app-install-chromedriver.sh RUN /tmp/app-install-chromedriver.sh +# purge because of vulnerability (see https://www.cvedetails.com/) +RUN apt-get purge -y imagemagick imagemagick-6-common + # Get rid of installation files we don't need in the image, to reduce size +# this should be included in install layer above if chromedriver layer removed RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/* # "fake" dbus address to prevent errors @@ -56,6 +61,10 @@ ENV LC_ALL en_US.UTF-8 ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ RUN chmod +rx /usr/local/bin/wait-for +# Create a dev user and group with a specific UID/GID +RUN groupadd --gid 1000 dev \ + && useradd --uid 1000 --gid dev --shell /bin/bash --create-home dev + # Create data directory RUN mkdir -p /data