ci: add celery container (#3665)

compose-dev.yml

@@ -50,6 +50,24 @@ services:
     image: rabbitmq:3.10
     restart: always
 
+  celery:
+    build:
+      context: .
+      dockerfile: docker/celery.Dockerfile
+    init: true
+    environment:
+      CELERY_APP: mlarchive.celeryapp:app
+      CELERY_ROLE: worker
+      UPDATE_REQUIREMENTS_FROM: requirements.txt
+      DEV_MODE: "yes"
+    depends_on:
+        - db
+    restart: unless-stopped
+    stop_grace_period: 1m
+    volumes:
+        - app-data:/data
+        - app-assets:/assets
+
 volumes:
     app-assets:
     app-data:

docker/base.Dockerfile

@@ -24,6 +24,7 @@ RUN apt-get update --fix-missing && apt-get install -qy \
     memcached \
     nodejs \
     rsyslog \
+    sudo \
     telnet \
     unzip \
     wget \

docker/celery.Dockerfile

@@ -0,0 +1,29 @@
+# Dockerfile for celery worker
+#
+FROM ghcr.io/ietf-tools/mailarchive-app-base:latest
+LABEL maintainer="IETF Tools Team <[email protected]>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get purge -y imagemagick imagemagick-6-common
+
+# Copy the startup file
+COPY docker/scripts/celery-init.sh /docker-init.sh
+RUN sed -i 's/\r$//' /docker-init.sh && \
+    chmod +x /docker-init.sh
+
+# Install current datatracker python dependencies
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
+RUN rm -rf /tmp/pip-tmp
+
+# Add watchmedo utility for dev containers
+RUN pip3 --disable-pip-version-check --no-cache-dir install watchdog[watchmedo]
+
+# Copy app files
+COPY . .
+
+# Copy settings
+COPY docker/configs/docker_env /workspace/.env
+
+ENTRYPOINT [ "/docker-init.sh" ]

docker/scripts/app-create-dirs.sh

@@ -8,4 +8,5 @@ for sub in \
         echo "Creating dir $sub"
         mkdir -p "$sub";
     fi
+    sudo chown -R dev:dev "/data"
 done

docker/scripts/app-init-dev.sh

@@ -43,17 +43,8 @@ cp $WORKSPACEDIR/docker/configs/settings_docker.py $WORKSPACEDIR/backend/mlarchi
 
 # Create data directories
 echo "Creating data directories..."
-for sub in \
-    /data/archive \
-    /data/log/mail-archive \
-    ; do
-    if [ ! -d "$sub"  ]; then
-        echo "Creating dir $sub"
-        mkdir -p "$sub";
-    fi
-    sudo chown -R dev:dev "/data"
-done
-
+chmod +x ./docker/scripts/app-create-dirs.sh
+./docker/scripts/app-create-dirs.sh
 
 # Wait for DB container
 

docker/scripts/celery-init.sh

@@ -0,0 +1,109 @@
+#!/bin/bash -e
+#
+# Environment parameters:
+#
+#   CELERY_APP - name of application to pass to celery (defaults to mlarchive.celeryapp:app)
+#
+#   CELERY_ROLE - 'worker' or 'beat' (defaults to 'worker')
+#
+#   CELERY_UID - numeric uid for the celery worker process
+#
+#   CELERY_GID - numeric gid for the celery worker process
+#
+#   UPDATES_REQUIREMENTS_FROM - path, relative to /workspace mount, to a pip requirements
+#       file that should be installed at container startup. Default is no package install/update.
+#
+#   DEBUG_TERM_TIMING - if non-empty, writes debug messages during shutdown after a TERM signal
+#
+#   DEV_MODE - if non-empty, restart celery worker on Python file change
+#
+WORKSPACEDIR="/workspace"
+CELERY_ROLE="${CELERY_ROLE:-worker}"
+
+cd "$WORKSPACEDIR" || exit 255
+
+if [[ -n "${UPDATE_REQUIREMENTS_FROM}" ]]; then
+  # Need to run as root in the container for this
+  reqs_file="${WORKSPACEDIR}/${UPDATE_REQUIREMENTS_FROM}"
+  echo "Updating requirements from ${reqs_file}..."
+  pip install --upgrade -r "${reqs_file}"
+fi
+
+CELERY_OPTS=( "${CELERY_ROLE}" )
+if [[ -n "${CELERY_UID}" ]]; then
+  # ensure that a user with the necessary UID exists in container
+  if ! id "${CELERY_UID}" ; then
+    adduser --system --uid "${CELERY_UID}" --no-create-home --disabled-login "celery-user-${CELERY_UID}"
+  fi
+  CELERY_OPTS+=("--uid=${CELERY_UID}")
+  CELERY_USERNAME="$(id -nu ${CELERY_UID})"
+fi
+
+if [[ -n "${CELERY_GID}" ]]; then
+  # ensure that some group with the necessary GID exists in container
+  if ! getent group "${CELERY_GID}" ; then
+    addgroup --gid "${CELERY_GID}" "celery-group-${CELERY_GID}"
+  fi
+  CELERY_OPTS+=("--gid=${CELERY_GID}")
+  CELERY_GROUP="$(getent group ${CELERY_GID} | awk -F: '{print $1}')"
+fi
+
+run_as_celery_uid () {
+  SU_OPTS=()
+  if [[ -n "${CELERY_GROUP}" ]]; then
+    SU_OPTS+=("-g" "${CELERY_GROUP}")
+  fi
+  su "${SU_OPTS[@]}" "${CELERY_USERNAME:-root}" -s /bin/sh -c "$@"
+}
+
+log_term_timing_msgs () {
+  # output periodic debug message
+  while true; do
+    echo "Waiting for celery worker shutdown ($(date --utc --iso-8601=ns))"
+    sleep 0.5s
+  done
+}
+
+cleanup () {
+  # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
+  if [[ -n "${celery_pid}" ]]; then
+    echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
+    kill -TERM "${celery_pid}"
+    if [[ -n "${DEBUG_TERM_TIMING}" ]]; then
+      log_term_timing_msgs &
+    fi
+    wait "${celery_pid}"
+  fi
+}
+
+# Create data directories
+echo "Creating data directories..."
+chmod +x ./docker/scripts/app-create-dirs.sh
+./docker/scripts/app-create-dirs.sh
+
+echo "Running checks as root to apply patches..."
+/usr/local/bin/python $WORKSPACEDIR/backend/manage.py check
+
+if [[ "${CELERY_ROLE}" == "worker" ]]; then
+    echo "Running initial checks..."
+    # Run checks as celery worker if one was specified
+    run_as_celery_uid /usr/local/bin/python $WORKSPACEDIR/backend/manage.py check
+fi
+
+trap 'trap "" TERM; cleanup' TERM
+# start celery in the background so we can trap the TERM signal
+if [[ -n "${DEV_MODE}" ]]; then
+  watchmedo auto-restart \
+            --patterns '*.py' \
+            --directory 'backend' \
+            --recursive \
+            --debounce-interval 5 \
+            -- \
+            celery --app="${CELERY_APP:-mlarchive.celeryapp:app}" "${CELERY_OPTS[@]}" "$@" &
+  celery_pid=$!
+else
+  celery --app="${CELERY_APP:-mlarchive.celeryapp:app}" "${CELERY_OPTS[@]}" "$@" &
+  celery_pid=$!
+fi
+
+wait "${celery_pid}"