Docker dev (#3656)

* ci: reduce app-base image size. add wait-for util

* ci: docker tweaks

compose-dev.yml

@@ -0,0 +1,49 @@
+version: '3.8'
+
+services:
+  app:
+    environment: 
+        - DJANGO_SETTINGS_MODULE=mlarchive.settings.settings_docker
+    build:
+      context: .
+      dockerfile: docker/app-dev.Dockerfile
+
+    init: true
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+    # network_mode: service:db
+
+    depends_on:
+        - db
+        - es
+        - rabbit
+
+    ipc: host
+
+  es:
+    image: "elasticsearch:7.17.4"
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+    environment:
+        - discovery.type=single-node
+        - xpack.security.enabled=false
+
+  db:
+    image: postgres:14.6
+    restart: always
+    environment:
+      POSTGRES_DB: mailarch
+      POSTGRES_USER: django
+      POSTGRES_PASSWORD: franticmarble
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  rabbit:
+    image: rabbitmq:3.10
+    restart: always
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)

docker/app-dev.Dockerfile

@@ -0,0 +1,58 @@
+FROM ghcr.io/ietf-tools/mailarchive-app-base:latest
+LABEL maintainer="IETF Tools Team <[email protected]>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Copy library scripts to execute
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/
+
+# [Option] Install zsh
+ARG INSTALL_ZSH="true"
+# [Option] Upgrade OS packages to their latest versions
+ARG UPGRADE_PACKAGES="true"
+# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
+ARG USERNAME=dev
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    # Install common packages, non-root user
+    && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true"
+
+# Setup default python tools in a venv via pipx to avoid conflicts
+ENV PIPX_HOME=/usr/local/py-utils \
+    PIPX_BIN_DIR=/usr/local/py-utils/bin
+ENV PATH=${PATH}:${PIPX_BIN_DIR}
+RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}"
+
+# Remove library scripts for final image
+RUN rm -rf /tmp/library-scripts
+
+# Copy the startup file
+COPY docker/scripts/app-init-dev.sh /docker-init.sh
+RUN sed -i 's/\r$//' /docker-init.sh && \
+    chmod +x /docker-init.sh
+
+# Fix user UID / GID to match host
+RUN groupmod --gid $USER_GID $USERNAME \
+    && usermod --uid $USER_UID --gid $USER_GID $USERNAME \
+    && chown -R $USER_UID:$USER_GID /home/$USERNAME \
+    || exit 0
+
+# Switch to local dev user
+USER dev:dev
+
+# Install current datatracker python dependencies
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location -r /tmp/pip-tmp/requirements.txt
+RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location pylint pylint-common pylint-django
+RUN sudo rm -rf /tmp/pip-tmp
+
+# Copy app files
+COPY . .
+
+# host with gunicorn
+CMD gunicorn --workers=4 -b 0.0.0.0:80 'backend.wsgi:application'
+
+# VOLUME [ "/assets" ]

docker/base.Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update \
     && apt-get -qy upgrade \
     && apt-get -y install --no-install-recommends apt-utils dialog 2>&1
 
-# Add PostgreSQL Source 
+# Add Postgresql Apt Repository to get 14 
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
@@ -34,6 +34,9 @@ RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \
     chmod +x /tmp/app-install-chromedriver.sh
 RUN /tmp/app-install-chromedriver.sh
 
+# Get rid of installation files we don't need in the image, to reduce size
+RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
 # "fake" dbus address to prevent errors
 # https://github.com/SeleniumHQ/docker-selenium/issues/87
 ENV DBUS_SESSION_BUS_ADDRESS=/dev/null
@@ -47,6 +50,10 @@ RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
     update-locale LC_ALL en_US.UTF-8
 ENV LC_ALL en_US.UTF-8
 
+# Fetch wait-for utility
+ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/
+RUN chmod +rx /usr/local/bin/wait-for
+
 # Create data directory
 RUN mkdir -p /data
 

docker/run-dev

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Usage info
+show_help() {
+    cat << EOF
+Usage: ${0##*/} [-h] [-p PORT] [-r]
+Run mail archive in dev containers using docker-compose.
+    -h          display this help and exit
+    -p PORT     use custom HTTP port for mail archive
+    -r          force rebuild the app container
+EOF
+}
+
+CUSTOM_PORT=8000
+FORCE_REBUILD=0
+
+while getopts "hp:r" opt; do
+    case $opt in
+        h)
+            show_help
+            exit 0
+            ;;
+        p)  
+            CUSTOM_PORT=$OPTARG
+            echo "Using custom port $CUSTOM_PORT..."
+            ;;
+        r)
+            FORCE_REBUILD=1
+            echo "Will force rebuild the app container..."
+            ;;
+        *)
+            CUSTOM_PORT=8000
+            echo "Using port 8000..."
+            ;;
+    esac
+done
+
+# Remove mounted temp directories
+rm -rf .parcel-cache __pycache__
+
+# Create extended docker-compose definition
+# cp docker-compose.extend.yml docker-compose.extend-custom.yml
+# sed -i -r -e "s/CUSTOM_PORT/$CUSTOM_PORT/" docker-compose.extend-custom.yml
+# cd ..
+
+# Set UID/GID mappings
+# NEW_UID=$(id -u)
+# NEW_GID=$(id -g)
+# if [ $NEW_UID -gt 0 ]; then
+#    echo "Will use the following user/group mapping:"
+#    echo "USER ID: $NEW_UID"
+#    echo "GROUP ID: $NEW_GID"
+#else
+#    echo "Running as root, will use default user/group mapping..."
+#    echo "NOT RECOMMENDED - You may experience permission issues."
+#    NEW_UID=1000
+#    NEW_GID=1000
+#fi
+
+# Build / Rebuild Containers
+if [ $FORCE_REBUILD ==  1 ]; then
+    docker compose -f compose-dev.yml down
+    docker compose -f compose-dev.yml rm -f
+    docker compose -f compose-dev.yml build --no-cache --pull
+    docker compose -f compose-dev.yml up -d --force-recreate
+else
+    docker compose -f compose-dev.yml build 
+    docker compose -f compose-dev.yml up -d
+fi
+
+# Output database port
+echo "Database exposed on port:"
+docker compose port db 5432
+
+# Start init script
+docker compose exec app /bin/zsh /docker-init.sh
+
+# Exit scripts
+docker compose -f compose-dev.yml stop
+cd docker
+# rm -f docker-compose.extend-custom.yml

docker/scripts/app-init-dev.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+WORKSPACEDIR="/workspace"
+
+sudo service rsyslog start &>/dev/null
+
+# Fix ownership of volumes
+echo "Fixing volumes ownership..."
+sudo chown -R dev:dev "$WORKSPACEDIR"
+sudo chown dev:dev "/data"
+
+echo "Fix chromedriver /dev/shm permissions..."
+sudo chmod 1777 /dev/shm
+
+# Build node packages that requrie native compilation
+# echo "Compiling native node packages..."
+# yarn rebuild
+
+# Generate static assets
+# echo "Building static assets... (this could take a minute or two)"
+# yarn build
+
+# Copy config files if needed
+
+if [ ! -f "$WORKSPACEDIR/.env" ]; then
+    echo "Setting up a default .env ..."
+    cp $WORKSPACEDIR/docker/configs/docker_env $WORKSPACEDIR/.env
+else
+    echo "Using existing .env file"
+    if ! cmp -s $WORKSPACEDIR/docker/configs/docker_env $WORKSPACEDIR/.env; then
+        echo "NOTE: Differences detected compared to docker/configs/docker_env!"
+        echo "We'll assume you made these deliberately."
+    fi
+fi
+
+if [ ! -f "$WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py" ]; then
+    echo "Setting up a default settings_docker.py ..."
+else
+    echo "Renaming existing backend/mlarchive/settings/settings_docker.py to backend/mlarchive/settings/settings_docker.py.bak"
+    mv -f $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py.bak
+fi
+cp $WORKSPACEDIR/docker/configs/settings_docker.py $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py
+
+# Create data directories
+echo "Creating data directories..."
+for sub in \
+    /data/archive \
+    /data/log/mail-archive \
+    ; do
+    if [ ! -d "$sub"  ]; then
+        echo "Creating dir $sub"
+        mkdir -p "$sub";
+    fi
+    sudo chown -R dev:dev "/data"
+done
+
+
+# Wait for DB container
+
+if [ -n "$EDITOR_VSCODE" ]; then
+    echo "Waiting for DB container to come online ..."
+    /usr/local/bin/wait-for localhost:5432 -- echo "DB ready"
+fi
+
+# Run memcached
+
+echo "Starting memcached..."
+/usr/bin/memcached -u dev -d
+
+# Initial checks
+
+echo "Running initial checks..."
+/usr/local/bin/python $WORKSPACEDIR/backend/manage.py check
+/usr/local/bin/python $WORKSPACEDIR/backend/manage.py migrate
+
+echo "-----------------------------------------------------------------"
+echo "Done!"
+echo "-----------------------------------------------------------------"
+
+if [ -z "$EDITOR_VSCODE" ]; then
+    CODE=0
+    python -m smtpd -n -c DebuggingServer localhost:2025 &
+    if [ -z "$*" ]; then
+        echo
+        echo "You can execute arbitrary commands now, e.g.,"
+        echo
+        echo "    backend/manage.py check && backend/manage.py runserver 0.0.0.0:8000"
+        echo
+        echo "to start a development instance of the Mail Archive."
+        echo
+        echo "    cd backend/mlarchive && pytest tests"
+        echo
+        echo "to run all the tests."
+        echo
+        zsh
+    else
+        echo "Executing \"$*\" and stopping container."
+        echo
+        zsh -c "$*"
+        CODE=$?
+    fi
+    sudo service rsyslog stop
+    exit $CODE
+fi

docker/scripts/app-init.sh

@@ -87,7 +87,7 @@ if [ -z "$EDITOR_VSCODE" ]; then
         echo
         echo "    backend/manage.py check && backend/manage.py runserver 0.0.0.0:8000"
         echo
-        echo "to start a development instance of the Datatracker."
+        echo "to start a development instance of the Mail Archive."
         echo
         echo "    cd backend/mlarchive && pytest tests"
         echo