-
Notifications
You must be signed in to change notification settings - Fork 25
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* ci: reduce app-base image size. add wait-for util * ci: docker tweaks
- Loading branch information
Showing
6 changed files
with
301 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
version: '3.8' | ||
|
||
services: | ||
app: | ||
environment: | ||
- DJANGO_SETTINGS_MODULE=mlarchive.settings.settings_docker | ||
build: | ||
context: . | ||
dockerfile: docker/app-dev.Dockerfile | ||
|
||
init: true | ||
|
||
# Overrides default command so things don't shut down after the process ends. | ||
command: sleep infinity | ||
|
||
# Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. | ||
# network_mode: service:db | ||
|
||
depends_on: | ||
- db | ||
- es | ||
- rabbit | ||
|
||
ipc: host | ||
|
||
es: | ||
image: "elasticsearch:7.17.4" | ||
# Use "forwardPorts" in **devcontainer.json** to forward an app port locally. | ||
# (Adding the "ports" property to this file will not forward from a Codespace.) | ||
environment: | ||
- discovery.type=single-node | ||
- xpack.security.enabled=false | ||
|
||
db: | ||
image: postgres:14.6 | ||
restart: always | ||
environment: | ||
POSTGRES_DB: mailarch | ||
POSTGRES_USER: django | ||
POSTGRES_PASSWORD: franticmarble | ||
POSTGRES_HOST_AUTH_METHOD: trust | ||
# Use "forwardPorts" in **devcontainer.json** to forward an app port locally. | ||
# (Adding the "ports" property to this file will not forward from a Codespace.) | ||
|
||
rabbit: | ||
image: rabbitmq:3.10 | ||
restart: always | ||
# Use "forwardPorts" in **devcontainer.json** to forward an app port locally. | ||
# (Adding the "ports" property to this file will not forward from a Codespace.) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
FROM ghcr.io/ietf-tools/mailarchive-app-base:latest | ||
LABEL maintainer="IETF Tools Team <[email protected]>" | ||
|
||
ENV DEBIAN_FRONTEND=noninteractive | ||
|
||
# Copy library scripts to execute | ||
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/ | ||
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/ | ||
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/ | ||
|
||
# [Option] Install zsh | ||
ARG INSTALL_ZSH="true" | ||
# [Option] Upgrade OS packages to their latest versions | ||
ARG UPGRADE_PACKAGES="true" | ||
# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies. | ||
ARG USERNAME=dev | ||
ARG USER_UID=1000 | ||
ARG USER_GID=$USER_UID | ||
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ | ||
# Install common packages, non-root user | ||
&& bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" | ||
|
||
# Setup default python tools in a venv via pipx to avoid conflicts | ||
ENV PIPX_HOME=/usr/local/py-utils \ | ||
PIPX_BIN_DIR=/usr/local/py-utils/bin | ||
ENV PATH=${PATH}:${PIPX_BIN_DIR} | ||
RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}" | ||
|
||
# Remove library scripts for final image | ||
RUN rm -rf /tmp/library-scripts | ||
|
||
# Copy the startup file | ||
COPY docker/scripts/app-init-dev.sh /docker-init.sh | ||
RUN sed -i 's/\r$//' /docker-init.sh && \ | ||
chmod +x /docker-init.sh | ||
|
||
# Fix user UID / GID to match host | ||
RUN groupmod --gid $USER_GID $USERNAME \ | ||
&& usermod --uid $USER_UID --gid $USER_GID $USERNAME \ | ||
&& chown -R $USER_UID:$USER_GID /home/$USERNAME \ | ||
|| exit 0 | ||
|
||
# Switch to local dev user | ||
USER dev:dev | ||
|
||
# Install current datatracker python dependencies | ||
COPY requirements.txt /tmp/pip-tmp/ | ||
RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location -r /tmp/pip-tmp/requirements.txt | ||
RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location pylint pylint-common pylint-django | ||
RUN sudo rm -rf /tmp/pip-tmp | ||
|
||
# Copy app files | ||
COPY . . | ||
|
||
# host with gunicorn | ||
CMD gunicorn --workers=4 -b 0.0.0.0:80 'backend.wsgi:application' | ||
|
||
# VOLUME [ "/assets" ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
#!/bin/bash | ||
|
||
# Usage info | ||
show_help() { | ||
cat << EOF | ||
Usage: ${0##*/} [-h] [-p PORT] [-r] | ||
Run mail archive in dev containers using docker-compose. | ||
-h display this help and exit | ||
-p PORT use custom HTTP port for mail archive | ||
-r force rebuild the app container | ||
EOF | ||
} | ||
|
||
CUSTOM_PORT=8000 | ||
FORCE_REBUILD=0 | ||
|
||
while getopts "hp:r" opt; do | ||
case $opt in | ||
h) | ||
show_help | ||
exit 0 | ||
;; | ||
p) | ||
CUSTOM_PORT=$OPTARG | ||
echo "Using custom port $CUSTOM_PORT..." | ||
;; | ||
r) | ||
FORCE_REBUILD=1 | ||
echo "Will force rebuild the app container..." | ||
;; | ||
*) | ||
CUSTOM_PORT=8000 | ||
echo "Using port 8000..." | ||
;; | ||
esac | ||
done | ||
|
||
# Remove mounted temp directories | ||
rm -rf .parcel-cache __pycache__ | ||
|
||
# Create extended docker-compose definition | ||
# cp docker-compose.extend.yml docker-compose.extend-custom.yml | ||
# sed -i -r -e "s/CUSTOM_PORT/$CUSTOM_PORT/" docker-compose.extend-custom.yml | ||
# cd .. | ||
|
||
# Set UID/GID mappings | ||
# NEW_UID=$(id -u) | ||
# NEW_GID=$(id -g) | ||
# if [ $NEW_UID -gt 0 ]; then | ||
# echo "Will use the following user/group mapping:" | ||
# echo "USER ID: $NEW_UID" | ||
# echo "GROUP ID: $NEW_GID" | ||
#else | ||
# echo "Running as root, will use default user/group mapping..." | ||
# echo "NOT RECOMMENDED - You may experience permission issues." | ||
# NEW_UID=1000 | ||
# NEW_GID=1000 | ||
#fi | ||
|
||
# Build / Rebuild Containers | ||
if [ $FORCE_REBUILD == 1 ]; then | ||
docker compose -f compose-dev.yml down | ||
docker compose -f compose-dev.yml rm -f | ||
docker compose -f compose-dev.yml build --no-cache --pull | ||
docker compose -f compose-dev.yml up -d --force-recreate | ||
else | ||
docker compose -f compose-dev.yml build | ||
docker compose -f compose-dev.yml up -d | ||
fi | ||
|
||
# Output database port | ||
echo "Database exposed on port:" | ||
docker compose port db 5432 | ||
|
||
# Start init script | ||
docker compose exec app /bin/zsh /docker-init.sh | ||
|
||
# Exit scripts | ||
docker compose -f compose-dev.yml stop | ||
cd docker | ||
# rm -f docker-compose.extend-custom.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
#!/bin/bash | ||
|
||
WORKSPACEDIR="/workspace" | ||
|
||
sudo service rsyslog start &>/dev/null | ||
|
||
# Fix ownership of volumes | ||
echo "Fixing volumes ownership..." | ||
sudo chown -R dev:dev "$WORKSPACEDIR" | ||
sudo chown dev:dev "/data" | ||
|
||
echo "Fix chromedriver /dev/shm permissions..." | ||
sudo chmod 1777 /dev/shm | ||
|
||
# Build node packages that requrie native compilation | ||
# echo "Compiling native node packages..." | ||
# yarn rebuild | ||
|
||
# Generate static assets | ||
# echo "Building static assets... (this could take a minute or two)" | ||
# yarn build | ||
|
||
# Copy config files if needed | ||
|
||
if [ ! -f "$WORKSPACEDIR/.env" ]; then | ||
echo "Setting up a default .env ..." | ||
cp $WORKSPACEDIR/docker/configs/docker_env $WORKSPACEDIR/.env | ||
else | ||
echo "Using existing .env file" | ||
if ! cmp -s $WORKSPACEDIR/docker/configs/docker_env $WORKSPACEDIR/.env; then | ||
echo "NOTE: Differences detected compared to docker/configs/docker_env!" | ||
echo "We'll assume you made these deliberately." | ||
fi | ||
fi | ||
|
||
if [ ! -f "$WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py" ]; then | ||
echo "Setting up a default settings_docker.py ..." | ||
else | ||
echo "Renaming existing backend/mlarchive/settings/settings_docker.py to backend/mlarchive/settings/settings_docker.py.bak" | ||
mv -f $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py.bak | ||
fi | ||
cp $WORKSPACEDIR/docker/configs/settings_docker.py $WORKSPACEDIR/backend/mlarchive/settings/settings_docker.py | ||
|
||
# Create data directories | ||
echo "Creating data directories..." | ||
for sub in \ | ||
/data/archive \ | ||
/data/log/mail-archive \ | ||
; do | ||
if [ ! -d "$sub" ]; then | ||
echo "Creating dir $sub" | ||
mkdir -p "$sub"; | ||
fi | ||
sudo chown -R dev:dev "/data" | ||
done | ||
|
||
|
||
# Wait for DB container | ||
|
||
if [ -n "$EDITOR_VSCODE" ]; then | ||
echo "Waiting for DB container to come online ..." | ||
/usr/local/bin/wait-for localhost:5432 -- echo "DB ready" | ||
fi | ||
|
||
# Run memcached | ||
|
||
echo "Starting memcached..." | ||
/usr/bin/memcached -u dev -d | ||
|
||
# Initial checks | ||
|
||
echo "Running initial checks..." | ||
/usr/local/bin/python $WORKSPACEDIR/backend/manage.py check | ||
/usr/local/bin/python $WORKSPACEDIR/backend/manage.py migrate | ||
|
||
echo "-----------------------------------------------------------------" | ||
echo "Done!" | ||
echo "-----------------------------------------------------------------" | ||
|
||
if [ -z "$EDITOR_VSCODE" ]; then | ||
CODE=0 | ||
python -m smtpd -n -c DebuggingServer localhost:2025 & | ||
if [ -z "$*" ]; then | ||
echo | ||
echo "You can execute arbitrary commands now, e.g.," | ||
echo | ||
echo " backend/manage.py check && backend/manage.py runserver 0.0.0.0:8000" | ||
echo | ||
echo "to start a development instance of the Mail Archive." | ||
echo | ||
echo " cd backend/mlarchive && pytest tests" | ||
echo | ||
echo "to run all the tests." | ||
echo | ||
zsh | ||
else | ||
echo "Executing \"$*\" and stopping container." | ||
echo | ||
zsh -c "$*" | ||
CODE=$? | ||
fi | ||
sudo service rsyslog stop | ||
exit $CODE | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters