Merge pull request #12 from maxplanck-ie/develop

Develop

.gitignore

@@ -3,3 +3,5 @@
 *.sqldump
 *_dump/
 *_dumps/
+rsnapshot/
+*.swp

.pre-commit-config.yaml

@@ -40,7 +40,13 @@ repos:
         entry: /usr/bin/env sed
         language: script
         files: 'Dockerfile'
-        args: [-i, '/^CMD gunicorn/s/-t [0-9]+/-t 600/']
+        args: [-i, '/^CMD \["gunicorn/s/"-t", "[0-9]+"/"-t", "600"/']
+    -   id: set-prod-docker-3
+        name: set-prod-docker-3
+        entry: /usr/bin/env sed
+        language: script
+        files: 'Dockerfile'
+        args: [-i, '/^CMD \["gunicorn/s/"--reload", //']
     -   id: set-prod-docker-compose
         name: set-prod-docker-compose
         entry: /usr/bin/env sed
@@ -65,6 +71,8 @@ repos:
 #    hooks:
 #    -   id: mypy
 
+# https://github.com/hadolint/hadolint/blob/master/docs/INTEGRATION.md#pre-commit
+
 -   repo: https://github.com/psf/black
     rev: 22.3.0
     hooks:

Dockerfile

@@ -4,8 +4,6 @@ WORKDIR /usr/src/app
 COPY ./parkour_app .
 RUN --mount=type=cache,target=/root/.cache pip install -r requirements/prod.txt
 EXPOSE 8000
-# RUN adduser --system --no-create-home staysafe
-# USER staysafe
 ENV PYTHONBREAKPOINT ipdb.set_trace
 ENV PYTHONUNBUFFERED 1
-CMD gunicorn wui.wsgi:application -t 600 -w 2 -b :8000  # TODO: -k uvicorn.workers.UvicornWorker
+CMD ["gunicorn", "wui.wsgi:application", "-t", "600", "-w", "2", "-b", ":8000"]

Makefile

@@ -8,7 +8,8 @@ deploy-full:  deploy-django deploy-caddy deploy-ready
 set-prod:
 	@sed -i -e '/^DJANGO_SETTINGS_MODULE/s/\(wui\.settings\.\).*/\1prod/' parkour.env
 	@sed -i -e '/^RUN .* pip install/s/\(requirements\/\).*\(\.txt\)/\1prod\2/' Dockerfile
-	@sed -E -i -e '/^CMD gunicorn/s/-t [0-9]+/-t 600/' Dockerfile
+	@sed -E -i -e '/^CMD \["gunicorn/s/"-t", "[0-9]+"/"-t", "600"/' Dockerfile
+	@sed -E -i -e '/^CMD \["gunicorn/s/"--reload", //' Dockerfile
 	@sed -E -i -e '/^ +tty/s/: .*/: false/' \
 			-e '/^ +stdin_open/s/: .*/: false/' docker-compose.yml
 
@@ -32,18 +33,18 @@ load-migrations:
 	docker compose run parkour2-django python manage.py migrate --noinput > /dev/null
 
 stop:
-	@docker compose -f docker-compose.yml -f caddy.yml -f nginx.yml -f ncdb.yml stop
+	@docker compose -f docker-compose.yml -f caddy.yml -f nginx.yml -f rsnapshot.yml -f ncdb.yml stop
 
 down:
-	@docker compose -f docker-compose.yml -f caddy.yml -f nginx.yml -f ncdb.yml down --volumes
+	@docker compose -f docker-compose.yml -f caddy.yml -f nginx.yml -f rsnapshot.yml -f ncdb.yml down --volumes
 
 clean: set-prod unset-caddy
 	@echo "Config reset OK. Cleaning? Try: make prune"
 
 prune:
 	@docker system prune -a -f --volumes
 
-prod: set-prod deploy-django deploy-nginx deploy-ready
+prod: set-prod deploy-django deploy-nginx deploy-ready deploy-rsnapshot
 
 dev0: set-dev set-caddy deploy-full load-backup
 
@@ -52,7 +53,7 @@ dev: set-dev deploy-django deploy-nginx deploy-ready load-backup load-migrations
 set-dev:
 	@sed -i -e '/^DJANGO_SETTINGS_MODULE/s/\(wui\.settings\.\).*/\1dev/' parkour.env
 	@sed -i -e '/^RUN .* pip install/s/\(requirements\/\).*\(\.txt\)/\1dev\2/' Dockerfile
-	@sed -E -i -e '/^CMD gunicorn/s/-t [0-9]+/-t 3600/' Dockerfile
+	@sed -E -i -e '/^CMD \["gunicorn/s/"-t", "[0-9]+"/"--reload", "-t", "3600"/' Dockerfile
 	@sed -E -i -e '/^ +tty/s/: .*/: true/' \
 			-e '/^ +stdin_open/s/: .*/: true/' docker-compose.yml
 
@@ -89,9 +90,14 @@ save-media:
 	@docker cp parkour2-django:/usr/src/app/media/ . && mv media media_dump
 
 save-postgres:
-	@docker exec parkour2-postgres pg_dump -d postgres -U postgres -f /tmp/postgres_dump -b -c -C --if-exists --inserts && \
+	@docker exec -it parkour2-postgres pg_dump -Fc postgres -U postgres -f /tmp/postgres_dump && \
 		docker cp parkour2-postgres:/tmp/postgres_dump latest.sqldump
 
+deploy-rsnapshot:
+	@docker compose -f rsnapshot.yml up -d && \
+		sleep 1m && \
+		docker exec -it parkour2-rsnapshot rsnapshot daily
+
 test: down clean prod
 	@echo "Testing on a 'clean' production deployment..."
 	@docker compose run parkour2-django python -Wa manage.py test

parkour_app/requirements/base.txt

@@ -31,11 +31,12 @@ charset-normalizer==2.1.0
     #   requests
 colorama==0.4.5
     # via sphinx-autobuild
-dj-database-url==0.5.0
+dj-database-url==1.0.0
     # via -r requirements/base.in
 django==3.2.15
     # via
     #   -r requirements/base.in
+    #   dj-database-url
     #   django-authtools
     #   django-extensions
     #   djangorestframework
@@ -55,7 +56,7 @@ docutils==0.17.1
     #   sphinx-rtd-theme
 fpdf==1.7.2
     # via -r requirements/base.in
-frozenlist==1.3.0
+frozenlist==1.3.1
     # via
     #   aiohttp
     #   aiosignal
@@ -159,7 +160,7 @@ urllib3==1.26.11
     # via requests
 xlwt==1.3.0
     # via -r requirements/base.in
-yarl==1.7.2
+yarl==1.8.1
     # via aiohttp
 zipp==3.8.1
     # via importlib-metadata

rsnapshot.yml

@@ -0,0 +1,21 @@
+version: "3.9"
+
+services:
+  parkour2-rsnapshot:
+    container_name: parkour2-rsnapshot
+    image: lscr.io/linuxserver/rsnapshot:latest
+    restart: always
+    environment:
+      - TZ=Europe/Berlin
+    volumes:
+      - ./rsnapshot/config:/config
+      - ./rsnapshot/backups:/.snapshots
+      # sources go into /data
+      - parkour2_pgdb:/data/parkour2_pgdb
+      - parkour2_media:/data/parkour2_media
+
+volumes:
+  parkour2_pgdb:
+    external: true
+  parkour2_media:
+    external: true

rsnapshot/config/crontabs/root

@@ -0,0 +1,11 @@
+# min   hour    day     month   weekday command
+*/15    *       *       *       *       run-parts /etc/periodic/15min
+0       *       *       *       *       run-parts /etc/periodic/hourly
+0       2       *       *       *       run-parts /etc/periodic/daily
+0       3       *       *       6       run-parts /etc/periodic/weekly
+0       5       1       *       *       run-parts /etc/periodic/monthly
+# https://github.com/rsnapshot/rsnapshot/blob/master/README.md#configuration
+00 00 * * *     /usr/local/bin/rsnapshot daily
+00 23 * * 6     /usr/local/bin/rsnapshot weekly
+00 22 1 * *     /usr/local/bin/rsnapshot monthly
+00 01 1 1 *     /usr/local/bin/rsnapshot yearly