From 1eef4e1516aea0d1a87fbae9ae5ca21865735167 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Wed, 10 Apr 2024 14:46:18 +0000 Subject: [PATCH 1/3] Use unix sockets, Update db statistics after big changes --- config.py | 6 +++--- config/postgres.conf | 4 ++-- config/redis.conf | 12 ++++++++++++ db.py | 2 +- default.nix | 1 + services/aed_service.py | 5 +++++ services/country_service.py | 7 +++++++ shell.nix | 4 ++-- 8 files changed, 33 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index c01a6fb..e1d7815 100644 --- a/config.py +++ b/config.py @@ -7,7 +7,7 @@ from pyproj import Transformer NAME = 'openaedmap-backend' -VERSION = '2.9.0' +VERSION = '2.9.1' CREATED_BY = f'{NAME} {VERSION}' WEBSITE = 'https://openaedmap.org' @@ -27,8 +27,8 @@ ) POSTGRES_LOG = os.getenv('POSTGRES_LOG', '0').strip().lower() in ('1', 'true', 'yes') -POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@127.0.0.1/postgres' -REDIS_URL = os.getenv('REDIS_URL', 'redis://127.0.0.1?protocol=3') +POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@/postgres?host=/tmp/openaedmap-postgres' +REDIS_URL = os.getenv('REDIS_URL', 'unix:///tmp/openaedmap-redis.sock?protocol=3') DEFAULT_CACHE_MAX_AGE = timedelta(minutes=1) DEFAULT_CACHE_STALE = timedelta(minutes=5) diff --git a/config/postgres.conf b/config/postgres.conf index b5cbba6..f7c1f24 100644 --- a/config/postgres.conf +++ b/config/postgres.conf @@ -7,14 +7,14 @@ # disable listening on unix socket # reason: unused, improved compatibility -unix_socket_directories = '' +unix_socket_directories = '/tmp/openaedmap-postgres' # adjust memory usage shared_buffers = 512MB effective_cache_size = 1GB # increase max connections -max_connections = 1000 +max_connections = 10000 # detect disconnected clients # reason: safeguard resource usage diff --git a/config/redis.conf b/config/redis.conf index 4778136..cc2c56c 100644 --- a/config/redis.conf +++ b/config/redis.conf @@ -5,6 +5,18 @@ # - 2GB RAM # - 6GB SSD +# listen on socket +# reason: reduce latency +unixsocket /tmp/openaedmap-redis.sock +unixsocketperm 700 + +# single-database mode +databases 1 + +# use simple locale collate +# reason: faster sorting without bias +locale-collate C + # disable persistence # reason: redis is cache only, use postgres for persistence save "" diff --git a/db.py b/db.py index f2cead2..40c80f1 100644 --- a/db.py +++ b/db.py @@ -12,7 +12,7 @@ echo_pool=POSTGRES_LOG, json_deserializer=JSON_DECODE, json_serializer=lambda x: JSON_ENCODE(x).decode(), - pool_size=8, + pool_size=10, max_overflow=-1, query_cache_size=128, ) diff --git a/default.nix b/default.nix index f79cc1e..223df37 100644 --- a/default.nix +++ b/default.nix @@ -42,6 +42,7 @@ with pkgs; dockerTools.buildLayeredImage { extraCommands = '' set -e + mkdir tmp mkdir app && cd app mkdir -p data/postgres data/photos cp "${./.}"/*.py . diff --git a/services/aed_service.py b/services/aed_service.py index 3a7d4d0..aa826d8 100644 --- a/services/aed_service.py +++ b/services/aed_service.py @@ -204,6 +204,11 @@ async def _update_db_snapshot() -> None: logging.info('Updating country codes') await _assign_country_codes(aeds) + logging.info('Updating statistics') + async with db_write() as session: + await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'}) + await session.execute(text(f'ANALYZE "{AED.__tablename__}"')) + logging.info('AED update finished (=%d)', len(aeds)) diff --git a/services/country_service.py b/services/country_service.py index 1e23de4..e19ced0 100644 --- a/services/country_service.py +++ b/services/country_service.py @@ -12,6 +12,7 @@ from country_code_assigner import CountryCodeAssigner from db import db_read, db_write from models.bbox import BBox +from models.db.aed import AED from models.db.country import Country from osm_countries import get_osm_countries from services.state_service import StateService @@ -108,6 +109,12 @@ async def _update_db() -> None: from services.aed_service import AEDService await AEDService.update_country_codes() + + logging.info('Updating statistics') + async with db_write() as session: + await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'}) + await session.execute(text(f'ANALYZE "{AED.__tablename__}", "{Country.__tablename__}"')) + logging.info('Country update finished') diff --git a/shell.nix b/shell.nix index 1fc1c08..e21aad9 100644 --- a/shell.nix +++ b/shell.nix @@ -65,12 +65,12 @@ let --pwfile=<(echo postgres) fi - mkdir -p data/supervisor + mkdir -p /tmp/openaedmap-postgres data/supervisor supervisord -c config/supervisord.conf echo "Supervisor started" echo "Waiting for Postgres to start..." - while ! pg_isready -q -h 127.0.0.1 -t 10; do sleep 0.1; done + while ! pg_isready -q -h /tmp/openaedmap-postgres -t 10; do sleep 0.1; done echo "Postgres started, running migrations" alembic-upgrade '') From d56872d61f6a181c5bf8ff32cf4ca0be25cbfc96 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Wed, 10 Apr 2024 19:02:34 +0000 Subject: [PATCH 2/3] Fix tmp permissions --- default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/default.nix b/default.nix index 223df37..fb78421 100644 --- a/default.nix +++ b/default.nix @@ -58,6 +58,7 @@ with pkgs; dockerTools.buildLayeredImage { fakeRootCommands = '' set -e ${dockerTools.shadowSetup} + chmod 0777 tmp groupadd --system -g 999 docker useradd --system --no-create-home -u 999 -g 999 docker chown -R docker:docker app From 780feb98f64fab89632399a961e7a1013992f46a Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Wed, 10 Apr 2024 19:13:38 +0000 Subject: [PATCH 3/3] Remove deprecated version from compose, Force cleanup pid on start --- default.nix | 1 + docker-compose.yml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/default.nix b/default.nix index fb78421..2450f5b 100644 --- a/default.nix +++ b/default.nix @@ -23,6 +23,7 @@ let entrypoint = pkgs.writeShellScriptBin "entrypoint" '' set -ex + rm -f data/postgres/postmaster.pid dev-start set -o allexport source "envs/app/${envTag}.env" set diff --git a/docker-compose.yml b/docker-compose.yml index c07fe53..ead156d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: app: image: backend:${TAG}