diff --git a/config.py b/config.py index c01a6fb..e1d7815 100644 --- a/config.py +++ b/config.py @@ -7,7 +7,7 @@ from pyproj import Transformer NAME = 'openaedmap-backend' -VERSION = '2.9.0' +VERSION = '2.9.1' CREATED_BY = f'{NAME} {VERSION}' WEBSITE = 'https://openaedmap.org' @@ -27,8 +27,8 @@ ) POSTGRES_LOG = os.getenv('POSTGRES_LOG', '0').strip().lower() in ('1', 'true', 'yes') -POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@127.0.0.1/postgres' -REDIS_URL = os.getenv('REDIS_URL', 'redis://127.0.0.1?protocol=3') +POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@/postgres?host=/tmp/openaedmap-postgres' +REDIS_URL = os.getenv('REDIS_URL', 'unix:///tmp/openaedmap-redis.sock?protocol=3') DEFAULT_CACHE_MAX_AGE = timedelta(minutes=1) DEFAULT_CACHE_STALE = timedelta(minutes=5) diff --git a/config/postgres.conf b/config/postgres.conf index b5cbba6..f7c1f24 100644 --- a/config/postgres.conf +++ b/config/postgres.conf @@ -7,14 +7,14 @@ # disable listening on unix socket # reason: unused, improved compatibility -unix_socket_directories = '' +unix_socket_directories = '/tmp/openaedmap-postgres' # adjust memory usage shared_buffers = 512MB effective_cache_size = 1GB # increase max connections -max_connections = 1000 +max_connections = 10000 # detect disconnected clients # reason: safeguard resource usage diff --git a/config/redis.conf b/config/redis.conf index 4778136..cc2c56c 100644 --- a/config/redis.conf +++ b/config/redis.conf @@ -5,6 +5,18 @@ # - 2GB RAM # - 6GB SSD +# listen on socket +# reason: reduce latency +unixsocket /tmp/openaedmap-redis.sock +unixsocketperm 700 + +# single-database mode +databases 1 + +# use simple locale collate +# reason: faster sorting without bias +locale-collate C + # disable persistence # reason: redis is cache only, use postgres for persistence save "" diff --git a/db.py b/db.py index f2cead2..40c80f1 100644 --- a/db.py +++ b/db.py @@ -12,7 +12,7 @@ echo_pool=POSTGRES_LOG, json_deserializer=JSON_DECODE, json_serializer=lambda x: JSON_ENCODE(x).decode(), - pool_size=8, + pool_size=10, max_overflow=-1, query_cache_size=128, ) diff --git a/default.nix b/default.nix index f79cc1e..2450f5b 100644 --- a/default.nix +++ b/default.nix @@ -23,6 +23,7 @@ let entrypoint = pkgs.writeShellScriptBin "entrypoint" '' set -ex + rm -f data/postgres/postmaster.pid dev-start set -o allexport source "envs/app/${envTag}.env" set @@ -42,6 +43,7 @@ with pkgs; dockerTools.buildLayeredImage { extraCommands = '' set -e + mkdir tmp mkdir app && cd app mkdir -p data/postgres data/photos cp "${./.}"/*.py . @@ -57,6 +59,7 @@ with pkgs; dockerTools.buildLayeredImage { fakeRootCommands = '' set -e ${dockerTools.shadowSetup} + chmod 0777 tmp groupadd --system -g 999 docker useradd --system --no-create-home -u 999 -g 999 docker chown -R docker:docker app diff --git a/docker-compose.yml b/docker-compose.yml index c07fe53..ead156d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: app: image: backend:${TAG} diff --git a/services/aed_service.py b/services/aed_service.py index a9bfaef..5f1e65e 100644 --- a/services/aed_service.py +++ b/services/aed_service.py @@ -204,6 +204,11 @@ async def _update_db_snapshot() -> None: logging.info('Updating country codes') await _assign_country_codes(aeds) + logging.info('Updating statistics') + async with db_write() as session: + await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'}) + await session.execute(text(f'ANALYZE "{AED.__tablename__}"')) + logging.info('AED update finished (=%d)', len(aeds)) diff --git a/services/country_service.py b/services/country_service.py index 1e23de4..e19ced0 100644 --- a/services/country_service.py +++ b/services/country_service.py @@ -12,6 +12,7 @@ from country_code_assigner import CountryCodeAssigner from db import db_read, db_write from models.bbox import BBox +from models.db.aed import AED from models.db.country import Country from osm_countries import get_osm_countries from services.state_service import StateService @@ -108,6 +109,12 @@ async def _update_db() -> None: from services.aed_service import AEDService await AEDService.update_country_codes() + + logging.info('Updating statistics') + async with db_write() as session: + await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'}) + await session.execute(text(f'ANALYZE "{AED.__tablename__}", "{Country.__tablename__}"')) + logging.info('Country update finished') diff --git a/shell.nix b/shell.nix index 1fc1c08..e21aad9 100644 --- a/shell.nix +++ b/shell.nix @@ -65,12 +65,12 @@ let --pwfile=<(echo postgres) fi - mkdir -p data/supervisor + mkdir -p /tmp/openaedmap-postgres data/supervisor supervisord -c config/supervisord.conf echo "Supervisor started" echo "Waiting for Postgres to start..." - while ! pg_isready -q -h 127.0.0.1 -t 10; do sleep 0.1; done + while ! pg_isready -q -h /tmp/openaedmap-postgres -t 10; do sleep 0.1; done echo "Postgres started, running migrations" alembic-upgrade '')