Merge pull request #76 from openstreetmap-polska/dev

Release to main

config.py

@@ -7,7 +7,7 @@
 from pyproj import Transformer
 
 NAME = 'openaedmap-backend'
-VERSION = '2.9.0'
+VERSION = '2.9.1'
 CREATED_BY = f'{NAME} {VERSION}'
 WEBSITE = 'https://openaedmap.org'
 
@@ -27,8 +27,8 @@
     )
 
 POSTGRES_LOG = os.getenv('POSTGRES_LOG', '0').strip().lower() in ('1', 'true', 'yes')
-POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@127.0.0.1/postgres'
-REDIS_URL = os.getenv('REDIS_URL', 'redis://127.0.0.1?protocol=3')
+POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@/postgres?host=/tmp/openaedmap-postgres'
+REDIS_URL = os.getenv('REDIS_URL', 'unix:///tmp/openaedmap-redis.sock?protocol=3')
 
 DEFAULT_CACHE_MAX_AGE = timedelta(minutes=1)
 DEFAULT_CACHE_STALE = timedelta(minutes=5)

config/postgres.conf

@@ -7,14 +7,14 @@
 
 # disable listening on unix socket
 # reason: unused, improved compatibility
-unix_socket_directories = ''
+unix_socket_directories = '/tmp/openaedmap-postgres'
 
 # adjust memory usage
 shared_buffers = 512MB
 effective_cache_size = 1GB
 
 # increase max connections
-max_connections = 1000
+max_connections = 10000
 
 # detect disconnected clients
 # reason: safeguard resource usage

config/redis.conf

@@ -5,6 +5,18 @@
 # - 2GB RAM
 # - 6GB SSD
 
+# listen on socket
+# reason: reduce latency
+unixsocket /tmp/openaedmap-redis.sock
+unixsocketperm 700
+
+# single-database mode
+databases 1
+
+# use simple locale collate
+# reason: faster sorting without bias
+locale-collate C
+
 # disable persistence
 # reason: redis is cache only, use postgres for persistence
 save ""

db.py

@@ -12,7 +12,7 @@
     echo_pool=POSTGRES_LOG,
     json_deserializer=JSON_DECODE,
     json_serializer=lambda x: JSON_ENCODE(x).decode(),
-    pool_size=8,
+    pool_size=10,
     max_overflow=-1,
     query_cache_size=128,
 )

default.nix

@@ -23,6 +23,7 @@ let
 
   entrypoint = pkgs.writeShellScriptBin "entrypoint" ''
     set -ex
+    rm -f data/postgres/postmaster.pid
     dev-start
     set -o allexport
     source "envs/app/${envTag}.env" set
@@ -42,6 +43,7 @@ with pkgs; dockerTools.buildLayeredImage {
 
   extraCommands = ''
     set -e
+    mkdir tmp
     mkdir app && cd app
     mkdir -p data/postgres data/photos
     cp "${./.}"/*.py .
@@ -57,6 +59,7 @@ with pkgs; dockerTools.buildLayeredImage {
   fakeRootCommands = ''
     set -e
     ${dockerTools.shadowSetup}
+    chmod 0777 tmp
     groupadd --system -g 999 docker
     useradd --system --no-create-home -u 999 -g 999 docker
     chown -R docker:docker app

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   app:
     image: backend:${TAG}

services/aed_service.py

@@ -204,6 +204,11 @@ async def _update_db_snapshot() -> None:
         logging.info('Updating country codes')
         await _assign_country_codes(aeds)
 
+        logging.info('Updating statistics')
+        async with db_write() as session:
+            await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'})
+            await session.execute(text(f'ANALYZE "{AED.__tablename__}"'))
+
     logging.info('AED update finished (=%d)', len(aeds))
 
 

services/country_service.py

@@ -12,6 +12,7 @@
 from country_code_assigner import CountryCodeAssigner
 from db import db_read, db_write
 from models.bbox import BBox
+from models.db.aed import AED
 from models.db.country import Country
 from osm_countries import get_osm_countries
 from services.state_service import StateService
@@ -108,6 +109,12 @@ async def _update_db() -> None:
     from services.aed_service import AEDService
 
     await AEDService.update_country_codes()
+
+    logging.info('Updating statistics')
+    async with db_write() as session:
+        await session.connection(execution_options={'isolation_level': 'AUTOCOMMIT'})
+        await session.execute(text(f'ANALYZE "{AED.__tablename__}", "{Country.__tablename__}"'))
+
     logging.info('Country update finished')
 
 

shell.nix

@@ -65,12 +65,12 @@ let
           --pwfile=<(echo postgres)
       fi
 
-      mkdir -p data/supervisor
+      mkdir -p /tmp/openaedmap-postgres data/supervisor
       supervisord -c config/supervisord.conf
       echo "Supervisor started"
 
       echo "Waiting for Postgres to start..."
-      while ! pg_isready -q -h 127.0.0.1 -t 10; do sleep 0.1; done
+      while ! pg_isready -q -h /tmp/openaedmap-postgres -t 10; do sleep 0.1; done
       echo "Postgres started, running migrations"
       alembic-upgrade
     '')