Update packages, Small refactoring

.github/workflows/deploy.yaml

@@ -22,13 +22,13 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Nix
-        uses: cachix/install-nix-action@v26
+        uses: cachix/install-nix-action@v27
         with:
           nix_path: nixpkgs=channel:nixpkgs-23.11-darwin
 
       - name: Extract nixpkgs hash
         run: |
-          nixpkgs_hash=$(grep -o -P '(?<=archive/)[0-9a-f]{40}(?=\.tar\.gz)' shell.nix)
+          nixpkgs_hash=$(egrep -o 'archive/[0-9a-f]{40}\.tar\.gz' shell.nix | cut -d'/' -f2 | cut -d'.' -f1)
           echo "NIXPKGS_HASH=$nixpkgs_hash" >> $GITHUB_ENV
 
       - name: Cache Nix store
@@ -43,11 +43,13 @@ jobs:
         run: |
           nix-store --import < /tmp/nix-cache
 
-      - name: Cache Python packages
+      - name: Cache Python venv
         uses: actions/cache@v4
         with:
           key: python-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
-          path: .venv
+          path: |
+            ~/.cache/pypoetry
+            .venv
 
       - name: Install dependencies
         run: |

config.py

@@ -7,25 +7,14 @@
 from pyproj import Transformer
 
 NAME = 'openaedmap-backend'
-VERSION = '2.10.0'
+VERSION = '2.10.1'
 CREATED_BY = f'{NAME} {VERSION}'
 WEBSITE = 'https://openaedmap.org'
 
 USER_AGENT = f'{NAME}/{VERSION} (+{WEBSITE})'
-ENVIRONMENT = os.getenv('ENVIRONMENT', None)
+ENVIRONMENT = os.getenv('ENVIRONMENT')
 LOG_LEVEL = 'DEBUG'
 
-if ENVIRONMENT:
-    sentry_sdk.init(
-        dsn='https://[email protected]/3',
-        release=VERSION,
-        environment=ENVIRONMENT,
-        enable_tracing=True,
-        traces_sample_rate=0.2,
-        trace_propagation_targets=None,
-        profiles_sample_rate=0.2,
-    )
-
 POSTGRES_LOG = os.getenv('POSTGRES_LOG', '0').strip().lower() in ('1', 'true', 'yes')
 POSTGRES_URL = 'postgresql+asyncpg://postgres:postgres@/postgres?host=/tmp/openaedmap-postgres'
 VALKEY_URL = os.getenv('VALKEY_URL', 'unix:///tmp/openaedmap-valkey.sock?protocol=3')
@@ -96,7 +85,7 @@
             'root': {'handlers': ['default'], 'level': LOG_LEVEL},
             **{
                 # reduce logging verbosity of some modules
-                module: {'handlers': ['default'], 'level': 'INFO'}
+                module: {'handlers': [], 'level': 'INFO'}
                 for module in (
                     'hpack',
                     'httpx',
@@ -105,6 +94,26 @@
                     'PIL',
                 )
             },
+            **{
+                # conditional database logging
+                module: {'handlers': [], 'level': 'INFO'}
+                for module in (
+                    'sqlalchemy.engine',
+                    'sqlalchemy.pool',
+                )
+                if POSTGRES_LOG
+            },
         },
     }
 )
+
+if SENTRY_DSN := os.getenv('SENTRY_DSN'):
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        release=VERSION,
+        environment=ENVIRONMENT,
+        enable_tracing=True,
+        traces_sample_rate=0.2,
+        trace_propagation_targets=None,
+        profiles_sample_rate=0.2,
+    )

config/postgres.conf

@@ -15,7 +15,7 @@ effective_cache_size = 1GB
 
 # increase statistics target
 # reason: more accurate query plans
-default_statistics_target = 150
+default_statistics_target = 200
 
 # increase max connections
 max_connections = 10000
@@ -52,9 +52,9 @@ checkpoint_warning = 10min
 
 # adjust configuration for SSDs
 # reason: improved performance on expected hardware
+random_page_cost = 1.1
 effective_io_concurrency = 200
 maintenance_io_concurrency = 200
-random_page_cost = 1.1
 
 # increase logging verbosity
 # reason: useful for development

db.py

@@ -3,18 +3,16 @@
 from redis.asyncio import ConnectionPool, Redis
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 
-from config import POSTGRES_LOG, POSTGRES_URL, VALKEY_URL
+from config import POSTGRES_URL, VALKEY_URL
 from utils import JSON_DECODE, JSON_ENCODE
 
 _db_engine = create_async_engine(
     POSTGRES_URL,
-    echo=POSTGRES_LOG,
-    echo_pool=POSTGRES_LOG,
     json_deserializer=JSON_DECODE,
     json_serializer=lambda x: JSON_ENCODE(x).decode(),
+    query_cache_size=128,
     pool_size=10,
     max_overflow=-1,
-    query_cache_size=128,
 )
 
 

envs/app/dev.env

@@ -1,2 +1,3 @@
 ENVIRONMENT=dev
+SENTRY_DSN=https://[email protected]/3
 OPENSTREETMAP_API_URL=https://master.apis.dev.openstreetmap.org/api/0.6/

envs/app/main.env

@@ -1 +1,2 @@
 ENVIRONMENT=main
+SENTRY_DSN=https://[email protected]/3

pyproject.toml

@@ -29,7 +29,7 @@ python-multipart = "<1"
 pytz = "*"
 redis = {extras = ["hiredis"], version = "^5.0.3"}
 scikit-learn = "^1.3.2"
-sentry-sdk = {extras = ["fastapi", "httpx", "sqlalchemy"], version = "^1.44.0"}
+sentry-sdk = {extras = ["fastapi", "httpx", "sqlalchemy", "pure_eval"], version = "^1.44.0"}
 shapely = "^2.0.2"
 sqlalchemy = {extras = ["postgresql-asyncpg"], version = "^2.0.29"}
 starlette-compress = "^1.0.1"

shell.nix

@@ -1,29 +1,26 @@
 { isDevelopment ? true }:
 
 let
-  # Currently using nixpkgs-unstable
-  # Update with `nixpkgs-update` command
-  pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/af8b9db5c00f1a8e4b83578acc578ff7d823b786.tar.gz") { };
+  # Update packages with `nixpkgs-update` command
+  pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/14de0380da76de3f4cd662a9ef2352eed0c95b7d.tar.gz") { };
 
   pythonLibs = with pkgs; [
     stdenv.cc.cc.lib
     file.out
     libxml2.out
     zlib.out
   ];
-
-  # Override LD_LIBRARY_PATH to load Python libraries
-  wrappedPython = with pkgs; symlinkJoin {
+  wrappedPython = with pkgs; (symlinkJoin {
     name = "python";
     paths = [
-      # Enable Python optimizations when in production
+      # Enable compiler optimizations when in production
       (if isDevelopment then python312 else python312.override { enableOptimizations = true; })
     ];
     buildInputs = [ makeWrapper ];
     postBuild = ''
       wrapProgram "$out/bin/python3.12" --prefix LD_LIBRARY_PATH : "${lib.makeLibraryPath pythonLibs}"
     '';
-  };
+  });
 
   packages' = with pkgs; [
     wrappedPython
@@ -57,8 +54,9 @@ let
       if [ ! -f data/postgres/PG_VERSION ]; then
         initdb -D data/postgres \
           --no-instructions \
-          --locale=C.UTF-8 \
-          --encoding=UTF8 \
+          --locale-provider=icu \
+          --icu-locale=und \
+          --no-locale \
           --text-search-config=pg_catalog.simple \
           --auth=password \
           --username=postgres \
@@ -70,7 +68,18 @@ let
       echo "Supervisor started"
 
       echo "Waiting for Postgres to start..."
-      while ! pg_isready -q -h /tmp/openaedmap-postgres -t 10; do sleep 0.1; done
+      time_start=$(date +%s)
+      while ! pg_isready -q -h /tmp/openaedmap-postgres; do
+        elapsed=$(($(date +%s) - $time_start))
+        if [ $elapsed -gt 10 ]; then
+          tail -n 15 data/supervisor/supervisord.log data/supervisor/postgres.log
+          echo "Postgres startup timeout, see above logs for details"
+          dev-stop
+          exit 1
+        fi
+        sleep 0.1
+      done
+
       echo "Postgres started, running migrations"
       alembic-upgrade
     '')
@@ -142,7 +151,7 @@ let
     make-version
   '';
 in
-pkgs.mkShell {
+pkgs.mkShellNoCC {
   buildInputs = packages';
   shellHook = shell';
 }