Switch to uv, Fix typing issues, Use AIOHttp for improved performance

.github/workflows/deploy.yaml

@@ -24,18 +24,18 @@ jobs:
       - name: Install Nix
         uses: cachix/install-nix-action@v27
         with:
-          nix_path: nixpkgs=channel:nixpkgs-23.11-darwin
+          nix_path: nixpkgs=channel:nixpkgs-24.05-darwin
 
-      - name: Extract nixpkgs hash
+      - name: Generate cache key
         run: |
           nixpkgs_hash=$(egrep -o 'archive/[0-9a-f]{40}\.tar\.gz' shell.nix | cut -d'/' -f2 | cut -d'.' -f1)
-          echo "NIXPKGS_HASH=$nixpkgs_hash" >> $GITHUB_ENV
+          echo "CACHE_KEY=${{ runner.os }}-$nixpkgs_hash" >> $GITHUB_ENV
 
       - name: Cache Nix store
         uses: actions/cache@v4
         id: nix-cache
         with:
-          key: nix-${{ runner.os }}-${{ env.NIXPKGS_HASH }}
+          key: nix-${{ env.CACHE_KEY }}
           path: /tmp/nix-cache
 
       - name: Import Nix store cache
@@ -46,9 +46,9 @@ jobs:
       - name: Cache Python venv
         uses: actions/cache@v4
         with:
-          key: python-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+          key: python-${{ env.CACHE_KEY }}-${{ hashFiles('uv.lock') }}
           path: |
-            ~/.cache/pypoetry
+            ~/.cache/uv
             .venv
 
       - name: Install dependencies

.gitignore

@@ -199,6 +199,9 @@ pyrightconfig.json
 
 # End of https://www.toptal.com/developers/gitignore/api/dotenv,python,visualstudiocode,direnv
 
+# Python version lock
+.python-version
+
 data/*
 cert/*
 

api/v1/photos.py

@@ -1,13 +1,13 @@
 from datetime import timedelta
-from io import BytesIO
 from typing import Annotated
 from urllib.parse import unquote_plus
 
 import magic
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, Tag
 from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile
 from fastapi.responses import FileResponse
 from feedgen.feed import FeedGenerator
+from pydantic import SecretStr
 
 from config import IMAGE_CONTENT_TYPES, IMAGE_REMOTE_MAX_FILE_SIZE
 from middlewares.cache_control_middleware import cache_control
@@ -16,29 +16,22 @@
 from services.aed_service import AEDService
 from services.photo_report_service import PhotoReportService
 from services.photo_service import PhotoService
-from utils import JSON_DECODE, get_http_client, get_wikimedia_commons_url
+from utils import JSON_DECODE, get_wikimedia_commons_url, http_get
 
 router = APIRouter(prefix='/photos')
 
 
 async def _fetch_image(url: str) -> tuple[bytes, str]:
     # NOTE: ideally we would verify whether url is not a private resource
-    async with get_http_client() as http:
-        r = await http.get(url)
-        r.raise_for_status()
+    async with http_get(url, allow_redirects=True, raise_for_status=True) as r:
+        # Early detection of unsupported types
+        content_type = r.headers.get('Content-Type')
+        if content_type and content_type not in IMAGE_CONTENT_TYPES:
+            raise HTTPException(500, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}')
 
-    # Early detection of unsupported types
-    content_type = r.headers.get('Content-Type')
-    if content_type and content_type not in IMAGE_CONTENT_TYPES:
-        raise HTTPException(500, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}')
-
-    with BytesIO() as buffer:
-        async for chunk in r.aiter_bytes(chunk_size=1024 * 1024):
-            buffer.write(chunk)
-            if buffer.tell() > IMAGE_REMOTE_MAX_FILE_SIZE:
-                raise HTTPException(500, f'File is too large, max allowed size is {IMAGE_REMOTE_MAX_FILE_SIZE} bytes')
-
-        file = buffer.getvalue()
+        file = await r.content.read(IMAGE_REMOTE_MAX_FILE_SIZE + 1)
+        if len(file) > IMAGE_REMOTE_MAX_FILE_SIZE:
+            raise HTTPException(500, f'File is too large, max allowed size is {IMAGE_REMOTE_MAX_FILE_SIZE} bytes')
 
     # Check if file type is supported
     content_type = magic.from_buffer(file[:2048], mime=True)
@@ -70,17 +63,18 @@ async def proxy_direct(url_encoded: str):
 @cache_control(timedelta(days=7), stale=timedelta(days=7))
 async def proxy_wikimedia_commons(path_encoded: str):
     meta_url = get_wikimedia_commons_url(unquote_plus(path_encoded))
+    async with http_get(meta_url, allow_redirects=True, raise_for_status=True) as r:
+        html = await r.text()
 
-    async with get_http_client() as http:
-        r = await http.get(meta_url)
-        r.raise_for_status()
-
-    bs = BeautifulSoup(r.text, 'lxml')
+    bs = BeautifulSoup(html, 'lxml')
     og_image = bs.find('meta', property='og:image')
-    if not og_image:
+    if not isinstance(og_image, Tag):
         return Response('Missing og:image meta tag', 404)
 
     image_url = og_image['content']
+    if not isinstance(image_url, str):
+        return Response('Invalid og:image meta tag (expected str)', 404)
+
     file, content_type = await _fetch_image(image_url)
     return Response(file, media_type=content_type)
 
@@ -92,13 +86,13 @@ async def upload(
     file_license: Annotated[str, Form()],
     file: Annotated[UploadFile, File()],
     oauth2_credentials: Annotated[str, Form()],
-) -> bool:
+):
     file_license = file_license.upper()
     accept_licenses = ('CC0',)
 
     if file_license not in accept_licenses:
         return Response(f'Unsupported license {file_license!r}, must be one of {accept_licenses}', 400)
-    if file.size <= 0:
+    if file.size is None or file.size <= 0:
         return Response('File must not be empty', 400)
 
     content_type = magic.from_buffer(file.file.read(2048), mime=True)
@@ -107,6 +101,7 @@ async def upload(
 
     try:
         oauth2_credentials_: dict = JSON_DECODE(oauth2_credentials)
+        oauth2_token = SecretStr(oauth2_credentials_['access_token'])
     except Exception:
         return Response('OAuth2 credentials must be a JSON object', 400)
     if 'access_token' not in oauth2_credentials_:
@@ -116,7 +111,7 @@ async def upload(
     if aed is None:
         return Response(f'Node {node_id} not found, perhaps it is not an AED?', 404)
 
-    osm = OpenStreetMap(oauth2_credentials_)
+    osm = OpenStreetMap(oauth2_token)
     osm_user = await osm.get_authorized_user()
     if osm_user is None:
         return Response('OAuth2 credentials are invalid', 401)
@@ -127,6 +122,8 @@ async def upload(
     photo = await PhotoService.upload(node_id, user_id, file)
     photo_url = f'{request.base_url}api/v1/photos/view/{photo.id}.webp'
     node_xml = await osm.get_node_xml(node_id)
+    if node_xml is None:
+        return Response(f'Node {node_id} not found on remote', 404)
 
     osm_change = update_node_tags_osm_change(
         node_xml,

api/v1/tile.py

@@ -7,7 +7,7 @@
 from anyio import create_task_group
 from fastapi import APIRouter, Path, Response
 from sentry_sdk import start_span, trace
-from shapely import get_coordinates, points, set_coordinates
+from shapely import get_coordinates, points, set_coordinates, simplify
 
 from config import (
     DEFAULT_CACHE_MAX_AGE,
@@ -109,7 +109,7 @@ def _mvt_encode(bbox: BBox, layers: Sequence[dict]) -> bytes:
 @trace
 async def _get_tile_country(z: int, bbox: BBox) -> bytes:
     countries = await CountryService.get_intersecting(bbox)
-    country_count_map: dict[str, str] = {}
+    country_count_map: dict[str, tuple[int, str]] = {}
 
     with start_span(description='Counting AEDs'):
 
@@ -121,8 +121,8 @@ async def count_task(country: Country) -> None:
             for country in countries:
                 tg.start_soon(count_task, country)
 
-    simplify_tol = 0.5 / 2**z if z < TILE_MAX_Z else None
-    geometries = (country.geometry.simplify(simplify_tol, preserve_topology=False) for country in countries)
+    simplify_tol = 0.5 / 2**z
+    geometries = (simplify(country.geometry, simplify_tol, preserve_topology=False) for country in countries)
 
     return _mvt_encode(
         bbox,

config.py

@@ -7,7 +7,7 @@
 from pyproj import Transformer
 
 NAME = 'openaedmap-backend'
-VERSION = '2.10.2'
+VERSION = '2.11.0'
 CREATED_BY = f'{NAME} {VERSION}'
 WEBSITE = 'https://openaedmap.org'
 

middlewares/cache_response_middleware.py

@@ -36,15 +36,17 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
 
         url = URL(scope=scope)
         cached = await _get_cached_response(url)
+        maybe_send: Send | None = send
+
         if cached is not None:
             if await _deliver_cached_response(cached, send):
                 # served fresh response
                 return
             else:
                 # served stale response, refresh cache
-                send = None
+                maybe_send = None
 
-        await CachingResponder(self.app, url)(scope, receive, send)
+        await CachingResponder(self.app, url)(scope, receive, maybe_send)
 
 
 async def _deliver_cached_response(cached: CachedResponse, send: Send) -> bool:

models/aed_group.py

@@ -6,7 +6,7 @@
 
 class AEDGroup(NamedTuple):
     position: Point
-    count: int
+    count: int  # pyright: ignore[reportIncompatibleMethodOverride]
     access: str
 
     @staticmethod

models/bbox.py

@@ -1,4 +1,5 @@
-from typing import NamedTuple, Self
+from collections.abc import Collection
+from typing import NamedTuple, cast
 
 import numpy as np
 from shapely import Point, get_coordinates, points
@@ -9,7 +10,7 @@ class BBox(NamedTuple):
     p1: Point
     p2: Point
 
-    def extend(self, percentage: float) -> Self:
+    def extend(self, percentage: float) -> 'BBox':
         p1_coords = get_coordinates(self.p1)[0]
         p2_coords = get_coordinates(self.p2)[0]
 
@@ -19,11 +20,11 @@ def extend(self, percentage: float) -> Self:
         p1_coords = np.clip(p1_coords - deltas, [-180, -90], [180, 90])
         p2_coords = np.clip(p2_coords + deltas, [-180, -90], [180, 90])
 
-        p1, p2 = points((p1_coords, p2_coords))
+        p1, p2 = cast(Collection[Point], points((p1_coords, p2_coords)))
         return BBox(p1, p2)
 
     @classmethod
-    def from_tuple(cls, bbox: tuple[float, float, float, float]) -> Self:
+    def from_tuple(cls, bbox: tuple[float, float, float, float]) -> 'BBox':
         p1, p2 = points(((bbox[0], bbox[1]), (bbox[2], bbox[3])))
         return cls(p1, p2)
 
@@ -58,7 +59,7 @@ def to_polygon(self, *, nodes_per_edge: int = 2) -> Polygon:
         all_coords = np.concatenate((bottom_edge, right_edge, top_edge[::-1], left_edge[::-1]))
         return Polygon(all_coords)
 
-    def correct_for_dateline(self) -> tuple[Self, ...]:
+    def correct_for_dateline(self) -> tuple['BBox', ...]:
         if self.p1.x > self.p2.x:
             b1_p1 = self.p1
             b2_p2 = self.p2

models/db/__init__.py

@@ -2,4 +2,4 @@
 
 # import all files in this directory
 modules = pathlib.Path(__file__).parent.glob('*.py')
-__all__ = tuple(f.stem for f in modules if f.is_file() and not f.name.startswith('_'))
+__all__ = tuple(f.stem for f in modules if f.is_file() and not f.name.startswith('_'))  # pyright: ignore[reportUnsupportedDunderAll]

models/db/country.py

@@ -1,5 +1,4 @@
-from asyncpg import Polygon
-from shapely import MultiPolygon, Point
+from shapely import MultiPolygon, Point, Polygon
 from sqlalchemy import Index, Unicode
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column

models/geometry.py

@@ -1,67 +1,61 @@
-from shapely import MultiPolygon, Point, Polygon, from_wkb, get_coordinates
+from abc import ABC
+from typing import override
+
+from shapely import from_wkb, get_coordinates
+from shapely.geometry.base import BaseGeometry
 from sqlalchemy import BindParameter
 from sqlalchemy.sql import func
 from sqlalchemy.types import UserDefinedType
 
 
-class PointType(UserDefinedType):
-    cache_ok = True
+class _GeometryType(UserDefinedType, ABC):
+    geometry_type: str
 
     def get_col_spec(self, **kw):
-        return 'geometry(Point, 4326)'
-
-    def bind_expression(self, bindvalue: BindParameter):
-        return func.ST_GeomFromText(bindvalue, 4326, type_=self)
+        return f'geometry({self.geometry_type}, 4326)'
 
+    @override
     def bind_processor(self, dialect):
-        def process(value: Point | None):
+        def process(value: BaseGeometry | None):
             if value is None:
                 return None
-
-            x, y = get_coordinates(value)[0]
-            return f'POINT({x} {y})'  # WKT
+            return value.wkt
 
         return process
 
-    def column_expression(self, col):
-        return func.ST_AsBinary(col, type_=self)
+    @override
+    def bind_expression(self, bindvalue: BindParameter):
+        return func.ST_GeomFromText(bindvalue, 4326, type_=self)
+
+    @override
+    def column_expression(self, colexpr):
+        return func.ST_AsBinary(colexpr, type_=self)
 
+    @override
     def result_processor(self, dialect, coltype):
         def process(value: bytes | None):
             if value is None:
                 return None
-
             return from_wkb(value)
 
         return process
 
 
-class PolygonType(UserDefinedType):
+class PointType(_GeometryType):
+    geometry_type = 'Point'
     cache_ok = True
 
-    def get_col_spec(self, **kw):
-        return 'geometry(Geometry, 4326)'
-
-    def bind_expression(self, bindvalue: BindParameter):
-        return func.ST_GeomFromText(bindvalue, 4326, type_=self)
-
+    @override
     def bind_processor(self, dialect):
-        def process(value: Polygon | MultiPolygon | None):
+        def process(value: BaseGeometry | None):
             if value is None:
                 return None
-
-            return value.wkt
+            x, y = get_coordinates(value)[0]
+            return f'POINT({x} {y})'  # WKT
 
         return process
 
-    def column_expression(self, col):
-        return func.ST_AsBinary(col, type_=self)
 
-    def result_processor(self, dialect, coltype):
-        def process(value: bytes | None):
-            if value is None:
-                return None
-
-            return from_wkb(value)
-
-        return process
+class PolygonType(_GeometryType):
+    geometry_type = 'Polygon'
+    cache_ok = True

models/osm_country.py

@@ -1,11 +1,11 @@
 from typing import NamedTuple
 
+from shapely import MultiPolygon, Polygon
 from shapely.geometry import Point
-from shapely.geometry.base import BaseGeometry
 
 
 class OSMCountry(NamedTuple):
     tags: dict[str, str]
-    geometry: BaseGeometry
+    geometry: Polygon | MultiPolygon
     representative_point: Point
     timestamp: float