Make tests cleaner and more robust

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-    "setuptools>=67.7.2",
+    "setuptools>=69",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -16,6 +16,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "License :: OSI Approved :: Apache Software License",
     "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
     "Topic :: Software Development :: Libraries",
@@ -70,8 +71,9 @@ fixable = [
     "D",
 ]
 ignore = [
-    "E",
-    "W",
+    "E111",
+    "E114",
+    "E116",
     "PLW",
     "RUF001",
     "RUF010",
@@ -144,7 +146,7 @@ check_untyped_defs = true
 no_site_packages = false
 
 [tool.pytest.ini_options]
-minversion = "7.1"
+minversion = "8.0"
 asyncio_mode = "strict"
 
 [tool.coverage.paths]

tests/conftest.py

@@ -16,50 +16,33 @@
 
 """Shared fixtures"""
 
-import asyncio
-
 import pytest
-from hexkit.providers.akafka.testutils import KafkaFixture, get_kafka_fixture
+from hexkit.providers.akafka.testutils import get_kafka_fixture
 from hexkit.providers.mongodb.testutils import MongoDbFixture, get_mongodb_fixture
 
-from tests.fixtures.datasets import DATASET_UPSERTION_EVENT
 from wps.config import Config
-from wps.inject import Consumer
+
+from .fixtures.datasets import DATASET
 
 kafka_fixture = get_kafka_fixture("session")
 mongodb_fixture = get_mongodb_fixture("session")
 
 
-@pytest.fixture(autouse=True, scope="function")
-def reset_db(mongodb_fixture: MongoDbFixture):
-    """Clear the database before tests."""
+@pytest.fixture(name="empty_mongodb")
+def empty_mongodb_fixture(mongodb_fixture: MongoDbFixture) -> MongoDbFixture:
+    """MongoDB Fixture with empty database."""
     mongodb_fixture.empty_collections()
-
-
-async def publish_and_process_dataset_upsertion_event(
-    config: Config, kafka_fixture: KafkaFixture, consumer: Consumer
-):
-    """Publish and process a dataset upsertion event"""
-    await kafka_fixture.publish_event(
-        payload=DATASET_UPSERTION_EVENT.model_dump(),
-        topic=config.dataset_change_event_topic,
-        type_=config.dataset_upsertion_event_type,
-        key="test-key-fixture",
-    )
-    await asyncio.wait_for(consumer.event_subscriber.run(forever=False), timeout=10)
-    await asyncio.sleep(0.125)
-
-
-@pytest.fixture(scope="function")
-def populate_db(
-    reset_db, config: Config, kafka_fixture: KafkaFixture, consumer: Consumer
-):
-    """Populate the database
-
-    Required for tests using the consumer or client fixtures.
-    """
-    asyncio.get_event_loop().run_until_complete(
-        publish_and_process_dataset_upsertion_event(
-            config=config, kafka_fixture=kafka_fixture, consumer=consumer
-        )
-    )
+    return mongodb_fixture
+
+
+@pytest.fixture(name="populated_mongodb")
+def populated_mongodb_fixture(
+    empty_mongodb: MongoDbFixture, config: Config
+) -> MongoDbFixture:
+    """MongoDB Fixture with a database populated with one dataset."""
+    database = empty_mongodb.client.get_database(config.db_name)
+    collection = database.get_collection(config.datasets_collection)
+    dataset = DATASET.model_dump()
+    dataset["_id"] = dataset.pop("id")
+    collection.insert_one(dataset)
+    return empty_mongodb

tests/fixtures/__init__.py

@@ -18,6 +18,8 @@
 from collections.abc import AsyncGenerator
 from datetime import timedelta
 
+import pytest
+import pytest_asyncio
 from ghga_service_commons.api.testing import AsyncTestClient
 from ghga_service_commons.auth.ghga import AuthContext
 from ghga_service_commons.utils.jwt_helpers import (
@@ -27,13 +29,11 @@
 from ghga_service_commons.utils.utc_dates import now_as_utc
 from hexkit.providers.akafka.testutils import KafkaFixture
 from hexkit.providers.mongodb.testutils import MongoDbFixture
-from pytest import fixture
-from pytest_asyncio import fixture as async_fixture
 
 from wps.adapters.outbound.dao import DatasetDaoConstructor, WorkPackageDaoConstructor
 from wps.config import Config
 from wps.core.repository import WorkPackageRepository
-from wps.inject import Consumer, prepare_consumer, prepare_rest_app
+from wps.inject import Consumer, prepare_rest_app
 
 from .access import AccessCheckMock
 
@@ -70,14 +70,14 @@ def headers_for_token(token: str) -> dict[str, str]:
     return {"Authorization": f"Bearer {token}"}
 
 
-@fixture(name="auth_headers")
+@pytest.fixture(name="auth_headers")
 def fixture_auth_headers() -> dict[str, str]:
     """Get auth headers for testing"""
     token = sign_and_serialize_token(AUTH_CLAIMS, AUTH_KEY_PAIR)
     return headers_for_token(token)
 
 
-@fixture(name="bad_auth_headers")
+@pytest.fixture(name="bad_auth_headers")
 def fixture_bad_auth_headers() -> dict[str, str]:
     """Get a invalid auth headers for testing"""
     claims = AUTH_CLAIMS.copy()
@@ -86,17 +86,17 @@ def fixture_bad_auth_headers() -> dict[str, str]:
     return headers_for_token(token)
 
 
-@fixture(name="auth_context")
+@pytest.fixture(name="auth_context")
 def fixture_auth_context() -> AuthContext:
     """Fixture for getting an auth context"""
     iat = now_as_utc() - timedelta(1)  # validity is actually assumed by the repository
-    return AuthContext(**AUTH_CLAIMS, iat=iat, exp=iat)  # pyright: ignore
+    return AuthContext(**AUTH_CLAIMS, iat=iat, exp=iat)  # type: ignore
 
 
-@async_fixture(name="config", scope="session")
-async def fixture_config(
+@pytest.fixture(name="config", scope="session")
+def fixture_config(
     kafka_fixture: KafkaFixture, mongodb_fixture: MongoDbFixture
-) -> AsyncGenerator[Config, None]:
+) -> Config:
     """Fixture for creating a test configuration."""
     return Config(
         auth_key=AUTH_KEY_PAIR.export_public(),  # pyright: ignore
@@ -107,7 +107,7 @@ async def fixture_config(
     )
 
 
-@async_fixture(name="repository", scope="session")
+@pytest_asyncio.fixture(name="repository", scope="session")
 async def fixture_repository(
     config: Config, mongodb_fixture: MongoDbFixture
 ) -> WorkPackageRepository:
@@ -129,24 +129,15 @@ async def fixture_repository(
     )
 
 
-@async_fixture(name="consumer", scope="session")
-async def fixture_consumer(config: Config) -> AsyncGenerator[Consumer, None]:
-    """Get test event subscriber for the work package service."""
-    async with prepare_consumer(config=config) as consumer:
-        # wait for event to be submitted and processed,
-        # so that the database is populated with the published datasets
-        yield consumer
-
-
-@async_fixture(name="client", scope="session")
+@pytest_asyncio.fixture(name="client", scope="session")
 async def fixture_client(config: Config) -> AsyncGenerator[AsyncTestClient, None]:
     """Get test client for the work package service."""
     async with prepare_rest_app(config=config) as app:
         async with AsyncTestClient(app=app) as client:
             yield client
 
 
-@fixture
+@pytest.fixture
 def non_mocked_hosts() -> list[str]:
     """Get hosts that are not mocked by pytest-httpx."""
     return ["test", "localhost"]

tests/test_access.py

@@ -18,24 +18,25 @@
 
 from collections.abc import AsyncGenerator
 
-from pytest import mark
-from pytest_asyncio import fixture as async_fixture
+import pytest
+import pytest_asyncio
 from pytest_httpx import HTTPXMock
 
 from wps.adapters.outbound.http import AccessCheckAdapter, AccessCheckConfig
 
+pytestmark = pytest.mark.asyncio()
+
 DOWNLOAD_ACCESS_URL = "http://test-access:1234"
 
 
-@async_fixture(name="access_check", scope="function")
+@pytest_asyncio.fixture(name="access_check", scope="function")
 async def fixture_access_check() -> AsyncGenerator[AccessCheckAdapter, None]:
     """Get configured access test adapter."""
     config = AccessCheckConfig(download_access_url=DOWNLOAD_ACCESS_URL)
     async with AccessCheckAdapter.construct(config=config) as adapter:
         yield adapter
 
 
-@mark.asyncio
 async def test_check_download_access(
     access_check: AccessCheckAdapter, httpx_mock: HTTPXMock
 ):
@@ -61,7 +62,6 @@ async def test_check_download_access(
     assert await check_access("some-user-id", "no-data-id") is False
 
 
-@mark.asyncio
 async def test_get_download_datasets(
     access_check: AccessCheckAdapter, httpx_mock: HTTPXMock
 ):

tests/test_api.py

@@ -16,10 +16,10 @@
 
 """Test the API of the work package service."""
 
+import pytest
 from fastapi import status
 from ghga_service_commons.api.testing import AsyncTestClient
 from ghga_service_commons.utils.jwt_helpers import decode_and_validate_token
-from pytest import mark
 from pytest_httpx import HTTPXMock
 
 from .fixtures import (  # noqa: F401
@@ -28,14 +28,16 @@
     fixture_bad_auth_headers,
     fixture_client,
     fixture_config,
-    fixture_consumer,
     fixture_repository,
     headers_for_token,
     non_mocked_hosts,
 )
 from .fixtures.crypt import decrypt, user_public_crypt4gh_key
 from .fixtures.datasets import DATASET
 
+pytestmark = pytest.mark.asyncio(scope="session")
+
+
 CREATION_DATA = {
     "dataset_id": "some-dataset-id",
     "type": "download",
@@ -46,7 +48,6 @@
 TIMEOUT = 5
 
 
-@mark.asyncio(scope="session")
 async def test_health_check(client: AsyncTestClient):
     """Test that the health check endpoint works."""
     response = await client.get("/health", timeout=TIMEOUT)
@@ -55,7 +56,6 @@ async def test_health_check(client: AsyncTestClient):
     assert response.json() == {"status": "OK"}
 
 
-@mark.asyncio(scope="session")
 async def test_create_work_package_unauthorized(
     client: AsyncTestClient, bad_auth_headers: dict[str, str]
 ):
@@ -68,19 +68,17 @@ async def test_create_work_package_unauthorized(
     assert response.status_code == status.HTTP_401_UNAUTHORIZED
 
 
-@mark.asyncio(scope="session")
 async def test_get_work_package_unauthorized(client: AsyncTestClient):
     """Test that getting a work package needs authorization."""
     response = await client.get("/work-packages/some-work-package-id")
     assert response.status_code == status.HTTP_403_FORBIDDEN
 
 
-@mark.asyncio(scope="session")
 async def test_create_work_order_token(
     client: AsyncTestClient,
     auth_headers: dict[str, str],
     httpx_mock: HTTPXMock,
-    populate_db,
+    populated_mongodb,
 ):
     """Test that work order tokens can be properly created."""
     # mock the access check for the test dataset
@@ -219,14 +217,12 @@ async def test_create_work_order_token(
     }
 
 
-@mark.asyncio(scope="session")
 async def test_get_datasets_unauthenticated(client: AsyncTestClient):
     """Test that the list of accessible datasets cannot be fetched unauthenticated."""
     response = await client.get("/users/[email protected]/datasets")
     assert response.status_code == status.HTTP_403_FORBIDDEN
 
 
-@mark.asyncio(scope="session")
 async def test_get_datasets_for_another_user(
     client: AsyncTestClient, auth_headers: dict[str, str]
 ):
@@ -237,12 +233,11 @@ async def test_get_datasets_for_another_user(
     assert response.status_code == status.HTTP_403_FORBIDDEN
 
 
-@mark.asyncio(scope="session")
 async def test_get_datasets_when_none_authorized(
     client: AsyncTestClient,
     auth_headers: dict[str, str],
     httpx_mock: HTTPXMock,
-    populate_db,
+    populated_mongodb,
 ):
     """Test that no datasets are fetched when none are accessible."""
     # mock the access check for the test dataset
@@ -265,12 +260,11 @@ async def test_get_datasets_when_none_authorized(
     assert response_data == []
 
 
-@mark.asyncio(scope="session")
 async def test_get_datasets(
     client: AsyncTestClient,
     auth_headers: dict[str, str],
     httpx_mock: HTTPXMock,
-    populate_db,
+    populated_mongodb,
 ):
     """Test that the list of accessible datasets can be fetched."""
     # mock the access check for the test dataset