From a4c1bb63852a632f02d8a4f32ec1a7802b5baa88 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> Date: Wed, 8 May 2024 13:48:18 +0200 Subject: [PATCH] Allow `identity`, ensure `dimensions` is returned, & use pre-commit.ci (#103) Update tests with valid entity using 'identity'. Ensure "identity" is respected everywhere. Ensure "dimensions" is always returned. Pre-commit is now run through pre-commit.ci. --- .github/workflows/ci_tests.yml | 2 +- .pre-commit-config.yaml | 17 ++- entities_service/cli/_utils/generics.py | 2 +- entities_service/main.py | 18 +-- entities_service/models/soft.py | 11 +- entities_service/service/backend/mongodb.py | 2 +- entities_service/service/routers/admin.py | 12 +- entities_service/service/utils.py | 50 ++++++++ tests/cli/commands/test_upload.py | 44 ++++--- tests/cli/commands/test_validate.py | 45 +++++--- tests/conftest.py | 72 ++++++++---- tests/service/backend/test_mongodb.py | 1 + tests/service/routers/test_admin.py | 120 +++++++++++++++----- tests/static/valid_entities.yaml | 14 +++ tests/test_main.py | 58 ++++++++-- 15 files changed, 344 insertions(+), 124 deletions(-) create mode 100644 entities_service/service/utils.py diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 9cc0e852..9390830a 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -16,7 +16,7 @@ jobs: install_extras: "[dev]" # pre-commit - python_version_pre-commit: "3.10" + run_pre-commit: false # pylint & safety python_version_pylint_safety: "3.10" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 78dc0471..c18925fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,16 @@ -# To install the git pre-commit hook run: -# pre-commit install -# To update the pre-commit hooks run: -# pre-commit autoupdate +# pre-commit.ci +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit hooks + + For more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_branch: 'main' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: 'weekly' + submodules: false + +# hooks repos: # pre-commit-hooks supplies a multitude of small hooks # To get an overview of them all as well as the ones used here, please see diff --git a/entities_service/cli/_utils/generics.py b/entities_service/cli/_utils/generics.py index ef58d90a..ee4720c5 100644 --- a/entities_service/cli/_utils/generics.py +++ b/entities_service/cli/_utils/generics.py @@ -119,7 +119,7 @@ def get_namespace_name_version(entity: Entity | dict[str, Any]) -> tuple[str, st The version is reversed to sort it in descending order (utilizing StrReversor). """ if isinstance(entity, dict): - uri = entity.get("uri", None) or ( + uri = entity.get("uri", entity.get("identity", None)) or ( f"{entity.get('namespace', '')}/{entity.get('version', '')}" f"/{entity.get('name', '')}" ) diff --git a/entities_service/main.py b/entities_service/main.py index 3658a068..4f22fd6d 100644 --- a/entities_service/main.py +++ b/entities_service/main.py @@ -19,6 +19,7 @@ from entities_service.service.config import CONFIG from entities_service.service.logger import setup_logger from entities_service.service.routers import get_routers +from entities_service.service.utils import _get_entity if TYPE_CHECKING: # pragma: no cover from typing import Any @@ -59,23 +60,6 @@ async def lifespan(_: FastAPI): APP.include_router(router) -async def _get_entity(version: str, name: str, db: str | None = None) -> dict[str, Any]: - """Utility function for the endpoints to retrieve an entity.""" - uri = f"{str(CONFIG.base_url).rstrip('/')}" - - if db: - uri += f"/{db}" - - uri += f"/{version}/{name}" - - entity = get_backend(db=db).read(uri) - - if entity is None: - raise ValueError(f"Could not find entity: uri={uri}") - - return entity - - @APP.get( "/{version}/{name}", response_model=Entity, diff --git a/entities_service/models/soft.py b/entities_service/models/soft.py index d9246590..13f91dc1 100644 --- a/entities_service/models/soft.py +++ b/entities_service/models/soft.py @@ -174,6 +174,7 @@ class SOFTEntity(BaseModel): "The universal identifier for the entity. This MUST start with the base" " URL." ), + validation_alias=AliasChoices("identity", "uri"), ), ] = None description: Annotated[str, Field(description="Description of the entity.")] = "" @@ -244,7 +245,7 @@ def _check_cross_dependent_fields(cls, data: Any) -> Any: if ( isinstance(data, dict) and any(data.get(_) is None for _ in ("name", "version", "namespace")) - and data.get("uri") is None + and data.get("uri", data.get("identity")) is None ): error_message = ( "Either `name`, `version`, and `namespace` or `uri` must be set.\n" @@ -254,13 +255,14 @@ def _check_cross_dependent_fields(cls, data: Any) -> Any: if ( isinstance(data, dict) and all(data.get(_) is not None for _ in ("name", "version", "namespace")) - and data.get("uri") is not None - and data["uri"] != f"{data['namespace']}/{data['version']}/{data['name']}" + and data.get("uri", data.get("identity")) is not None + and data.get("uri", data.get("identity", "")) + != f"{data['namespace']}/{data['version']}/{data['name']}" ): # Ensure that `uri` is consistent with `name`, `version`, and `namespace`. diff = "\n ".join( difflib.ndiff( - [data["uri"]], + [data.get("uri", data.get("identity", ""))], [f"{data['namespace']}/{data['version']}/{data['name']}"], ) ) @@ -269,4 +271,5 @@ def _check_cross_dependent_fields(cls, data: Any) -> Any: f"`namespace`:\n\n {diff}\n\n" ) raise ValueError(error_message) + return data diff --git a/entities_service/service/backend/mongodb.py b/entities_service/service/backend/mongodb.py index ea5bdb0c..0d8d3bd9 100644 --- a/entities_service/service/backend/mongodb.py +++ b/entities_service/service/backend/mongodb.py @@ -419,7 +419,7 @@ def _single_uri_query(self, uri: str) -> dict[str, Any]: def _prepare_entity(self, entity: Entity | dict[str, Any]) -> dict[str, Any]: """Clean and prepare the entity for interactions with the MongoDB backend.""" if isinstance(entity, dict): - uri = entity.get("uri", None) or ( + uri = entity.get("uri", entity.get("identity", None)) or ( f"{entity.get('namespace', '')}/{entity.get('version', '')}" f"/{entity.get('name', '')}" ) diff --git a/entities_service/service/routers/admin.py b/entities_service/service/routers/admin.py index 232f4c3a..a75edbc7 100644 --- a/entities_service/service/routers/admin.py +++ b/entities_service/service/routers/admin.py @@ -19,6 +19,7 @@ from entities_service.service.backend import get_backend from entities_service.service.config import CONFIG from entities_service.service.security import verify_token +from entities_service.service.utils import _add_dimensions if TYPE_CHECKING: # pragma: no cover from typing import Any @@ -83,6 +84,12 @@ async def create_entities( CONFIG.backend, auth_level="write", db=namespace ) + LOGGER.debug( + "Creating %s entities in namespace '%s'", + len(namespaced_entities), + namespace, + ) + try: created_namespaced_entities = namespaced_entities_backend.create( namespaced_entities @@ -97,7 +104,7 @@ async def create_entities( "Already created entities: uris=%s", ", ".join( ( - entity.get("uri", "") + entity.get("uri", entity.get("identity", "")) or ( f"{entity.get('namespace', '')}" f"/{entity.get('version', '')}" @@ -123,6 +130,9 @@ async def create_entities( ): raise write_fail_exception + # Ensure the returned entities have the dimensions key + await _add_dimensions(created_namespaced_entities) + if isinstance(created_namespaced_entities, dict): created_entities.append(created_namespaced_entities) else: diff --git a/entities_service/service/utils.py b/entities_service/service/utils.py new file mode 100644 index 00000000..e679b09e --- /dev/null +++ b/entities_service/service/utils.py @@ -0,0 +1,50 @@ +"""Utility functions for the service.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from entities_service.service.backend import get_backend +from entities_service.service.config import CONFIG + +if TYPE_CHECKING: # pragma: no cover + from typing import Any + + +async def _add_dimensions(entity: dict[str, Any] | list[dict[str, Any]]) -> None: + """Utility function for the endpoints to add dimensions to an entity.""" + if isinstance(entity, list): + for entity_ in entity: + await _add_dimensions(entity_) + return + + if "dimensions" not in entity: + # SOFT5 + if isinstance(entity["properties"], list): + entity["dimensions"] = [] + + # SOFT7 + elif isinstance(entity["properties"], dict): + entity["dimensions"] = {} + + else: + raise ValueError(f"Invalid entity: uri={entity['uri']}") + + +async def _get_entity(version: str, name: str, db: str | None = None) -> dict[str, Any]: + """Utility function for the endpoints to retrieve an entity.""" + uri = f"{str(CONFIG.base_url).rstrip('/')}" + + if db: + uri += f"/{db}" + + uri += f"/{version}/{name}" + + entity = get_backend(db=db).read(uri) + + if entity is None: + raise ValueError(f"Could not find entity: uri={uri}") + + await _add_dimensions(entity) + + return entity diff --git a/tests/cli/commands/test_upload.py b/tests/cli/commands/test_upload.py index 750fba1b..3b54bd5a 100644 --- a/tests/cli/commands/test_upload.py +++ b/tests/cli/commands/test_upload.py @@ -60,7 +60,11 @@ def test_upload_filepath( core_namespace = str(CONFIG.base_url).rstrip("/") current_namespace = f"{core_namespace}/{namespace}" if namespace else core_namespace - assert "uri" in raw_entity + id_key = "uri" if "uri" in raw_entity else "identity" + assert id_key in raw_entity + + # Match raw_entity to a backend entity (always use 'uri') + raw_entity["uri"] = raw_entity.pop(id_key) if namespace: # Update the entity's namespace to the current namespace @@ -82,7 +86,7 @@ def test_upload_filepath( # Mock response for "Check if entity already exists" httpx_mock.add_response( - url=raw_entity["uri"], + url=raw_entity[id_key], status_code=404, # not found ) @@ -176,11 +180,13 @@ def test_upload_directory( directory = tmp_path / "valid_entities" directory.mkdir(parents=True, exist_ok=True) for index, raw_entity in enumerate(raw_entities): + id_key = "uri" if "uri" in raw_entity else "identity" + # Update the entity's namespace to the current namespace if "namespace" in raw_entity: raw_entity["namespace"] = current_namespace - if "uri" in raw_entity: - raw_entity["uri"] = raw_entity["uri"].replace( + if id_key in raw_entity: + raw_entity[id_key] = raw_entity[id_key].replace( f"{core_namespace}/", f"{current_namespace}/" ) @@ -196,9 +202,9 @@ def test_upload_directory( # Mock response for "Check if entity already exists" for raw_entity in raw_entities: - assert "uri" in raw_entity + assert any(_ in raw_entity for _ in ("uri", "identity")) httpx_mock.add_response( - url=raw_entity["uri"], + url=raw_entity.get("uri", raw_entity.get("identity")), status_code=404, # not found ) @@ -322,14 +328,16 @@ def test_existing_entity_different_content( core_namespace = str(CONFIG.base_url).rstrip("/") current_namespace = f"{core_namespace}/{namespace}" if namespace else core_namespace + assert "uri" in raw_entity + if namespace: # Update the entity's namespace to the current namespace if "namespace" in raw_entity: raw_entity["namespace"] = current_namespace - if "uri" in raw_entity: - raw_entity["uri"] = raw_entity["uri"].replace( - f"{core_namespace}/", f"{current_namespace}/" - ) + + raw_entity["uri"] = raw_entity["uri"].replace( + f"{core_namespace}/", f"{current_namespace}/" + ) # Write the updated entity to file directory = tmp_path / "valid_entities" @@ -345,7 +353,6 @@ def test_existing_entity_different_content( ) # Mock response for "Check if entity already exists" - assert "uri" in raw_entity httpx_mock.add_response( url=raw_entity["uri"], status_code=200, # ok @@ -549,14 +556,16 @@ def test_existing_entity_errors( core_namespace = str(CONFIG.base_url).rstrip("/") current_namespace = f"{core_namespace}/{namespace}" if namespace else core_namespace + assert "uri" in raw_entity + if namespace: # Update the entity's namespace to the current namespace if "namespace" in raw_entity: raw_entity["namespace"] = current_namespace - if "uri" in raw_entity: - raw_entity["uri"] = raw_entity["uri"].replace( - f"{core_namespace}/", f"{current_namespace}/" - ) + + raw_entity["uri"] = raw_entity["uri"].replace( + f"{core_namespace}/", f"{current_namespace}/" + ) # Write the updated entity to file directory = tmp_path / "valid_entities" @@ -572,7 +581,6 @@ def test_existing_entity_errors( ) # Mock response for "Check if entity already exists" - assert "uri" in raw_entity httpx_mock.add_response( url=raw_entity["uri"], status_code=200, # ok @@ -859,7 +867,9 @@ def test_using_stdin( valid_entities_dir = static_dir / "valid_entities" entity_uris: list[str] = [ - json.loads(filepath.read_text())["uri"] + json.loads(filepath.read_text()).get( + "uri", json.loads(filepath.read_text()).get("identity") + ) for filepath in valid_entities_dir.glob("*.json") ] diff --git a/tests/cli/commands/test_validate.py b/tests/cli/commands/test_validate.py index 54b4fe71..31b00f0d 100644 --- a/tests/cli/commands/test_validate.py +++ b/tests/cli/commands/test_validate.py @@ -55,20 +55,22 @@ def test_validate_filepath( core_namespace = str(CONFIG.base_url).rstrip("/") current_namespace = f"{core_namespace}/{namespace}" if namespace else core_namespace + assert "uri" in raw_entity + if namespace: # Update the entity's namespace to the current namespace if "namespace" in raw_entity: raw_entity["namespace"] = current_namespace - if "uri" in raw_entity: - raw_entity["uri"] = raw_entity["uri"].replace( - f"{core_namespace}/", f"{current_namespace}/" - ) + + raw_entity["uri"] = raw_entity["uri"].replace( + f"{core_namespace}/", f"{current_namespace}/" + ) + # Write the updated entity to file entity_filepath = tmp_path / "Person.json" entity_filepath.write_text(json.dumps(raw_entity)) # Mock response for "Check if entity already exists" - assert "uri" in raw_entity httpx_mock.add_response( url=raw_entity["uri"], status_code=404, # not found @@ -116,6 +118,9 @@ def test_validate_filepath_invalid( # This is to ensure the same error is given when hitting the specific namespace # endpoint invalid_entity: dict[str, Any] = json.loads(invalid_entity_filepath.read_text()) + + assert "identity" not in invalid_entity + if "namespace" in invalid_entity: invalid_entity["namespace"] = current_namespace if "uri" in invalid_entity: @@ -246,10 +251,12 @@ def test_validate_directory( directory.mkdir(parents=True, exist_ok=True) for index, raw_entity in enumerate(raw_entities): # Update the entity's namespace to the current namespace + id_key = "identity" if "identity" in raw_entity else "uri" + if "namespace" in raw_entity: raw_entity["namespace"] = current_namespace - if "uri" in raw_entity: - raw_entity["uri"] = raw_entity["uri"].replace( + if id_key in raw_entity: + raw_entity[id_key] = raw_entity[id_key].replace( f"{core_namespace}/", f"{current_namespace}/" ) @@ -258,9 +265,10 @@ def test_validate_directory( # Mock response for "Check if entity already exists" for raw_entity in raw_entities: - assert "uri" in raw_entity + id_key = "identity" if "identity" in raw_entity else "uri" + assert id_key in raw_entity httpx_mock.add_response( - url=raw_entity["uri"], + url=raw_entity[id_key], status_code=404, # not found ) @@ -368,12 +376,15 @@ def test_validate_directory_invalid_entities( # This is to ensure the same error is given when hitting the specific # namespace endpoint invalid_entity: dict[str, Any] = json.loads(filepath.read_text()) + + id_key = "identity" if "identity" in invalid_entity else "uri" + if "namespace" in invalid_entity: invalid_entity["namespace"] = invalid_entity["namespace"].replace( core_namespace, current_namespace ) - if "uri" in invalid_entity: - invalid_entity["uri"] = invalid_entity["uri"].replace( + if id_key in invalid_entity: + invalid_entity[id_key] = invalid_entity[id_key].replace( f"{core_namespace}/", f"{current_namespace}/" ) @@ -943,9 +954,10 @@ def test_list_of_entities_in_single_file( # Mock response for "Check if entity already exists" for raw_entity in raw_entities: - assert "uri" in raw_entity + id_key = "identity" if "identity" in raw_entity else "uri" + assert id_key in raw_entity httpx_mock.add_response( - url=raw_entity["uri"], + url=raw_entity[id_key], status_code=404, # not found ) @@ -1172,7 +1184,8 @@ def test_validate_strict( file_inputs += f" {filepath}" - assert "uri" in raw_entity + id_key = "identity" if "identity" in raw_entity else "uri" + assert id_key in raw_entity # Let's say half exist externally already if index % 2 == 0: @@ -1198,7 +1211,7 @@ def test_validate_strict( number_existing_changed_entities += 1 httpx_mock.add_response( - url=existing_entity_content["uri"], + url=existing_entity_content[id_key], status_code=200, # ok json=existing_entity_content, ) @@ -1209,7 +1222,7 @@ def test_validate_strict( # While the other half do not exist externally... else: httpx_mock.add_response( - url=raw_entity["uri"], + url=raw_entity[id_key], status_code=404, # not found ) diff --git a/tests/conftest.py b/tests/conftest.py index 106e2449..2bf4b9f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -217,7 +217,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: for entity in entities: name: str | None = entity.get("name") if name is None: - uri: str | None = entity.get("uri") + uri: str | None = entity.get("uri", entity.get("identity")) if uri is None: raise ValueError( "Could not retrieve neither uri and name from test entity." @@ -326,13 +326,14 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: from entities_service.service.config import CONFIG - def get_version_name(uri: str) -> tuple[str, str]: + def get_uri_parts(uri: str) -> tuple[str, str]: """Return the version and name part of a uri.""" # namespace = "http://onto-ns.com/meta" namespace = str(CONFIG.base_url).rstrip("/") match = re.match( - rf"^{re.escape(namespace)}/(?P[^/]+)/(?P[^/]+)$", uri + rf"^{re.escape(namespace)}/(?P[^/]+)/(?P[^/]+)$", + uri, ) assert match is not None, ( f"Could not retrieve version and name from {uri!r}. " @@ -341,7 +342,10 @@ def get_version_name(uri: str) -> tuple[str, str]: "Hint: Did you (inadvertently) set the base_url to something?" ) - return match.group("version") or "", match.group("name") or "" + return ( + match.group("version") or "", + match.group("name") or "", + ) def get_uri(entity: dict[str, Any]) -> str: """Return the uri for an entity.""" @@ -364,13 +368,22 @@ def get_uri(entity: dict[str, Any]) -> str: ) for entity in entities: - uri = entity.get("uri") or get_uri(entity) + uri = entity.get("uri", entity.get("identity")) or get_uri(entity) - version, name = get_version_name(uri) + version, name = get_uri_parts(uri) - # Replace $ref with ref backend_entity = deepcopy(entity) + # Replace 'identity' with 'uri' + if "identity" in backend_entity: + if "uri" in backend_entity: + raise ValueError( + "Both 'identity' and 'uri' keys are present in the test entity. " + "Please remove one of them." + ) + backend_entity["uri"] = backend_entity.pop("identity") + + # Replace '$ref' with 'ref' # SOFT5 if isinstance(backend_entity["properties"], list): backend_entity["properties"] = [ @@ -557,11 +570,11 @@ def _reset_mongo_test_collections( # First, prepare the test data - # Convert all '$ref' to 'ref' in the valid_entities.yaml file entities: list[dict[str, Any]] = yaml.safe_load( (static_dir / "valid_entities.yaml").read_text() ) for entity in entities: + ## Convert all '$ref' to 'ref' in the valid_entities.yaml file # SOFT5 if isinstance(entity["properties"], list): for index, property_value in enumerate(list(entity["properties"])): @@ -581,16 +594,23 @@ def _reset_mongo_test_collections( f"Invalid type for entity['properties']: {type(entity['properties'])}" ) + ## Convert all "identity" to "uri" + if "identity" in entity: + entity["uri"] = entity.pop("identity") + # For the specific namespace collection, rename all uris (and namespaces) to be - # within the specific namespace + # within the specific namespace. + specific_namespaced_entities = deepcopy(entities) core_namespace = str(CONFIG.base_url).rstrip("/") - specific_namespace_entities = deepcopy(entities) - for entity in specific_namespace_entities: + + for entity in specific_namespaced_entities: if "uri" in entity: + # Ensure the backend uses `uri` instead of `identity` entity["uri"] = entity["uri"].replace( core_namespace, f"{core_namespace}/{existing_specific_namespace}", ) + if "namespace" in entity: entity["namespace"] = f"{core_namespace}/{existing_specific_namespace}" @@ -608,9 +628,7 @@ def _reset_mongo_test_collections( ) backend._collection.drop() backend._collection.insert_many( - specific_namespace_entities - if namespace == existing_specific_namespace - else entities + entities if namespace is None else specific_namespaced_entities ) @@ -628,22 +646,26 @@ def _mock_lifespan(live_backend: bool, monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture() def _empty_backend_collection( - live_backend: bool, get_backend_user: GetBackendUserFixture + get_backend_user: GetBackendUserFixture, + existing_specific_namespace: str, ) -> None: """Empty the backend collection.""" from entities_service.service.backend import get_backend - backend_settings = {} - if live_backend: - backend_user = get_backend_user("write") - backend_settings = { - "mongo_username": backend_user["username"], - "mongo_password": backend_user["password"], - } + backend_user = get_backend_user("write") - backend: MongoDBBackend = get_backend(settings=backend_settings) - backend._collection.delete_many({}) - assert backend._collection.count_documents({}) == 0 + # None is equal to the core namespace + for namespace in (None, existing_specific_namespace): + backend: MongoDBBackend = get_backend( + auth_level="write", + settings={ + "mongo_username": backend_user["username"], + "mongo_password": backend_user["password"], + }, + db=namespace, + ) + backend._collection.delete_many({}) + assert backend._collection.count_documents({}) == 0 @pytest.fixture() diff --git a/tests/service/backend/test_mongodb.py b/tests/service/backend/test_mongodb.py index 1def7fb5..59abec54 100644 --- a/tests/service/backend/test_mongodb.py +++ b/tests/service/backend/test_mongodb.py @@ -36,6 +36,7 @@ def _mongo_backend(auth: Literal["read", "write"] | None = None) -> MongoDBBacke "mongo_username": backend_user["username"], "mongo_password": backend_user["password"], }, + db=None, ) return _mongo_backend diff --git a/tests/service/routers/test_admin.py b/tests/service/routers/test_admin.py index 117dea1e..740ba402 100644 --- a/tests/service/routers/test_admin.py +++ b/tests/service/routers/test_admin.py @@ -50,8 +50,10 @@ def test_create_single_entity( # Update namespace in entity if "namespace" in entity: entity["namespace"] = current_namespace - if "uri" in entity: - entity["uri"] = entity["uri"].replace(core_namespace, current_namespace) + + id_key = "uri" if "uri" in entity else "identity" + if id_key in entity: + entity[id_key] = entity[id_key].replace(core_namespace, current_namespace) # Create single entity with client(auth_role="write") as client_: @@ -63,6 +65,20 @@ def test_create_single_entity( response_json = response.json() + # Update entity according to the expected response + if "identity" in entity: + entity["uri"] = entity.pop("identity") + + if "dimensions" not in entity: + # SOFT5 + if isinstance(entity["properties"], list): + entity["dimensions"] = [] + # SOFT7 + elif isinstance(entity["properties"], dict): + entity["dimensions"] = {} + else: + pytest.fail(f"Invalid entity: {entity}") + # Check response assert response.status_code == 201, response_json assert isinstance(response_json, dict), response_json @@ -78,6 +94,8 @@ def test_create_multiple_entities( get_backend_user: GetBackendUserFixture, ) -> None: """Test creating multiple entities.""" + from copy import deepcopy + import yaml from entities_service.service.backend import get_backend @@ -95,11 +113,15 @@ def test_create_multiple_entities( # Add specific namespace entities core_namespace = str(CONFIG.base_url).rstrip("/") specific_namespace = f"{core_namespace}/{existing_specific_namespace}" + for entity in list(entities): + id_key = "uri" if "uri" in entity else "identity" + if id_key in entity: + entity[id_key] = entity[id_key].replace(core_namespace, specific_namespace) + if "namespace" in entity: entity["namespace"] = specific_namespace - if "uri" in entity: - entity["uri"] = entity["uri"].replace(core_namespace, specific_namespace) + entities.append(entity) # Create multiple entities @@ -112,14 +134,53 @@ def test_create_multiple_entities( response_json = response.json() + # Update entities according to the expected response + expected_response_entities = [] + expected_backend_entities = [] + + for entity in entities: + new_response_entity = deepcopy(entity) + new_backend_entity = deepcopy(entity) + + if "identity" in entity: + new_response_entity["uri"] = new_response_entity.pop("identity") + new_backend_entity["uri"] = new_backend_entity.pop("identity") + + # SOFT5 style + if isinstance(entity["properties"], list): + if "dimensions" not in entity: + new_response_entity["dimensions"] = [] + + new_backend_entity["properties"] = [ + {key.replace("$ref", "ref"): value for key, value in property_.items()} + for property_ in entity["properties"] + ] + + # SOFT7 + elif isinstance(entity["properties"], dict): + if "dimensions" not in entity: + new_response_entity["dimensions"] = {} + + for property_name, property_value in list(entity["properties"].items()): + new_backend_entity["properties"][property_name] = { + key.replace("$ref", "ref"): value + for key, value in property_value.items() + } + + else: + pytest.fail(f"Invalid entity: {entity}") + + expected_response_entities.append(new_response_entity) + expected_backend_entities.append(new_backend_entity) + # Check response assert response.status_code == 201, response_json assert isinstance(response_json, list), response_json - assert response_json == entities, response_json + assert response_json == expected_response_entities, response_json assert len(response_json) == 2 * original_length, response_json # Check they can be retrieved - for entity in entities: + for entity in expected_response_entities: uri = entity.get("uri", None) or ( f"{entity.get('namespace', '')}/{entity.get('version', '')}" f"/{entity.get('name', '')}" @@ -149,29 +210,12 @@ def test_create_multiple_entities( }, db=existing_specific_namespace, ) - for entity in entities: + for entity in expected_backend_entities: uri = entity.get("uri", None) or ( f"{entity.get('namespace', '')}/{entity.get('version', '')}" f"/{entity.get('name', '')}" ) - # Match the entity with how they are stored in the backend (MongoDB) - # SOFT5 style - if isinstance(entity.get("properties", None), list): - entity["properties"] = [ - {key.replace("$ref", "ref"): value for key, value in property_.items()} - for property_ in entity["properties"] - ] - # SOFT7 style - elif isinstance(entity.get("properties", None), dict): - for property_name, property_value in list(entity["properties"].items()): - entity["properties"][property_name] = { - key.replace("$ref", "ref"): value - for key, value in property_value.items() - } - else: - pytest.fail("Invalid entity: {entity}") - if uri.startswith(specific_namespace): assert specific_backend.read(uri) == entity, ( f"uri={uri} collection={specific_backend._collection.name} " @@ -252,7 +296,7 @@ def test_create_invalid_entity( # Create single invalid entities for entity in entities: - uri = entity.get("uri", None) or ( + uri = entity.get("uri", entity.get("identity", None)) or ( f"{entity.get('namespace', '')}/{entity.get('version', '')}" f"/{entity.get('name', '')}" ) @@ -352,7 +396,7 @@ def mock_create(*args: Any, **kwargs: Any) -> None: # noqa: ARG001 assert "detail" in response_json, response_json assert response_json["detail"] == ( "Could not create entities with uris: " - f"{', '.join(entity['uri'] for entity in entities)}" + f"{', '.join(entity.get('uri', entity.get('identity')) for entity in entities)}" ), response_json @@ -434,8 +478,26 @@ def test_create_entity_in_new_namespace( # Update namespace in entity if "namespace" in entity: entity["namespace"] = current_namespace - if "uri" in entity: - entity["uri"] = entity["uri"].replace(core_namespace, current_namespace) + + id_key = "uri" if "uri" in entity else "identity" + if id_key in entity: + entity[id_key] = entity[id_key].replace(core_namespace, current_namespace) + + # Create expected entity + expected_entity = deepcopy(entity) + + if "identity" in expected_entity: + expected_entity["uri"] = expected_entity.pop("identity") + + if "dimensions" not in expected_entity: + # SOFT5 + if isinstance(expected_entity["properties"], list): + expected_entity["dimensions"] = [] + # SOFT7 + elif isinstance(expected_entity["properties"], dict): + expected_entity["dimensions"] = {} + else: + pytest.fail(f"Invalid entity: {expected_entity}") # Ensure the backend does not exist backend_user = get_backend_user() @@ -463,7 +525,7 @@ def test_create_entity_in_new_namespace( # Check response assert response.status_code == 201, response_json assert isinstance(response_json, dict), response_json - assert response_json == entity, response_json + assert response_json == expected_entity, response_json # Check backend current_collections = new_backend._collection.database.list_collection_names() diff --git a/tests/static/valid_entities.yaml b/tests/static/valid_entities.yaml index f371bc7e..3ec64e0c 100644 --- a/tests/static/valid_entities.yaml +++ b/tests/static/valid_entities.yaml @@ -79,3 +79,17 @@ shape: - n_dogs description: The person's dogs. + +# Test identity +- identity: http://onto-ns.com/meta/1.0/Owl + description: An owl. + properties: + specific-species: + type: string + description: Specific species of owl. + age: + type: int + description: Age of the owl. + color: + type: string + description: Color of the owl. diff --git a/tests/test_main.py b/tests/test_main.py index e61ad4c7..bb9e2c16 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -18,6 +18,7 @@ def test_get_entity( ) -> None: """Test the route to retrieve an entity.""" import json + from copy import deepcopy from fastapi import status @@ -47,9 +48,27 @@ def test_get_entity( core_namespace = str(CONFIG.base_url).rstrip("/") current_namespace = f"{core_namespace}/{namespace}" if namespace else core_namespace retrieved_entity = response_json - print(retrieved_entity) + + # Created expected entity + expected_entity = deepcopy(parameterized_entity.entity) + if "identity" in expected_entity: + expected_entity["uri"] = expected_entity.pop("identity") + + # Assert necessary keys are present: + # uri OR namespace, version, name MUST be present + # dimensions and properties MUST be present + # properties MUST NOT be empty + assert "uri" in retrieved_entity or all( + _ in retrieved_entity for _ in ("namespace", "version", "name") + ) + assert "dimensions" in retrieved_entity + assert "properties" in retrieved_entity + assert retrieved_entity["properties"] + for key, value in retrieved_entity.items(): - assert key in parameterized_entity.entity, retrieved_entity + if key != "dimensions": + # Dimensions may have been added as an empty list or dict by the service + assert key in expected_entity, retrieved_entity if key == "uri": assert value == ( @@ -58,8 +77,11 @@ def test_get_entity( ), f"key: {key}" elif key == "namespace": assert value == current_namespace, f"key: {key}" + elif key == "dimensions": + assert isinstance(value, (list, dict)), f"key: {key}" + assert value == expected_entity.get(key, value), f"key: {key}" else: - assert value == parameterized_entity.entity[key], f"key: {key}" + assert value == expected_entity[key], f"key: {key}" @pytest.mark.skipif( @@ -74,18 +96,38 @@ def test_get_entity_instance( """Validate that we can instantiate a DLite Instance from the response""" from dlite import Instance + from entities_service.service.config import CONFIG + url_path = namespace or "" url_path += f"/{parameterized_entity.version}/{parameterized_entity.name}" with client() as client_: response = client_.get(url_path, timeout=5) - # Convert SOFT5 properties' 'dims' to 'shape' - for entity_property in parameterized_entity.entity["properties"]: - if "dims" in entity_property: - entity_property["shape"] = entity_property.pop("dims") + response_json = response.json() - Instance.from_dict(response.json()) + # Assert 'uri' is always returned, even if 'identity' was in the uploaded entity + if "identity" in parameterized_entity.entity: + assert "uri" in response_json + assert "identity" not in response_json + + if namespace: + assert response_json["uri"] == ( + f"{str(CONFIG.base_url).rstrip('/')}/{namespace}/{parameterized_entity.version}/{parameterized_entity.name}" + ) + else: + assert response_json["uri"] == parameterized_entity.entity["identity"] + + # Ensure at least an empty dimension is always returned + if ( + "dimensions" not in parameterized_entity.entity + or not parameterized_entity.entity["dimensions"] + ): + assert "dimensions" in response_json + assert isinstance(response_json["dimensions"], (list, dict)) + assert not response_json["dimensions"] + + Instance.from_dict(response_json) def test_get_entity_not_found(client: ClientFixture, namespace: str | None) -> None: