From 1d358759a69b6d456f7e15f59c3087b354e7267f Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 1 Feb 2024 10:50:21 +0100 Subject: [PATCH 01/40] WIP --- .dev/requirements_dev.txt | 1 + .vscode/launch.json | 20 +++ Dockerfile | 8 +- README.md | 68 ++++++++++ app/main.py | 5 +- app/models/parser.py | 65 +++++++++ app/routers/datafilter.py | 3 + app/routers/dataresource.py | 4 + app/routers/function.py | 3 + app/routers/mapping.py | 10 +- app/routers/parser.py | 162 +++++++++++++++++++++++ app/routers/redisadmin.py | 2 +- app/routers/session.py | 17 ++- app/routers/transformation.py | 5 + app/routers/triplestore.py | 2 +- composerun.py | 42 ++++++ docker-compose_dev.yml | 43 +++--- entrypoint.sh | 2 +- requirements.txt | 1 + test.ttl | 40 ++++++ tests/routers/test_parser.py | 18 +++ tests/static/test_strategies/function.py | 10 +- tests/static/test_strategies/resource.py | 4 +- 23 files changed, 494 insertions(+), 41 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 app/models/parser.py create mode 100644 app/routers/parser.py create mode 100644 composerun.py create mode 100644 test.ttl create mode 100644 tests/routers/test_parser.py diff --git a/.dev/requirements_dev.txt b/.dev/requirements_dev.txt index 90ecedda..16ddded5 100644 --- a/.dev/requirements_dev.txt +++ b/.dev/requirements_dev.txt @@ -1,4 +1,5 @@ dulwich~=0.21.7 +debugpy>=1.8.0 fakeredis~=2.20 httpx~=0.26.0 invoke~=2.2 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..1b5a08c0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ] + } + ] +} diff --git a/Dockerfile b/Dockerfile index 5b47895f..55a23957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,12 +27,12 @@ COPY .git .git/ COPY .dev/requirements_dev.txt .pre-commit-config.yaml pyproject.toml ./ # Run static security check, linters, and pytest with code coverage -RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt \ +RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt # Ignore ID 44715 for now. # See this NumPy issue for more information: https://github.com/numpy/numpy/issues/19038 - && pre-commit run --all-files \ - && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715 \ - && pytest --cov app +# && pre-commit run --all-files +# && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715 +# && pytest --cov app # Run app with reload option EXPOSE 8080 diff --git a/README.md b/README.md index 07a26951..63a662f0 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,74 @@ networks: otenet: ``` +## Debugging OTEAPI Service in Visual Studio + +### Prerequisites + * Ensure you have `docker` and `docker-compose` installed on your system. + * Install Visual Studio Code along with the Python extension and the Remote - Containers extension. + * Have the docker-compose_dev.yml file configured for your OTEAPI Service. + * Ensure `debugpy` is installed in your virtual environment + +### Configuring Visual Studio Code: + + * Open your project in Visual Studio Code. + * Go to the Run and Debug view (Ctrl+Shift+D or ⌘+Shift+D on macOS). + * Create a launch.json file in the .vscode folder at the root of your project (if not already present). This file should contain something like this: + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ] + } + ] + } + ``` + +### Update the entrypoint.sh +In order to enable remote debugging, update the file `entrypoint.sh` such that the oteapi is started using the following command: + +``` +python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" +``` + +This will run the OTEAPI service in remote debug mode. The debugpy debugger will wait until a remote debuggin client is attached to port 5678. This happens when activating the "Run and Debug | Python: Remote Attach" from Visual Studio Code. For other IDEs please follow the relevant documentation. + + + +### Starting the docker-compose + * Open a terminal and navigate to the directory containing your docker-compose_dev.yml file. + * Start the service using the command: + + ```sh + docker-compose -f docker-compose_dev.yml up -d + ``` + +The `-d` option starts the OTEAPI sevice in detached mode. In order to access and follow the logs output you can run `docker-compose logs -f`. + +### Attaching to the Remote Process: + + * With the `launch.json` configured, go to the Run and Debug view in Visual Studio Code. + * Select the "Python: Remote Attach" configuration from the dropdown menu.* Click the green play button or press F5 to start the debugging session. + * Visual Studio Code will attach to the remote debugging session running inside the Docker container. + +### Debugging the Application: + +Set breakpoints in your code as needed. Interact with your FastAPI application as you normally would. Visual Studio Code will pause execution when a breakpoint is hit, allowing you to inspect variables, step through code, and debug your application. + ## Acknowledgment OTEAPI Core has been supported by the following projects: diff --git a/app/main.py b/app/main.py index f2355d69..927b9608 100644 --- a/app/main.py +++ b/app/main.py @@ -21,6 +21,7 @@ session, transformation, triplestore, + parser ) if TYPE_CHECKING: # pragma: no cover @@ -81,11 +82,13 @@ def create_app() -> FastAPI: available_routers = [ session, dataresource, + parser, + mapping, datafilter, function, - mapping, transformation, triplestore, + ] if CONFIG.include_redisadmin: available_routers.append(redisadmin) diff --git a/app/models/parser.py b/app/models/parser.py new file mode 100644 index 00000000..80eec389 --- /dev/null +++ b/app/models/parser.py @@ -0,0 +1,65 @@ +"""Parser specific pydantic response models.""" +from typing import Annotated +from uuid import uuid4 + +from pydantic import Field +from oteapi.models import AttrDict +from app.models.response import Session +from app.models.response import ( + CreateResponse, + GetResponse, + InitializeResponse, +) + +IDPREFIX = "parser" + +class CreateParserResponse(CreateResponse): + """ Create a response resource + + Router: `POST /parser` + """ + parser_id: Annotated[ + str, + Field( + default_factory=lambda: f"{IDPREFIX}-{uuid4()}", + description="The parser id.", + pattern=( + rf"^{IDPREFIX}-[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-" + r"[0-9a-f]{12}$" + ), + ), + ] + +class GetParserResponse(GetResponse): + """ Get a parser resource + + Router: `GET /parser/{parser_id}` + """ + +class InitializeParserResponse(InitializeResponse): + """ Initialize a parser resource + + Router: `POST /parser/{parser_id}/initialize` + """ + +class DeleteAllParsersResponse(AttrDict): + """### Delete all sessions + + Router: `DELETE /parser` + """ + + number_of_deleted_parsers: Annotated[ + int, Field(description="The number of deleted parsers in the Redis cache.") + ] + + +class ListParsersResponse(Session): + """ List all parser ids + + Router: `GET /parser` + """ + + keys_: Annotated[ + list[str], + Field(description="List of all parser ids in the cache.", alias="keys"), + ] diff --git a/app/routers/datafilter.py b/app/routers/datafilter.py index 4346746c..b9310290 100644 --- a/app/routers/datafilter.py +++ b/app/routers/datafilter.py @@ -30,6 +30,7 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["datafilter"], ) async def create_filter( cache: TRedisPlugin, @@ -60,6 +61,7 @@ async def create_filter( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["datafilter"], ) async def get_filter( cache: TRedisPlugin, @@ -108,6 +110,7 @@ async def get_filter( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["datafilter"], ) async def initialize_filter( cache: TRedisPlugin, diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index bd80a615..7c69f8ab 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -33,6 +33,7 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["dataresource"], ) async def create_dataresource( cache: TRedisPlugin, @@ -80,6 +81,7 @@ async def create_dataresource( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["dataresource"], ) async def info_dataresource( cache: TRedisPlugin, @@ -106,6 +108,7 @@ async def info_dataresource( status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, }, + tags=["dataresource"], ) async def read_dataresource( cache: TRedisPlugin, @@ -170,6 +173,7 @@ async def read_dataresource( status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, }, + tags=["dataresource"], ) async def initialize_dataresource( cache: TRedisPlugin, diff --git a/app/routers/function.py b/app/routers/function.py index ae82e943..177a8a77 100644 --- a/app/routers/function.py +++ b/app/routers/function.py @@ -26,6 +26,7 @@ "/", response_model=CreateFunctionResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["function"], ) async def create_function( cache: TRedisPlugin, @@ -61,6 +62,7 @@ async def create_function( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["function"], ) async def get_function( cache: TRedisPlugin, @@ -109,6 +111,7 @@ async def get_function( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["function"], ) async def initialize_function( cache: TRedisPlugin, diff --git a/app/routers/mapping.py b/app/routers/mapping.py index a46e7236..1595a3d9 100644 --- a/app/routers/mapping.py +++ b/app/routers/mapping.py @@ -26,6 +26,7 @@ "/", response_model=CreateMappingResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["mapping"], ) async def create_mapping( cache: TRedisPlugin, @@ -59,6 +60,7 @@ async def create_mapping( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["mapping"], ) async def get_mapping( cache: TRedisPlugin, @@ -107,13 +109,19 @@ async def get_mapping( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["mapping"], ) async def initialize_mapping( cache: TRedisPlugin, mapping_id: str, session_id: Optional[str] = None, ) -> InitializeMappingResponse: - """Initialize and update session.""" + """ + Initialize and update session. + + - **mapping_id**: Unique identifier of a mapping configuration + - **session_id**: Optional reference to a session object + """ if not await cache.exists(mapping_id): raise httpexception_404_item_id_does_not_exist(mapping_id, "mapping_id") if session_id and not await cache.exists(session_id): diff --git a/app/routers/parser.py b/app/routers/parser.py new file mode 100644 index 00000000..9c075793 --- /dev/null +++ b/app/routers/parser.py @@ -0,0 +1,162 @@ +from typing import Optional, Any, TYPE_CHECKING +import json +import logging + +from fastapi import APIRouter, Request, status +from oteapi.models import ParserConfig +from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session + +from app.models.parser import ( + IDPREFIX, + CreateParserResponse, + GetParserResponse, + InitializeParserResponse, + DeleteAllParsersResponse, + ListParsersResponse +) +from app.models.error import ( + HTTPNotFoundError, + HTTPValidationError, + httpexception_404_item_id_does_not_exist, + httpexception_422_resource_id_is_unprocessable, +) +from app.redis_cache import TRedisPlugin +from app.routers.session import _update_session, _update_session_list_item + +if TYPE_CHECKING: # pragma: no cover + from oteapi.interfaces import IParseStrategy + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("app.routers.parser") + +ROUTER = APIRouter(prefix=f"/{IDPREFIX}") + + +async def _validate_cache_key( + cache: TRedisPlugin, + key: str, + key_type: str) -> None: + + """Validate if a key exists in cache and is of expected type (str or bytes).""" + if not await cache.exists(key): + raise httpexception_404_item_id_does_not_exist(key, key_type) + + cache_value = await cache.get(key) + if not isinstance(cache_value, (str, bytes)): + raise TypeError( + f"Expected cache value of {key} to be a string or bytes, " + f"found it to be of type: `{type(cache_value)!r}`." + ) + + +# Create parser +@ROUTER.post( + "/", + response_model=CreateParserResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"]) + +async def create_parser( + cache: TRedisPlugin, + config: ParserConfig, + request: Request, + session_id: Optional[str] = None + ) -> CreateParserResponse: + + """Create a new parser and store its configuration in cache.""" + new_parser = CreateParserResponse() + + await cache.set(new_parser.parser_id, config.model_dump_json()) + + if session_id: + await _validate_cache_key(cache, session_id, "session_id") + await _update_session_list_item( + session_id=session_id, + list_key="resource_info", + list_items=[new_parser.parser_id], + redis=cache, + ) + + return new_parser + +@ROUTER.delete( + "/", + response_model=DeleteAllParsersResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"], +) +async def delete_all_parsers( + cache: TRedisPlugin, + ) -> DeleteAllParsersResponse: + """ + Delete all parser configurations in the current memory database + """ + + keylist = await cache.keys(pattern=f"{IDPREFIX}*") + return DeleteAllParsersResponse( + number_of_deleted_parsers = await cache.delete(*keylist) + ) + +# List parsers +@ROUTER.get( + "/", + response_model=ListParsersResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"]) + +async def list_parsers( + cache: TRedisPlugin + ) -> ListParsersResponse: + """Retrieve all parser IDs from cache.""" + keylist = [key for key in await cache.keys(pattern=f"{IDPREFIX}*") if isinstance(key, (str, bytes))] + return ListParsersResponse(keys=keylist) + + +# Get parser info +@ROUTER.get( + "/{parser_id}/info", + response_model=ParserConfig, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"]) +async def info_parser( + cache: TRedisPlugin, + parser_id: str + ) -> ParserConfig: + """Get information about a specific parser.""" + await _validate_cache_key(cache, parser_id, "parser_id") + cache_value = await cache.get(parser_id) + return ParserConfig(**json.loads(cache_value)) + + +# Run `get` on parser +@ROUTER.get( + "/{parser_id}", + response_model=GetParserResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"]) + +async def get_parser( + cache: TRedisPlugin, + parser_id: str, + session_id: Optional[str] = None + ) -> GetParserResponse: + """Retrieve and parse data using a specified parser.""" + await _validate_cache_key(cache, parser_id, "parser_id") + config = ParserConfig(**json.loads(await cache.get(parser_id))) + + session_data: "Optional[dict[str, Any]]" = None + if session_id: + await _validate_cache_key(cache, session_id, "session_id") + session_data = json.loads(await cache.get(session_id)) + + populate_config_from_session(session_data, config) + strategy: "IParseStrategy" = create_strategy("parse", config) + + logger.debug(str(strategy.parse_config.model_dump())) + session_update = strategy.get() + + if session_update and session_id: + await _update_session(session_id=session_id, updated_session=session_update, redis=cache) + + return GetParserResponse(**session_update) diff --git a/app/routers/redisadmin.py b/app/routers/redisadmin.py index b60fef64..5c02382d 100644 --- a/app/routers/redisadmin.py +++ b/app/routers/redisadmin.py @@ -10,7 +10,7 @@ ROUTER = APIRouter(prefix="/redis") -@ROUTER.get("/{key}", include_in_schema=False) +@ROUTER.get("/{key}", include_in_schema=False, tags=["admin"]) async def get_key( cache: TRedisPlugin, key: str, diff --git a/app/routers/session.py b/app/routers/session.py index 8e953aaf..0635e441 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING from fastapi import APIRouter, status -from oteapi.models import SessionUpdate +from oteapi.models import AttrDict from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist from app.models.response import Session @@ -20,6 +20,10 @@ from typing import Any, Union +class SessionUpdate(AttrDict): + """Session Update Data Model for returning values.""" + + ROUTER = APIRouter(prefix=f"/{IDPREFIX}") @@ -27,6 +31,7 @@ "/", response_model=CreateSessionResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["session"], ) async def create_session( cache: TRedisPlugin, @@ -47,7 +52,6 @@ async def create_session( """ new_session = CreateSessionResponse() - await cache.set(new_session.session_id, session.model_dump_json()) return new_session @@ -56,6 +60,7 @@ async def create_session( "/", response_model=ListSessionsResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["session"], ) async def list_sessions( cache: TRedisPlugin, @@ -63,12 +68,12 @@ async def list_sessions( """Get all session keys""" keylist = [] for key in await cache.keys(pattern=f"{IDPREFIX}*"): - if not isinstance(key, bytes): + if not isinstance(key, (str, bytes)): raise TypeError( "Found a key that is not stored as bytes (stored as type " f"{type(key)!r})." ) - keylist.append(key.decode(encoding="utf-8")) + keylist.append(key) return ListSessionsResponse(keys=keylist) @@ -76,6 +81,7 @@ async def list_sessions( "/", response_model=DeleteAllSessionsResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["session"], ) async def delete_all_sessions( cache: TRedisPlugin, @@ -154,6 +160,7 @@ async def _update_session_list_item( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["session"], ) async def update_session( cache: TRedisPlugin, @@ -182,6 +189,7 @@ async def update_session( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["session"], ) async def get_session( cache: TRedisPlugin, @@ -207,6 +215,7 @@ async def get_session( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["session"], ) async def delete_session( cache: TRedisPlugin, diff --git a/app/routers/transformation.py b/app/routers/transformation.py index 31a3bb1d..b28ab23b 100644 --- a/app/routers/transformation.py +++ b/app/routers/transformation.py @@ -31,6 +31,7 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["transformation"], ) async def create_transformation( cache: TRedisPlugin, @@ -66,6 +67,7 @@ async def create_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["transformation"], ) async def get_transformation_status( cache: TRedisPlugin, @@ -96,6 +98,7 @@ async def get_transformation_status( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["transformation"], ) async def get_transformation( cache: TRedisPlugin, @@ -146,6 +149,7 @@ async def get_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["transformation"], ) async def execute_transformation( cache: TRedisPlugin, @@ -197,6 +201,7 @@ async def execute_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, + tags=["transformation"], ) async def initialize_transformation( cache: TRedisPlugin, diff --git a/app/routers/triplestore.py b/app/routers/triplestore.py index 95cb45f4..919f5f60 100644 --- a/app/routers/triplestore.py +++ b/app/routers/triplestore.py @@ -7,7 +7,7 @@ from oteapi.models import TripleStoreConfig from oteapi.triplestore import TripleStore -ROUTER = APIRouter(prefix="/triples") +ROUTER = APIRouter(prefix="/triples", tags=["triplestore"]) @ROUTER.post("/fetch") diff --git a/composerun.py b/composerun.py new file mode 100644 index 00000000..d0d0698c --- /dev/null +++ b/composerun.py @@ -0,0 +1,42 @@ +import argparse +import re + +def extract_env_variables(file_path): + pattern = re.compile(r'\$\{([^}:]+)(?::-([^}]*))?\}') + env_vars = {} + with open(file_path, 'r') as file: + content = file.read() + matches = pattern.findall(content) + for match in matches: + env_vars[match[0]] = match[1] + return env_vars + +def interactive_mode(env_vars): + user_values = {} + for var, default in env_vars.items(): + user_input = input(f"{var} [{default}]: ") or default + user_values[var] = user_input + return user_values + +def write_to_env_file(user_values, output_file): + with open(output_file, 'w') as file: + for var, value in user_values.items(): + file.write(f"{var}={value}\n") + +def main(): + parser = argparse.ArgumentParser(description="Extract environment variables from a Docker Compose file.") + parser.add_argument('-f', '--inputfile', type=str, help='Path to the Docker Compose file', required=True) + parser.add_argument('-i', '--interactive', action='store_true', help='Interactive mode') + args = parser.parse_args() + + env_vars = extract_env_variables(args.inputfile) + + if args.interactive: + user_values = interactive_mode(env_vars) + write_to_env_file(user_values, 'output.env') + else: + for var, default in env_vars.items(): + print(f"{var}={default}") + +if __name__ == "__main__": + main() diff --git a/docker-compose_dev.yml b/docker-compose_dev.yml index bbd74183..2a0ad68d 100644 --- a/docker-compose_dev.yml +++ b/docker-compose_dev.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.3" services: oteapi: @@ -7,6 +7,7 @@ services: target: "${DOCKER_OTEAPI_TARGET:-development}" ports: - "${PORT:-8080}:8080" + - "5678:5678" # Debug port environment: OTEAPI_REDIS_TYPE: redis OTEAPI_REDIS_HOST: redis @@ -32,30 +33,30 @@ services: networks: - otenet - sftp: - image: atmoz/sftp - volumes: - - sftp-storage:${HOME:-/home/foo}/download - command: ${USER:-foo}:${PASSWORD:-pass}:1001 - networks: - - otenet +# sftp: +# image: atmoz/sftp +# volumes: +# - sftp-storage:${HOME:-/home/foo}/download +# command: ${USER:-foo}:${PASSWORD:-pass}:1001 +# networks: +# - otenet - agraph: - image: franzinc/agraph:v7.2.0 - volumes: - - agraph-data:/agraph/data - - ./agraph.cfg:/agraph/etc/agraph.cfg - ports: - - "10000-10035:10000-10035" - restart: on-failure - shm_size: 4g - networks: - - otenet +# agraph: +# image: franzinc/agraph:v7.2.0 +# volumes: +# - agraph-data:/agraph/data +# - ./agraph.cfg:/agraph/etc/agraph.cfg +# ports: +# - "10000-10035:10000-10035" +# restart: on-failure +# shm_size: 4g +# networks: +# - otenet volumes: redis-persist: - sftp-storage: - agraph-data: +# sftp-storage: +# agraph-data: networks: otenet: diff --git a/entrypoint.sh b/entrypoint.sh index b18ea192..d1b86517 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,4 +33,4 @@ else echo "No extra plugin packages provided. Specify 'OTEAPI_PLUGIN_PACKAGES' to specify plugin packages." fi -hypercorn asgi:app --bind 0.0.0.0:8080 "$@" +python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" diff --git a/requirements.txt b/requirements.txt index 2a059f83..ca3ccc2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ click-didyoumean==0.3.0 click-plugins==1.1.1 click-repl==0.3.0 cryptography==41.0.7 +debugpy>=1.8.0 diskcache==5.6.3 et-xmlfile==1.1.0 exceptiongroup==1.2.0 diff --git a/test.ttl b/test.ttl new file mode 100644 index 00000000..1c8516a4 --- /dev/null +++ b/test.ttl @@ -0,0 +1,40 @@ +@prefix prov: . +@prefix rdfs: . +@prefix ex: . + +ex:InitialMethodDevelopmentOutcomes + a prov:Activity ; + rdfs:label "Initial method development outcomes at SINTEF" ; + prov:qualifiedAssociation [ + a prov:Association ; + prov:agent ex:SINTEFOcean ; + ] . + +ex:SINTEFOcean + a prov:Agent ; + rdfs:label "SINTEF Ocean" . + +ex:PolymerSolubility + a prov:Entity ; + rdfs:label "Solubility of polymers" ; + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . + +ex:PyGCMSMethodTestingOutcomes + a prov:Entity ; + rdfs:label "Py-GCMS method testing and initial outcome and observations" ; + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . + +ex:CalibrationCurvesAndLOD + a prov:Entity ; + rdfs:label "Calibration curves and LOD" ; + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . + +ex:AnalysisMethods + a prov:Entity ; + rdfs:label "Analysis methods" ; + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . + +ex:PSTestRecovery + a prov:Entity ; + rdfs:label "Test of PS NP recovery from MilliQ and Novachem" ; + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . \ No newline at end of file diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py new file mode 100644 index 00000000..84e204d6 --- /dev/null +++ b/tests/routers/test_parser.py @@ -0,0 +1,18 @@ +""" Test parser """ + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + +def test_create_parser(client: "TestClient") -> None: + """ Test creating a parser """ + response = client.post( + "/parser/", + json={ + "parserType": "parser/demo", + "configuration": {}, + }, + ) + assert "parser_id" in response.json() + assert response.status_code == 200 \ No newline at end of file diff --git a/tests/static/test_strategies/function.py b/tests/static/test_strategies/function.py index ca992e60..6eca3fb6 100644 --- a/tests/static/test_strategies/function.py +++ b/tests/static/test_strategies/function.py @@ -1,7 +1,7 @@ """Demo function strategy class.""" from typing import TYPE_CHECKING -from oteapi.models import FunctionConfig, SessionUpdate +from oteapi.models import AttrDict, FunctionConfig from pydantic.dataclasses import dataclass if TYPE_CHECKING: @@ -20,7 +20,7 @@ class DemoFunctionStrategy: function_config: FunctionConfig - def initialize(self, session: "Optional[dict[str, Any]]" = None) -> SessionUpdate: + def initialize(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: """Initialize strategy. This method will be called through the `/initialize` endpoint of the OTEAPI @@ -37,9 +37,9 @@ def initialize(self, session: "Optional[dict[str, Any]]" = None) -> SessionUpdat del session # unused del self.function_config # unused - return SessionUpdate() + return AttrDict() - def get(self, session: "Optional[dict[str, Any]]" = None) -> SessionUpdate: + def get(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: """Execute the strategy. This method will be called through the strategy-specific endpoint of the @@ -55,4 +55,4 @@ def get(self, session: "Optional[dict[str, Any]]" = None) -> SessionUpdate: """ del session # unused del self.function_config # unused - return SessionUpdate() + return AttrDict() diff --git a/tests/static/test_strategies/resource.py b/tests/static/test_strategies/resource.py index e27e6725..fcb458e8 100644 --- a/tests/static/test_strategies/resource.py +++ b/tests/static/test_strategies/resource.py @@ -1,8 +1,8 @@ """Demo resource strategy class.""" from typing import TYPE_CHECKING, Annotated, Optional +from oteapi.models import AttrDict from oteapi.models.resourceconfig import ResourceConfig -from oteapi.models.sessionupdate import SessionUpdate from oteapi.plugins.factories import create_strategy from pydantic import Field from pydantic.dataclasses import dataclass @@ -11,7 +11,7 @@ from typing import Any -class ResourceResult(SessionUpdate): +class ResourceResult(AttrDict): """Update session with the FilterResult model""" output: Annotated[Optional[str], Field(description="Optional result")] = None From 93bdde6885066c5b4d8ed2b15b556e7e96e99651 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Fri, 2 Feb 2024 11:35:33 +0100 Subject: [PATCH 02/40] Add test_parser and update session delete --- app/models/session.py | 6 +++++- app/routers/parser.py | 5 +++-- app/routers/session.py | 4 ++++ tests/routers/test_parser.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/app/models/session.py b/app/models/session.py index 02275b41..89a1408c 100644 --- a/app/models/session.py +++ b/app/models/session.py @@ -1,5 +1,5 @@ """Session-specific pydantic response models.""" -from typing import Annotated +from typing import Annotated, Optional from uuid import uuid4 from pydantic import Field @@ -56,6 +56,10 @@ class DeleteAllSessionsResponse(Session): number_of_deleted_sessions: Annotated[ int, Field(description="The number of deleted sessions in the Redis cache.") ] + message: Optional[str] = Field( + "All session keys deleted.", + description="Optional message indicating the result of the operation." + ) class DeleteSessionResponse(Session): diff --git a/app/routers/parser.py b/app/routers/parser.py index 9c075793..af5f8d53 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -149,8 +149,8 @@ async def get_parser( if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) - - populate_config_from_session(session_data, config) + populate_config_from_session(session_data, config) + strategy: "IParseStrategy" = create_strategy("parse", config) logger.debug(str(strategy.parse_config.model_dump())) @@ -160,3 +160,4 @@ async def get_parser( await _update_session(session_id=session_id, updated_session=session_update, redis=cache) return GetParserResponse(**session_update) + diff --git a/app/routers/session.py b/app/routers/session.py index 0635e441..e7ad6284 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -93,6 +93,10 @@ async def delete_all_sessions( """ keylist = await cache.keys(pattern=f"{IDPREFIX}*") + # Check if the keylist is empty + if not keylist: + return DeleteAllSessionsResponse(number_of_deleted_sessions=0,message="No session keys found to delete.") + return DeleteAllSessionsResponse( number_of_deleted_sessions=await cache.delete(*keylist) ) diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py index 84e204d6..7c713f89 100644 --- a/tests/routers/test_parser.py +++ b/tests/routers/test_parser.py @@ -15,4 +15,31 @@ def test_create_parser(client: "TestClient") -> None: }, ) assert "parser_id" in response.json() - assert response.status_code == 200 \ No newline at end of file + assert response.status_code == 200 + + +def test_delete_all_parsers(client: "TestClient") -> None: + """Test deleting all parsers""" + response = client.delete("/parser/") + assert response.status_code == 200 + +def test_list_parsers(client: "TestClient") -> None: + """Test listing parsers""" + response = client.get("/parser/") + assert response.status_code == 200 + assert "keys" in response.json() + +def test_info_parser(client: "TestClient") -> None: + """Test getting information about a parser""" + response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa/info") + assert response.status_code == 200 + assert "parserType" in response.json() + assert "configuration" in response.json() + +def test_get_parser(client: "TestClient") -> None: + """Test getting and parsing data using a specified parser""" + response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa") + assert response.status_code == 200 + assert "parserType" in response.json() + assert "configuration" in response.json() + From 2f9b8c57c4fd795697ed1953aa0f0e8d19607a8d Mon Sep 17 00:00:00 2001 From: "Thomas F. Hagelien" Date: Wed, 7 Feb 2024 11:50:42 +0100 Subject: [PATCH 03/40] Updated the dataresource. Removed instantiation of the download and parse strategies --- app/routers/dataresource.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 7c69f8ab..cb1c16f9 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -143,22 +143,10 @@ async def read_dataresource( None if not session_id else json.loads(cache_value) ) - if config.downloadUrl and config.mediaType: - # Download strategy - session_update = create_strategy("download", config).get(session=session_data) - if session_update and session_id: - session_data = await _update_session(session_id, session_update, cache) - # Parse strategy - session_update = create_strategy("parse", config).get(session=session_data) - if session_update and session_id: - await _update_session(session_id, session_update, cache) - - elif config.accessUrl and config.accessService: - # Resource strategy - session_update = create_strategy("resource", config).get(session=session_data) - if session_update and session_id: - await _update_session(session_id, session_update, cache) + session_update = create_strategy("resource", config).get(session=session_data) + if session_update and session_id: + await _update_session(session_id, session_update, cache) else: raise httpexception_422_resource_id_is_unprocessable(resource_id) From 608b2d539eeb65981971b5612fd31375b57c4d9a Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Thu, 8 Feb 2024 10:42:52 +0100 Subject: [PATCH 04/40] fix parser API to fetch from session --- app/routers/parser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/routers/parser.py b/app/routers/parser.py index af5f8d53..ca8ddeca 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -143,15 +143,14 @@ async def get_parser( ) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" await _validate_cache_key(cache, parser_id, "parser_id") - config = ParserConfig(**json.loads(await cache.get(parser_id))) - + config = json.loads(await cache.get(parser_id)) session_data: "Optional[dict[str, Any]]" = None if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) populate_config_from_session(session_data, config) - strategy: "IParseStrategy" = create_strategy("parse", config) + strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) logger.debug(str(strategy.parse_config.model_dump())) session_update = strategy.get() From c8ebc221d7883804d003fd002b2dd6440ee445ed Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Thu, 8 Feb 2024 11:51:43 +0100 Subject: [PATCH 05/40] add checks and fix initialize in dataresource strategy --- app/models/error.py | 2 +- app/routers/dataresource.py | 54 +++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/app/models/error.py b/app/models/error.py index 4ba2df3c..5fbabd95 100644 --- a/app/models/error.py +++ b/app/models/error.py @@ -46,7 +46,7 @@ def httpexception_422_resource_id_is_unprocessable(resource_id: str) -> HTTPExce detail=[ { "loc": ["resource_id"], - "msg": "Missing downloadUrl/mediaType or " + "msg": "Missing resourceType or downloadUrl/mediaType or " f"accessUrl/accessService identifier in {resource_id=}", "type": "Error", } diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index cb1c16f9..d10acaf6 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -38,7 +38,6 @@ async def create_dataresource( cache: TRedisPlugin, config: ResourceConfig, - request: Request, session_id: Optional[str] = None, ) -> CreateResourceResponse: """### Register an external data resource. @@ -56,7 +55,7 @@ async def create_dataresource( """ new_resource = CreateResourceResponse() - config.token = request.headers.get("Authorization") or config.token + # config.token = request.headers.get("Authorization") or config.token resource_config = config.model_dump_json() @@ -139,18 +138,26 @@ async def read_dataresource( f"Expected cache value of {session_id} to be a string or bytes, " f"found it to be of type {type(cache_value)!r}." ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - + + + if not config.resourceType: + raise httpexception_422_resource_id_is_unprocessable(resource_id) + + if config.downloadUrl and config.mediaType: + session_update = create_strategy("resource", config).get() + if session_update and session_id: + await _update_session( + session_id=session_id, updated_session=session_update, redis=cache + ) - session_update = create_strategy("resource", config).get(session=session_data) - if session_update and session_id: - await _update_session(session_id, session_update, cache) + elif config.accessUrl and config.accessService: + session_update = create_strategy("resource", config).get() + if session_update and session_id: + await _update_session(session_id, session_update, cache) else: raise httpexception_422_resource_id_is_unprocessable(resource_id) - + return GetResourceResponse(**session_update) @@ -189,36 +196,23 @@ async def initialize_dataresource( f"Expected cache value of {session_id} to be a string or bytes, " f"found it to be of type {type(cache_value)!r}." ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) + + if not config.resourceType: + raise httpexception_422_resource_id_is_unprocessable(resource_id) if config.downloadUrl and config.mediaType: # Download strategy - session_update = create_strategy("download", config).initialize( - session=session_data - ) - if session_update and session_id: - session_data = await _update_session( - session_id=session_id, updated_session=session_update, redis=cache - ) - - # Parse strategy - session_update = create_strategy("parse", config).initialize( - session=session_data - ) + session_update = create_strategy("resource", config).initialize() if session_update and session_id: - session_data = await _update_session( + await _update_session( session_id=session_id, updated_session=session_update, redis=cache ) elif config.accessUrl and config.accessService: # Resource strategy - session_update = create_strategy("resource", config).initialize( - session=session_data - ) + session_update = create_strategy("resource", config).initialize() if session_update and session_id: - session_data = await _update_session( + await _update_session( session_id=session_id, updated_session=session_update, redis=cache ) From 203eccc071a949b5105d8f2b455c779fa7c5f4c5 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Thu, 8 Feb 2024 12:16:46 +0100 Subject: [PATCH 06/40] modify parser API's --- app/routers/parser.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/routers/parser.py b/app/routers/parser.py index ca8ddeca..47e0ca7e 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -2,7 +2,7 @@ import json import logging -from fastapi import APIRouter, Request, status +from fastapi import APIRouter, status from oteapi.models import ParserConfig from oteapi.plugins import create_strategy from oteapi.utils.config_updater import populate_config_from_session @@ -11,15 +11,12 @@ IDPREFIX, CreateParserResponse, GetParserResponse, - InitializeParserResponse, DeleteAllParsersResponse, ListParsersResponse ) from app.models.error import ( HTTPNotFoundError, - HTTPValidationError, httpexception_404_item_id_does_not_exist, - httpexception_422_resource_id_is_unprocessable, ) from app.redis_cache import TRedisPlugin from app.routers.session import _update_session, _update_session_list_item @@ -60,7 +57,6 @@ async def _validate_cache_key( async def create_parser( cache: TRedisPlugin, config: ParserConfig, - request: Request, session_id: Optional[str] = None ) -> CreateParserResponse: @@ -92,8 +88,9 @@ async def delete_all_parsers( """ Delete all parser configurations in the current memory database """ - keylist = await cache.keys(pattern=f"{IDPREFIX}*") + if not keylist: + return DeleteAllParsersResponse(number_of_deleted_parsers=0) return DeleteAllParsersResponse( number_of_deleted_parsers = await cache.delete(*keylist) ) @@ -159,4 +156,3 @@ async def get_parser( await _update_session(session_id=session_id, updated_session=session_update, redis=cache) return GetParserResponse(**session_update) - From d50f461b06fd0b1a7f3b4c37e3da95a9b3c55374 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 12 Feb 2024 12:37:07 +0100 Subject: [PATCH 07/40] Add initialize endpoint for parser --- app/routers/parser.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/app/routers/parser.py b/app/routers/parser.py index 47e0ca7e..8473e6b3 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -12,6 +12,7 @@ CreateParserResponse, GetParserResponse, DeleteAllParsersResponse, + InitializeParserResponse, ListParsersResponse ) from app.models.error import ( @@ -156,3 +157,35 @@ async def get_parser( await _update_session(session_id=session_id, updated_session=session_update, redis=cache) return GetParserResponse(**session_update) + +@ROUTER.post( + "/{parser_id}/initialize", + response_model=InitializeParserResponse, + responses={ + status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, + }, + tags=["parser"], +) +async def initialize_dataresource( + cache: TRedisPlugin, + parser_id: str, + session_id: Optional[str] = None, +) -> InitializeParserResponse: + """Initialize parser.""" + await _validate_cache_key(cache, parser_id, "parser_id") + config = json.loads(await cache.get(parser_id)) + session_data: "Optional[dict[str, Any]]" = None + if session_id: + await _validate_cache_key(cache, session_id, "session_id") + session_data = json.loads(await cache.get(session_id)) + populate_config_from_session(session_data, config) + + strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) + + logger.debug(str(strategy.parse_config.model_dump())) + session_update = strategy.initialize() + + if session_update and session_id: + await _update_session(session_id=session_id, updated_session=session_update, redis=cache) + + return InitializeParserResponse(**session_update) From f691179bbd3bb7af47c61c42d9d24ebbc3cf9904 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 19 Feb 2024 14:26:15 +0100 Subject: [PATCH 08/40] rename endpoint --- app/routers/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/parser.py b/app/routers/parser.py index 8473e6b3..8058bc1b 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -166,7 +166,7 @@ async def get_parser( }, tags=["parser"], ) -async def initialize_dataresource( +async def initialize_parser( cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None, From 65b03cfdefd72eca7ae35c059ecf42e525c39859 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 19 Feb 2024 14:35:47 +0100 Subject: [PATCH 09/40] fix pre-commit errors --- app/main.py | 3 +- app/models/parser.py | 23 +++++---- app/models/session.py | 2 +- app/routers/dataresource.py | 33 ++++-------- app/routers/parser.py | 97 +++++++++++++++++------------------- app/routers/session.py | 6 ++- composerun.py | 29 ++++++++--- test.ttl | 2 +- tests/routers/test_parser.py | 9 ++-- 9 files changed, 104 insertions(+), 100 deletions(-) diff --git a/app/main.py b/app/main.py index 927b9608..09d0d70e 100644 --- a/app/main.py +++ b/app/main.py @@ -17,11 +17,11 @@ dataresource, function, mapping, + parser, redisadmin, session, transformation, triplestore, - parser ) if TYPE_CHECKING: # pragma: no cover @@ -88,7 +88,6 @@ def create_app() -> FastAPI: function, transformation, triplestore, - ] if CONFIG.include_redisadmin: available_routers.append(redisadmin) diff --git a/app/models/parser.py b/app/models/parser.py index 80eec389..e3c035fb 100644 --- a/app/models/parser.py +++ b/app/models/parser.py @@ -2,22 +2,20 @@ from typing import Annotated from uuid import uuid4 -from pydantic import Field from oteapi.models import AttrDict -from app.models.response import Session -from app.models.response import ( - CreateResponse, - GetResponse, - InitializeResponse, -) +from pydantic import Field + +from app.models.response import CreateResponse, GetResponse, InitializeResponse, Session IDPREFIX = "parser" + class CreateParserResponse(CreateResponse): - """ Create a response resource + """Create a response resource Router: `POST /parser` """ + parser_id: Annotated[ str, Field( @@ -30,18 +28,21 @@ class CreateParserResponse(CreateResponse): ), ] + class GetParserResponse(GetResponse): - """ Get a parser resource + """Get a parser resource Router: `GET /parser/{parser_id}` """ + class InitializeParserResponse(InitializeResponse): - """ Initialize a parser resource + """Initialize a parser resource Router: `POST /parser/{parser_id}/initialize` """ + class DeleteAllParsersResponse(AttrDict): """### Delete all sessions @@ -54,7 +55,7 @@ class DeleteAllParsersResponse(AttrDict): class ListParsersResponse(Session): - """ List all parser ids + """List all parser ids Router: `GET /parser` """ diff --git a/app/models/session.py b/app/models/session.py index 89a1408c..813ca829 100644 --- a/app/models/session.py +++ b/app/models/session.py @@ -58,7 +58,7 @@ class DeleteAllSessionsResponse(Session): ] message: Optional[str] = Field( "All session keys deleted.", - description="Optional message indicating the result of the operation." + description="Optional message indicating the result of the operation.", ) diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index d10acaf6..7e516282 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -1,4 +1,5 @@ """Data Resource.""" + import json from typing import TYPE_CHECKING, Optional @@ -138,26 +139,21 @@ async def read_dataresource( f"Expected cache value of {session_id} to be a string or bytes, " f"found it to be of type {type(cache_value)!r}." ) - - + if not config.resourceType: raise httpexception_422_resource_id_is_unprocessable(resource_id) - - if config.downloadUrl and config.mediaType: + + if (config.downloadUrl and config.mediaType) or ( + config.accessUrl and config.accessService + ): session_update = create_strategy("resource", config).get() if session_update and session_id: await _update_session( session_id=session_id, updated_session=session_update, redis=cache ) - - elif config.accessUrl and config.accessService: - session_update = create_strategy("resource", config).get() - if session_update and session_id: - await _update_session(session_id, session_update, cache) - else: raise httpexception_422_resource_id_is_unprocessable(resource_id) - + return GetResourceResponse(**session_update) @@ -196,26 +192,19 @@ async def initialize_dataresource( f"Expected cache value of {session_id} to be a string or bytes, " f"found it to be of type {type(cache_value)!r}." ) - + if not config.resourceType: raise httpexception_422_resource_id_is_unprocessable(resource_id) - if config.downloadUrl and config.mediaType: + if (config.downloadUrl and config.mediaType) or ( + config.accessUrl and config.accessService + ): # Download strategy session_update = create_strategy("resource", config).initialize() if session_update and session_id: await _update_session( session_id=session_id, updated_session=session_update, redis=cache ) - - elif config.accessUrl and config.accessService: - # Resource strategy - session_update = create_strategy("resource", config).initialize() - if session_update and session_id: - await _update_session( - session_id=session_id, updated_session=session_update, redis=cache - ) - else: raise httpexception_422_resource_id_is_unprocessable(resource_id) diff --git a/app/routers/parser.py b/app/routers/parser.py index 8058bc1b..94a44441 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -1,23 +1,20 @@ -from typing import Optional, Any, TYPE_CHECKING import json import logging +from typing import TYPE_CHECKING, Any, Optional from fastapi import APIRouter, status from oteapi.models import ParserConfig from oteapi.plugins import create_strategy from oteapi.utils.config_updater import populate_config_from_session +from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist from app.models.parser import ( IDPREFIX, CreateParserResponse, - GetParserResponse, DeleteAllParsersResponse, + GetParserResponse, InitializeParserResponse, - ListParsersResponse -) -from app.models.error import ( - HTTPNotFoundError, - httpexception_404_item_id_does_not_exist, + ListParsersResponse, ) from app.redis_cache import TRedisPlugin from app.routers.session import _update_session, _update_session_list_item @@ -31,11 +28,7 @@ ROUTER = APIRouter(prefix=f"/{IDPREFIX}") -async def _validate_cache_key( - cache: TRedisPlugin, - key: str, - key_type: str) -> None: - +async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: """Validate if a key exists in cache and is of expected type (str or bytes).""" if not await cache.exists(key): raise httpexception_404_item_id_does_not_exist(key, key_type) @@ -50,17 +43,14 @@ async def _validate_cache_key( # Create parser @ROUTER.post( - "/", - response_model=CreateParserResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"]) - + "/", + response_model=CreateParserResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"], +) async def create_parser( - cache: TRedisPlugin, - config: ParserConfig, - session_id: Optional[str] = None - ) -> CreateParserResponse: - + cache: TRedisPlugin, config: ParserConfig, session_id: Optional[str] = None +) -> CreateParserResponse: """Create a new parser and store its configuration in cache.""" new_parser = CreateParserResponse() @@ -77,6 +67,7 @@ async def create_parser( return new_parser + @ROUTER.delete( "/", response_model=DeleteAllParsersResponse, @@ -85,7 +76,7 @@ async def create_parser( ) async def delete_all_parsers( cache: TRedisPlugin, - ) -> DeleteAllParsersResponse: +) -> DeleteAllParsersResponse: """ Delete all parser configurations in the current memory database """ @@ -93,21 +84,24 @@ async def delete_all_parsers( if not keylist: return DeleteAllParsersResponse(number_of_deleted_parsers=0) return DeleteAllParsersResponse( - number_of_deleted_parsers = await cache.delete(*keylist) + number_of_deleted_parsers=await cache.delete(*keylist) ) + # List parsers @ROUTER.get( - "/", - response_model=ListParsersResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"]) - -async def list_parsers( - cache: TRedisPlugin - ) -> ListParsersResponse: + "/", + response_model=ListParsersResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"], +) +async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: """Retrieve all parser IDs from cache.""" - keylist = [key for key in await cache.keys(pattern=f"{IDPREFIX}*") if isinstance(key, (str, bytes))] + keylist = [ + key + for key in await cache.keys(pattern=f"{IDPREFIX}*") + if isinstance(key, (str, bytes)) + ] return ListParsersResponse(keys=keylist) @@ -116,11 +110,9 @@ async def list_parsers( "/{parser_id}/info", response_model=ParserConfig, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"]) -async def info_parser( - cache: TRedisPlugin, - parser_id: str - ) -> ParserConfig: + tags=["parser"], +) +async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" await _validate_cache_key(cache, parser_id, "parser_id") cache_value = await cache.get(parser_id) @@ -129,16 +121,14 @@ async def info_parser( # Run `get` on parser @ROUTER.get( - "/{parser_id}", - response_model=GetParserResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"]) - + "/{parser_id}", + response_model=GetParserResponse, + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, + tags=["parser"], +) async def get_parser( - cache: TRedisPlugin, - parser_id: str, - session_id: Optional[str] = None - ) -> GetParserResponse: + cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None +) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" await _validate_cache_key(cache, parser_id, "parser_id") config = json.loads(await cache.get(parser_id)) @@ -147,17 +137,20 @@ async def get_parser( await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) populate_config_from_session(session_data, config) - + strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) logger.debug(str(strategy.parse_config.model_dump())) session_update = strategy.get() if session_update and session_id: - await _update_session(session_id=session_id, updated_session=session_update, redis=cache) + await _update_session( + session_id=session_id, updated_session=session_update, redis=cache + ) return GetParserResponse(**session_update) + @ROUTER.post( "/{parser_id}/initialize", response_model=InitializeParserResponse, @@ -179,13 +172,15 @@ async def initialize_parser( await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) populate_config_from_session(session_data, config) - + strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) logger.debug(str(strategy.parse_config.model_dump())) session_update = strategy.initialize() if session_update and session_id: - await _update_session(session_id=session_id, updated_session=session_update, redis=cache) + await _update_session( + session_id=session_id, updated_session=session_update, redis=cache + ) return InitializeParserResponse(**session_update) diff --git a/app/routers/session.py b/app/routers/session.py index e7ad6284..c66d88d6 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -95,8 +95,10 @@ async def delete_all_sessions( keylist = await cache.keys(pattern=f"{IDPREFIX}*") # Check if the keylist is empty if not keylist: - return DeleteAllSessionsResponse(number_of_deleted_sessions=0,message="No session keys found to delete.") - + return DeleteAllSessionsResponse( + number_of_deleted_sessions=0, message="No session keys found to delete." + ) + return DeleteAllSessionsResponse( number_of_deleted_sessions=await cache.delete(*keylist) ) diff --git a/composerun.py b/composerun.py index d0d0698c..50fa8368 100644 --- a/composerun.py +++ b/composerun.py @@ -1,16 +1,18 @@ import argparse import re + def extract_env_variables(file_path): - pattern = re.compile(r'\$\{([^}:]+)(?::-([^}]*))?\}') + pattern = re.compile(r"\$\{([^}:]+)(?::-([^}]*))?\}") env_vars = {} - with open(file_path, 'r') as file: + with open(file_path) as file: content = file.read() matches = pattern.findall(content) for match in matches: env_vars[match[0]] = match[1] return env_vars + def interactive_mode(env_vars): user_values = {} for var, default in env_vars.items(): @@ -18,25 +20,38 @@ def interactive_mode(env_vars): user_values[var] = user_input return user_values + def write_to_env_file(user_values, output_file): - with open(output_file, 'w') as file: + with open(output_file, "w") as file: for var, value in user_values.items(): file.write(f"{var}={value}\n") + def main(): - parser = argparse.ArgumentParser(description="Extract environment variables from a Docker Compose file.") - parser.add_argument('-f', '--inputfile', type=str, help='Path to the Docker Compose file', required=True) - parser.add_argument('-i', '--interactive', action='store_true', help='Interactive mode') + parser = argparse.ArgumentParser( + description="Extract environment variables from a Docker Compose file." + ) + parser.add_argument( + "-f", + "--inputfile", + type=str, + help="Path to the Docker Compose file", + required=True, + ) + parser.add_argument( + "-i", "--interactive", action="store_true", help="Interactive mode" + ) args = parser.parse_args() env_vars = extract_env_variables(args.inputfile) if args.interactive: user_values = interactive_mode(env_vars) - write_to_env_file(user_values, 'output.env') + write_to_env_file(user_values, "output.env") else: for var, default in env_vars.items(): print(f"{var}={default}") + if __name__ == "__main__": main() diff --git a/test.ttl b/test.ttl index 1c8516a4..54a882d6 100644 --- a/test.ttl +++ b/test.ttl @@ -37,4 +37,4 @@ ex:AnalysisMethods ex:PSTestRecovery a prov:Entity ; rdfs:label "Test of PS NP recovery from MilliQ and Novachem" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . \ No newline at end of file + prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py index 7c713f89..66b0f8af 100644 --- a/tests/routers/test_parser.py +++ b/tests/routers/test_parser.py @@ -5,8 +5,9 @@ if TYPE_CHECKING: from fastapi.testclient import TestClient + def test_create_parser(client: "TestClient") -> None: - """ Test creating a parser """ + """Test creating a parser""" response = client.post( "/parser/", json={ @@ -17,18 +18,20 @@ def test_create_parser(client: "TestClient") -> None: assert "parser_id" in response.json() assert response.status_code == 200 - + def test_delete_all_parsers(client: "TestClient") -> None: """Test deleting all parsers""" response = client.delete("/parser/") assert response.status_code == 200 + def test_list_parsers(client: "TestClient") -> None: """Test listing parsers""" response = client.get("/parser/") assert response.status_code == 200 assert "keys" in response.json() + def test_info_parser(client: "TestClient") -> None: """Test getting information about a parser""" response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa/info") @@ -36,10 +39,10 @@ def test_info_parser(client: "TestClient") -> None: assert "parserType" in response.json() assert "configuration" in response.json() + def test_get_parser(client: "TestClient") -> None: """Test getting and parsing data using a specified parser""" response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa") assert response.status_code == 200 assert "parserType" in response.json() assert "configuration" in response.json() - From 55cd80e7903980b3970256e328922779c7bd370f Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Tue, 20 Feb 2024 10:27:27 +0100 Subject: [PATCH 10/40] fix pre-commit --- app/models/parser.py | 1 + app/models/session.py | 1 + requirements.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/parser.py b/app/models/parser.py index e3c035fb..efbc4678 100644 --- a/app/models/parser.py +++ b/app/models/parser.py @@ -1,4 +1,5 @@ """Parser specific pydantic response models.""" + from typing import Annotated from uuid import uuid4 diff --git a/app/models/session.py b/app/models/session.py index 813ca829..955f2b3a 100644 --- a/app/models/session.py +++ b/app/models/session.py @@ -1,4 +1,5 @@ """Session-specific pydantic response models.""" + from typing import Annotated, Optional from uuid import uuid4 diff --git a/requirements.txt b/requirements.txt index f53daa97..2a1e7ec5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,8 @@ click==8.1.7 click-didyoumean==0.3.0 click-plugins==1.1.1 click-repl==0.3.0 -debugpy>=1.8.0 cryptography==42.0.2 +debugpy>=1.8.0 diskcache==5.6.3 et-xmlfile==1.1.0 exceptiongroup==1.2.0 From 454e17ba75620a213b46134b883450134d625d4d Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Thu, 22 Feb 2024 15:08:56 +0100 Subject: [PATCH 11/40] updated tests and new test for dataresource --- app/models/parser.py | 5 +- app/routers/parser.py | 8 +-- tests/conftest.py | 28 ++++++++-- tests/routers/test_parser.py | 8 +-- tests/routers/test_resource.py | 39 ++++++++++++++ tests/static/test_strategies/download.py | 67 +++++++++++++++++++++++- tests/static/test_strategies/parse.py | 53 ++++++++++++++++--- tests/static/test_strategies/resource.py | 32 +++-------- 8 files changed, 191 insertions(+), 49 deletions(-) create mode 100644 tests/routers/test_resource.py diff --git a/app/models/parser.py b/app/models/parser.py index efbc4678..5ca1cfac 100644 --- a/app/models/parser.py +++ b/app/models/parser.py @@ -1,6 +1,6 @@ """Parser specific pydantic response models.""" -from typing import Annotated +from typing import Annotated, Optional from uuid import uuid4 from oteapi.models import AttrDict @@ -51,7 +51,8 @@ class DeleteAllParsersResponse(AttrDict): """ number_of_deleted_parsers: Annotated[ - int, Field(description="The number of deleted parsers in the Redis cache.") + Optional[int], + Field(description="The number of deleted parsers in the Redis cache."), ] diff --git a/app/routers/parser.py b/app/routers/parser.py index 94a44441..936a6308 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -131,14 +131,14 @@ async def get_parser( ) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" await _validate_cache_key(cache, parser_id, "parser_id") - config = json.loads(await cache.get(parser_id)) + config = ParserConfig(**json.loads(await cache.get(parser_id))) session_data: "Optional[dict[str, Any]]" = None if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) populate_config_from_session(session_data, config) - strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) + strategy: "IParseStrategy" = create_strategy("parse", config) logger.debug(str(strategy.parse_config.model_dump())) session_update = strategy.get() @@ -166,14 +166,14 @@ async def initialize_parser( ) -> InitializeParserResponse: """Initialize parser.""" await _validate_cache_key(cache, parser_id, "parser_id") - config = json.loads(await cache.get(parser_id)) + config = ParserConfig(**json.loads(await cache.get(parser_id))) session_data: "Optional[dict[str, Any]]" = None if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = json.loads(await cache.get(session_id)) populate_config_from_session(session_data, config) - strategy: "IParseStrategy" = create_strategy("parse", ParserConfig(**config)) + strategy: "IParseStrategy" = create_strategy("parse", config) logger.debug(str(strategy.parse_config.model_dump())) session_update = strategy.initialize() diff --git a/tests/conftest.py b/tests/conftest.py index c360b8f6..116453c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,11 @@ async def exists(self, key: str) -> bool: """Mock `exists()` method.""" return key in self.obj.keys() + async def delete(self, *keys): + for key in keys: + if key in self.obj: # Use self.obj instead of self.cache + del self.obj[key] + def pytest_configure(config): """Method that runs before pytest collects tests so no modules are imported""" @@ -75,6 +80,12 @@ def test_data() -> "dict[str, str]": "functionType": "function/demo", "configuration": {}, }, + # dataresource + "dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e": { + "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "mediaType": "application/json", + "resourceType": "resource/demo", + }, # mapping "mapping-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e": { "mappingType": "mapping/demo", @@ -91,6 +102,15 @@ def test_data() -> "dict[str, str]": "name": "script/dummy", "configuration": {}, }, + # parser + "parser-f752c613-fde0-4d43-a7f6-c50f68642daa": { + "parserType": "parser/demo", + "entity": "http://example.com/entity", + "configuration": { + "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "mediaType": "application/json", + }, + }, }.items() } @@ -108,8 +128,8 @@ def load_test_strategies() -> None: test_strategies = [ { - "name": "tests.file", - "value": "tests.static.test_strategies.download:FileStrategy", + "name": "tests.https", + "value": "tests.static.test_strategies.download:HTTPSStrategy", "group": "oteapi.download", }, { @@ -128,12 +148,12 @@ def load_test_strategies() -> None: "group": "oteapi.mapping", }, { - "name": "tests.text/json", + "name": "tests.parser/demo", "value": "tests.static.test_strategies.parse:DemoJSONDataParseStrategy", "group": "oteapi.parse", }, { - "name": "tests.demo-access-service", + "name": "tests.resource/demo", "value": "tests.static.test_strategies.resource:DemoResourceStrategy", "group": "oteapi.resource", }, diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py index 66b0f8af..04da2ba3 100644 --- a/tests/routers/test_parser.py +++ b/tests/routers/test_parser.py @@ -12,7 +12,11 @@ def test_create_parser(client: "TestClient") -> None: "/parser/", json={ "parserType": "parser/demo", - "configuration": {}, + "entity": "http://example.com/entity", + "configuration": { + "downloadUrl": "https://filesamples.com/samples/code/json/sample2.json", + "mediaType": "application/json", + }, }, ) assert "parser_id" in response.json() @@ -44,5 +48,3 @@ def test_get_parser(client: "TestClient") -> None: """Test getting and parsing data using a specified parser""" response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa") assert response.status_code == 200 - assert "parserType" in response.json() - assert "configuration" in response.json() diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py new file mode 100644 index 00000000..fbb9167e --- /dev/null +++ b/tests/routers/test_resource.py @@ -0,0 +1,39 @@ +""" Test parser """ + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastapi.testclient import TestClient + + +def test_create_dataresource(client: "TestClient"): + response = client.post( + "/dataresource/", + json={ + "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "mediaType": "application/json", + "resourceType": "resource/demo", + }, + ) + assert response.status_code == 200 + + +def test_get_dataresource_info(client: "TestClient"): + response = client.get( + "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/info" + ) + assert response.status_code == 200 + + +def test_read_dataresource(client: "TestClient"): + response = client.get( + "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e" + ) + assert response.status_code == 200 + + +def test_initialize_dataresource(client: "TestClient"): + response = client.post( + "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/initialize" + ) + assert response.status_code == 200 diff --git a/tests/static/test_strategies/download.py b/tests/static/test_strategies/download.py index 26168ce3..54c22cfd 100644 --- a/tests/static/test_strategies/download.py +++ b/tests/static/test_strategies/download.py @@ -2,9 +2,10 @@ from typing import TYPE_CHECKING, Annotated, Optional +import requests from oteapi.datacache.datacache import DataCache -from oteapi.models.resourceconfig import ResourceConfig -from pydantic import BaseModel, Field +from oteapi.models import AttrDict, DataCacheConfig, ResourceConfig +from pydantic import AnyHttpUrl, BaseModel, Field from pydantic.dataclasses import dataclass if TYPE_CHECKING: @@ -64,3 +65,65 @@ def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": key = cache.add(handle.read()) return {"key": key} + + +class HTTPSConfig(AttrDict): + """HTTP(S)-specific Configuration Data Model.""" + + datacache_config: Optional[DataCacheConfig] = Field( + None, + description=( + "Configurations for the data cache for storing the downloaded file " + "content." + ), + ) + + +class HTTPSDemoConfig(ResourceConfig): + """HTTP(S) download strategy filter config.""" + + downloadUrl: AnyHttpUrl = Field( + ..., description="The HTTP(S) URL, which will be downloaded." + ) + configuration: HTTPSConfig = Field( + HTTPSConfig(), description="HTTP(S) download strategy-specific configuration." + ) + + +class HTTPDownloadContent(AttrDict): + """Class for returning values from Download HTTPS strategy.""" + + key: str = Field(..., description="Key to access the data in the cache.") + + +@dataclass +class HTTPSStrategy: + """Strategy for retrieving data via http. + + **Registers strategies**: + + - `("scheme", "http")` + - `("scheme", "https")` + + """ + + download_config: HTTPSDemoConfig + + def initialize(self) -> AttrDict: + """Initialize.""" + return AttrDict() + + def get(self) -> HTTPDownloadContent: + """Download via http/https and store on local cache.""" + cache = DataCache(self.download_config.configuration.datacache_config) + if cache.config.accessKey and cache.config.accessKey in cache: + key = cache.config.accessKey + else: + req = requests.get( + str(self.download_config.downloadUrl), + allow_redirects=True, + timeout=(3, 27), # timeout: (connect, read) in seconds + ) + key = cache.add(req.content) + + return HTTPDownloadContent(key=key) diff --git a/tests/static/test_strategies/parse.py b/tests/static/test_strategies/parse.py index 017107b5..f3c04ef3 100644 --- a/tests/static/test_strategies/parse.py +++ b/tests/static/test_strategies/parse.py @@ -2,22 +2,57 @@ # pylint: disable=unused-argument import json -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated, Literal, Optional +from oteapi.datacache import DataCache from oteapi.datacache.datacache import DataCache -from oteapi.models.resourceconfig import ResourceConfig -from oteapi.plugins.factories import create_strategy +from oteapi.models import AttrDict, DataCacheConfig, ParserConfig +from oteapi.plugins import create_strategy +from pydantic import Field from pydantic.dataclasses import dataclass +from pydantic.networks import Url, UrlConstraints +HostlessAnyUrl = Annotated[Url, UrlConstraints(host_required=False)] if TYPE_CHECKING: from typing import Any, Optional +class DEMOConfig(AttrDict): + """JSON parse-specific Configuration Data Model.""" + + downloadUrl: Optional[HostlessAnyUrl] = Field( + None, description="The HTTP(S) URL, which will be downloaded." + ) + mediaType: Optional[str] = Field( + "application/json", + description=("The media type"), + ) + datacache_config: Optional[DataCacheConfig] = Field( + None, + description=( + "Configurations for the data cache for storing the downloaded file " + "content." + ), + ) + + +class DEMOParserConfig(ParserConfig): + """JSON parse strategy filter config.""" + + parserType: Literal["parser/demo"] = Field( + "parser/demo", + description=ParserConfig.model_fields["parserType"].description, + ) + configuration: DEMOConfig = Field( + ..., description="JSON parse strategy-specific configuration." + ) + + @dataclass class DemoJSONDataParseStrategy: """Parse Strategy.""" - resource_config: ResourceConfig + parse_config: DEMOParserConfig def initialize( self, session: "Optional[dict[str, Any]]" = None @@ -25,14 +60,16 @@ def initialize( """Initialize""" del session # unused - del self.resource_config # unused + del self.parse_config # unused return {} - def parse(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def get(self) -> "dict[str, Any]": """Parse json.""" - downloader = create_strategy("download", self.resource_config) + downloader = create_strategy( + "download", self.parse_config.configuration.model_dump() + ) output = downloader.get() - cache = DataCache(self.resource_config.configuration) + cache = DataCache(self.parse_config.configuration.datacache_config) content = cache.get(output["key"]) if isinstance(content, dict): diff --git a/tests/static/test_strategies/resource.py b/tests/static/test_strategies/resource.py index 6646aa22..7180197b 100644 --- a/tests/static/test_strategies/resource.py +++ b/tests/static/test_strategies/resource.py @@ -1,22 +1,8 @@ """Demo resource strategy class.""" -from typing import TYPE_CHECKING, Annotated, Optional - -from oteapi.models import AttrDict from oteapi.models.resourceconfig import ResourceConfig -from oteapi.plugins.factories import create_strategy -from pydantic import Field from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any - - -class ResourceResult(AttrDict): - """Update session with the FilterResult model""" - - output: Annotated[Optional[str], Field(description="Optional result")] = None - @dataclass class DemoResourceStrategy: @@ -24,16 +10,10 @@ class DemoResourceStrategy: resource_config: ResourceConfig - def initialize(self, session: "Optional[dict[str, Any]]" = None) -> ResourceResult: - """Initialize""" - - del self.resource_config - del session # unused - return ResourceResult() + def initialize(self) -> dict: + """Initialize.""" + return {} - def get(self, session: "Optional[dict[str, Any]]" = None) -> ResourceResult: - """Manage mapping and return shared map""" - # Example of the plugin using the download strategy to fetch the data - download_strategy = create_strategy("download", self.resource_config) - read_output = download_strategy.get(session) - return ResourceResult(output=read_output) + def get(self) -> dict: + """resource distribution.""" + return dict(self.resource_config) From 8ecc76a034c7cc828633e18a1cbba90e6283fa03 Mon Sep 17 00:00:00 2001 From: Treesa Joseph <66374813+Treesarj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:51:13 +0100 Subject: [PATCH 12/40] Update app/models/session.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- app/models/session.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/models/session.py b/app/models/session.py index 955f2b3a..d9932f6a 100644 --- a/app/models/session.py +++ b/app/models/session.py @@ -57,10 +57,12 @@ class DeleteAllSessionsResponse(Session): number_of_deleted_sessions: Annotated[ int, Field(description="The number of deleted sessions in the Redis cache.") ] - message: Optional[str] = Field( - "All session keys deleted.", - description="Optional message indicating the result of the operation.", - ) + message: Annotated[ + Optional[str], + Field( + description="Optional message indicating the result of the operation.", + ), + ] = "All session keys deleted." class DeleteSessionResponse(Session): From cedc8ef3229f9d545e61cd624e2441247d7af234 Mon Sep 17 00:00:00 2001 From: Treesa Joseph <66374813+Treesarj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:59:24 +0100 Subject: [PATCH 13/40] Update app/routers/session.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- app/routers/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/session.py b/app/routers/session.py index 77dd6a80..73b7539f 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -67,7 +67,7 @@ async def list_sessions( cache: TRedisPlugin, ) -> ListSessionsResponse: """Get all session keys""" - keylist = [] + keylist: list[str, bytes] = [] for key in await cache.keys(pattern=f"{IDPREFIX}*"): if not isinstance(key, (str, bytes)): raise TypeError( From 9d5280255055baa9fd6514811a3e2d1b13d6780b Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Thu, 22 Feb 2024 16:23:01 +0100 Subject: [PATCH 14/40] resolve comments --- app/routers/datafilter.py | 5 +-- app/routers/dataresource.py | 8 ++--- app/routers/function.py | 5 +-- app/routers/mapping.py | 5 +-- app/routers/parser.py | 43 +++++++++++++++++--------- app/routers/redisadmin.py | 4 +-- app/routers/session.py | 8 +---- app/routers/transformation.py | 7 +---- composerun.py | 57 ----------------------------------- 9 files changed, 38 insertions(+), 104 deletions(-) delete mode 100644 composerun.py diff --git a/app/routers/datafilter.py b/app/routers/datafilter.py index 3d7010a2..5a8d1d00 100644 --- a/app/routers/datafilter.py +++ b/app/routers/datafilter.py @@ -22,7 +22,7 @@ from oteapi.interfaces import IFilterStrategy -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["datafilter"]) @ROUTER.post( @@ -31,7 +31,6 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["datafilter"], ) async def create_filter( cache: TRedisPlugin, @@ -62,7 +61,6 @@ async def create_filter( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["datafilter"], ) async def get_filter( cache: TRedisPlugin, @@ -111,7 +109,6 @@ async def get_filter( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["datafilter"], ) async def initialize_filter( cache: TRedisPlugin, diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 7e516282..b831ed2e 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -3,7 +3,7 @@ import json from typing import TYPE_CHECKING, Optional -from fastapi import APIRouter, Request, status +from fastapi import APIRouter, status from oteapi.models import ResourceConfig from oteapi.plugins import create_strategy @@ -25,7 +25,7 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["dataresource"]) @ROUTER.post( @@ -34,7 +34,6 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["dataresource"], ) async def create_dataresource( cache: TRedisPlugin, @@ -81,7 +80,6 @@ async def create_dataresource( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["dataresource"], ) async def info_dataresource( cache: TRedisPlugin, @@ -108,7 +106,6 @@ async def info_dataresource( status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, }, - tags=["dataresource"], ) async def read_dataresource( cache: TRedisPlugin, @@ -164,7 +161,6 @@ async def read_dataresource( status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, }, - tags=["dataresource"], ) async def initialize_dataresource( cache: TRedisPlugin, diff --git a/app/routers/function.py b/app/routers/function.py index 0053575f..0b87c9d3 100644 --- a/app/routers/function.py +++ b/app/routers/function.py @@ -20,14 +20,13 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["function"]) @ROUTER.post( "/", response_model=CreateFunctionResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["function"], ) async def create_function( cache: TRedisPlugin, @@ -63,7 +62,6 @@ async def create_function( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["function"], ) async def get_function( cache: TRedisPlugin, @@ -112,7 +110,6 @@ async def get_function( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["function"], ) async def initialize_function( cache: TRedisPlugin, diff --git a/app/routers/mapping.py b/app/routers/mapping.py index f35d1379..6ac8b621 100644 --- a/app/routers/mapping.py +++ b/app/routers/mapping.py @@ -20,14 +20,13 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["mapping"]) @ROUTER.post( "/", response_model=CreateMappingResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["mapping"], ) async def create_mapping( cache: TRedisPlugin, @@ -61,7 +60,6 @@ async def create_mapping( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["mapping"], ) async def get_mapping( cache: TRedisPlugin, @@ -110,7 +108,6 @@ async def get_mapping( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["mapping"], ) async def initialize_mapping( cache: TRedisPlugin, diff --git a/app/routers/parser.py b/app/routers/parser.py index 936a6308..c5f8090a 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -1,3 +1,5 @@ +"""Parser""" + import json import logging from typing import TYPE_CHECKING, Any, Optional @@ -25,7 +27,7 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger("app.routers.parser") -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["parser"]) async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: @@ -46,7 +48,6 @@ async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> N "/", response_model=CreateParserResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"], ) async def create_parser( cache: TRedisPlugin, config: ParserConfig, session_id: Optional[str] = None @@ -72,7 +73,6 @@ async def create_parser( "/", response_model=DeleteAllParsersResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"], ) async def delete_all_parsers( cache: TRedisPlugin, @@ -93,7 +93,6 @@ async def delete_all_parsers( "/", response_model=ListParsersResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"], ) async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: """Retrieve all parser IDs from cache.""" @@ -110,13 +109,15 @@ async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: "/{parser_id}/info", response_model=ParserConfig, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"], ) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" await _validate_cache_key(cache, parser_id, "parser_id") cache_value = await cache.get(parser_id) - return ParserConfig(**json.loads(cache_value)) + if cache_value is None: + raise ValueError("Cache value is None") + config_dict = json.loads(cache_value) + return ParserConfig(**config_dict) # Run `get` on parser @@ -124,19 +125,24 @@ async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: "/{parser_id}", response_model=GetParserResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["parser"], ) async def get_parser( cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None ) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" await _validate_cache_key(cache, parser_id, "parser_id") - config = ParserConfig(**json.loads(await cache.get(parser_id))) + cache_value = await cache.get(parser_id) + if cache_value is None: + raise ValueError("Cache value is None") + config_dict = json.loads(cache_value) + config = ParserConfig(**config_dict) session_data: "Optional[dict[str, Any]]" = None if session_id: await _validate_cache_key(cache, session_id, "session_id") - session_data = json.loads(await cache.get(session_id)) - populate_config_from_session(session_data, config) + session_data = await cache.get(session_id) + if session_data is None: + raise ValueError("Session data is None") + populate_config_from_session(json.loads(session_data), config) strategy: "IParseStrategy" = create_strategy("parse", config) @@ -157,7 +163,6 @@ async def get_parser( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["parser"], ) async def initialize_parser( cache: TRedisPlugin, @@ -166,12 +171,22 @@ async def initialize_parser( ) -> InitializeParserResponse: """Initialize parser.""" await _validate_cache_key(cache, parser_id, "parser_id") - config = ParserConfig(**json.loads(await cache.get(parser_id))) + cache_value = await cache.get(parser_id) + if cache_value is None: + raise ValueError("Cache value is None") + config_dict = json.loads(cache_value) + cache_value = await cache.get(parser_id) + if cache_value is None: + raise ValueError("Cache value is None") + config_dict = json.loads(cache_value) + config = ParserConfig(**config_dict) session_data: "Optional[dict[str, Any]]" = None if session_id: await _validate_cache_key(cache, session_id, "session_id") - session_data = json.loads(await cache.get(session_id)) - populate_config_from_session(session_data, config) + session_data = await cache.get(session_id) + if session_data is None: + raise ValueError("Session data is None") + populate_config_from_session(json.loads(session_data), config) strategy: "IParseStrategy" = create_strategy("parse", config) diff --git a/app/routers/redisadmin.py b/app/routers/redisadmin.py index 922ca57b..62ea4d2b 100644 --- a/app/routers/redisadmin.py +++ b/app/routers/redisadmin.py @@ -8,10 +8,10 @@ from app.models.error import httpexception_404_item_id_does_not_exist from app.redis_cache import TRedisPlugin -ROUTER = APIRouter(prefix="/redis") +ROUTER = APIRouter(prefix="/redis", tags=["admin"]) -@ROUTER.get("/{key}", include_in_schema=False, tags=["admin"]) +@ROUTER.get("/{key}", include_in_schema=False) async def get_key( cache: TRedisPlugin, key: str, diff --git a/app/routers/session.py b/app/routers/session.py index 77dd6a80..54180d1a 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -25,14 +25,13 @@ class SessionUpdate(AttrDict): """Session Update Data Model for returning values.""" -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["session"]) @ROUTER.post( "/", response_model=CreateSessionResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["session"], ) async def create_session( cache: TRedisPlugin, @@ -61,7 +60,6 @@ async def create_session( "/", response_model=ListSessionsResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["session"], ) async def list_sessions( cache: TRedisPlugin, @@ -82,7 +80,6 @@ async def list_sessions( "/", response_model=DeleteAllSessionsResponse, responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, - tags=["session"], ) async def delete_all_sessions( cache: TRedisPlugin, @@ -167,7 +164,6 @@ async def _update_session_list_item( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["session"], ) async def update_session( cache: TRedisPlugin, @@ -196,7 +192,6 @@ async def update_session( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["session"], ) async def get_session( cache: TRedisPlugin, @@ -222,7 +217,6 @@ async def get_session( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["session"], ) async def delete_session( cache: TRedisPlugin, diff --git a/app/routers/transformation.py b/app/routers/transformation.py index e25d5bd9..ade87548 100644 --- a/app/routers/transformation.py +++ b/app/routers/transformation.py @@ -23,7 +23,7 @@ from oteapi.interfaces import ITransformationStrategy -ROUTER = APIRouter(prefix=f"/{IDPREFIX}") +ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["transformation"]) @ROUTER.post( @@ -32,7 +32,6 @@ responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["transformation"], ) async def create_transformation( cache: TRedisPlugin, @@ -68,7 +67,6 @@ async def create_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["transformation"], ) async def get_transformation_status( cache: TRedisPlugin, @@ -99,7 +97,6 @@ async def get_transformation_status( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["transformation"], ) async def get_transformation( cache: TRedisPlugin, @@ -150,7 +147,6 @@ async def get_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["transformation"], ) async def execute_transformation( cache: TRedisPlugin, @@ -202,7 +198,6 @@ async def execute_transformation( responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, }, - tags=["transformation"], ) async def initialize_transformation( cache: TRedisPlugin, diff --git a/composerun.py b/composerun.py deleted file mode 100644 index 50fa8368..00000000 --- a/composerun.py +++ /dev/null @@ -1,57 +0,0 @@ -import argparse -import re - - -def extract_env_variables(file_path): - pattern = re.compile(r"\$\{([^}:]+)(?::-([^}]*))?\}") - env_vars = {} - with open(file_path) as file: - content = file.read() - matches = pattern.findall(content) - for match in matches: - env_vars[match[0]] = match[1] - return env_vars - - -def interactive_mode(env_vars): - user_values = {} - for var, default in env_vars.items(): - user_input = input(f"{var} [{default}]: ") or default - user_values[var] = user_input - return user_values - - -def write_to_env_file(user_values, output_file): - with open(output_file, "w") as file: - for var, value in user_values.items(): - file.write(f"{var}={value}\n") - - -def main(): - parser = argparse.ArgumentParser( - description="Extract environment variables from a Docker Compose file." - ) - parser.add_argument( - "-f", - "--inputfile", - type=str, - help="Path to the Docker Compose file", - required=True, - ) - parser.add_argument( - "-i", "--interactive", action="store_true", help="Interactive mode" - ) - args = parser.parse_args() - - env_vars = extract_env_variables(args.inputfile) - - if args.interactive: - user_values = interactive_mode(env_vars) - write_to_env_file(user_values, "output.env") - else: - for var, default in env_vars.items(): - print(f"{var}={default}") - - -if __name__ == "__main__": - main() From 1a45ab70fdcfec3d4318c926fc5f778eb0f87b2f Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Fri, 23 Feb 2024 10:12:54 +0100 Subject: [PATCH 15/40] fix test configuration --- entrypoint.sh | 3 ++- tests/conftest.py | 5 +++-- tests/routers/test_resource.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index d1b86517..e911019a 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,4 +33,5 @@ else echo "No extra plugin packages provided. Specify 'OTEAPI_PLUGIN_PACKAGES' to specify plugin packages." fi -python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" +hypercorn --bind 0.0.0.0:8080 asgi:app "$@" +# python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" diff --git a/tests/conftest.py b/tests/conftest.py index 116453c1..b5fc0f3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,6 +40,7 @@ async def exists(self, key: str) -> bool: return key in self.obj.keys() async def delete(self, *keys): + """Delete Keys""" for key in keys: if key in self.obj: # Use self.obj instead of self.cache del self.obj[key] @@ -82,7 +83,7 @@ def test_data() -> "dict[str, str]": }, # dataresource "dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e": { - "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", "mediaType": "application/json", "resourceType": "resource/demo", }, @@ -107,7 +108,7 @@ def test_data() -> "dict[str, str]": "parserType": "parser/demo", "entity": "http://example.com/entity", "configuration": { - "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", "mediaType": "application/json", }, }, diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py index fbb9167e..0d112bce 100644 --- a/tests/routers/test_resource.py +++ b/tests/routers/test_resource.py @@ -10,7 +10,7 @@ def test_create_dataresource(client: "TestClient"): response = client.post( "/dataresource/", json={ - "downloadUrl": "https://raw.githubusercontent.com/EMMC-ASBL/oteapi-core/master/tests/static/sample2.json", + "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", "mediaType": "application/json", "resourceType": "resource/demo", }, From d6d6a88cb8645c6ae79f0cb28d32e9220a7229a4 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Fri, 23 Feb 2024 11:15:10 +0100 Subject: [PATCH 16/40] fix tests --- app/routers/parser.py | 4 ---- app/routers/session.py | 2 +- tests/conftest.py | 5 ++++- tests/static/test_strategies/parse.py | 7 +------ 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/app/routers/parser.py b/app/routers/parser.py index c5f8090a..5c7bcdb5 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -175,10 +175,6 @@ async def initialize_parser( if cache_value is None: raise ValueError("Cache value is None") config_dict = json.loads(cache_value) - cache_value = await cache.get(parser_id) - if cache_value is None: - raise ValueError("Cache value is None") - config_dict = json.loads(cache_value) config = ParserConfig(**config_dict) session_data: "Optional[dict[str, Any]]" = None if session_id: diff --git a/app/routers/session.py b/app/routers/session.py index b093c3d1..5fe769b7 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -65,7 +65,7 @@ async def list_sessions( cache: TRedisPlugin, ) -> ListSessionsResponse: """Get all session keys""" - keylist: list[str, bytes] = [] + keylist: list = [] for key in await cache.keys(pattern=f"{IDPREFIX}*"): if not isinstance(key, (str, bytes)): raise TypeError( diff --git a/tests/conftest.py b/tests/conftest.py index b5fc0f3a..c2e4dc75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,7 +108,10 @@ def test_data() -> "dict[str, str]": "parserType": "parser/demo", "entity": "http://example.com/entity", "configuration": { - "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", + "downloadUrl": ( + "https://raw.githubusercontent.com/EMMC-ASBL/" + "oteapi-core/master/tests/static/sample2.json" + ), "mediaType": "application/json", }, }, diff --git a/tests/static/test_strategies/parse.py b/tests/static/test_strategies/parse.py index f3c04ef3..cfeda32d 100644 --- a/tests/static/test_strategies/parse.py +++ b/tests/static/test_strategies/parse.py @@ -54,13 +54,8 @@ class DemoJSONDataParseStrategy: parse_config: DEMOParserConfig - def initialize( - self, session: "Optional[dict[str, Any]]" = None - ) -> "dict[str, Any]": + def initialize(self) -> "dict[str, Any]": """Initialize""" - - del session # unused - del self.parse_config # unused return {} def get(self) -> "dict[str, Any]": From ba5c771c3aea555f4efeb2db9f593da32cc905c7 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Fri, 23 Feb 2024 12:39:51 +0100 Subject: [PATCH 17/40] requested changes --- Dockerfile | 7 +------ app/routers/dataresource.py | 28 +++++++++++++++------------- app/routers/session.py | 8 ++------ entrypoint.sh | 7 +++++-- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 55a23957..8e973a9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,12 +28,7 @@ COPY .dev/requirements_dev.txt .pre-commit-config.yaml pyproject.toml ./ # Run static security check, linters, and pytest with code coverage RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt -# Ignore ID 44715 for now. -# See this NumPy issue for more information: https://github.com/numpy/numpy/issues/19038 -# && pre-commit run --all-files -# && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715 -# && pytest --cov app - +ENV DEV_ENV=1 # Run app with reload option EXPOSE 8080 CMD if [ "${PATH_TO_OTEAPI_CORE}" != "/dev/null" ] && [ -n "${PATH_TO_OTEAPI_CORE}" ]; then \ diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index b831ed2e..0e490011 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -3,9 +3,10 @@ import json from typing import TYPE_CHECKING, Optional -from fastapi import APIRouter, status +from fastapi import APIRouter, Request, status from oteapi.models import ResourceConfig from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session from app.models.dataresource import ( IDPREFIX, @@ -20,6 +21,7 @@ httpexception_422_resource_id_is_unprocessable, ) from app.redis_cache import TRedisPlugin +from app.routers.parser import _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -37,6 +39,7 @@ ) async def create_dataresource( cache: TRedisPlugin, + request: Request, config: ResourceConfig, session_id: Optional[str] = None, ) -> CreateResourceResponse: @@ -55,7 +58,7 @@ async def create_dataresource( """ new_resource = CreateResourceResponse() - # config.token = request.headers.get("Authorization") or config.token + config.token = request.headers.get("Authorization") or config.token resource_config = config.model_dump_json() @@ -130,12 +133,11 @@ async def read_dataresource( config = ResourceConfig(**json.loads(cache_value)) if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + await _validate_cache_key(cache, session_id, "session_id") + session_data = await cache.get(session_id) + if session_data is None: + raise ValueError("Session data is None") + populate_config_from_session(json.loads(session_data), config) if not config.resourceType: raise httpexception_422_resource_id_is_unprocessable(resource_id) @@ -182,12 +184,12 @@ async def initialize_dataresource( config = ResourceConfig(**json.loads(cache_value)) if session_id: + await _validate_cache_key(cache, session_id, "session_id") cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + session_data = await cache.get(session_id) + if session_data is None: + raise ValueError("Session data is None") + populate_config_from_session(json.loads(session_data), config) if not config.resourceType: raise httpexception_422_resource_id_is_unprocessable(resource_id) diff --git a/app/routers/session.py b/app/routers/session.py index 5fe769b7..b6e2708a 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -21,10 +21,6 @@ from typing import Any, Union -class SessionUpdate(AttrDict): - """Session Update Data Model for returning values.""" - - ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["session"]) @@ -120,7 +116,7 @@ async def _get_session( async def _update_session( session_id: str, - updated_session: "Union[SessionUpdate, dict[str, Any]]", + updated_session: "Union[AttrDict, dict[str, Any]]", redis: TRedisPlugin, ) -> Session: """Update an existing session (to be called internally).""" @@ -168,7 +164,7 @@ async def _update_session_list_item( async def update_session( cache: TRedisPlugin, session_id: str, - updated_session: SessionUpdate, + updated_session: AttrDict, ) -> Session: """Update session object.""" if not await cache.exists(session_id): diff --git a/entrypoint.sh b/entrypoint.sh index e911019a..d7a42b00 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,5 +33,8 @@ else echo "No extra plugin packages provided. Specify 'OTEAPI_PLUGIN_PACKAGES' to specify plugin packages." fi -hypercorn --bind 0.0.0.0:8080 asgi:app "$@" -# python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" +if [ "$DEV_ENV" = "1" ]; then + python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" +else + hypercorn --bind 0.0.0.0:8080 asgi:app "$@" +fi From 65f9eff8812df09c23ac1d714adee81361648b61 Mon Sep 17 00:00:00 2001 From: Treesa Joseph <66374813+Treesarj@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:44:56 +0100 Subject: [PATCH 18/40] Update tests/static/test_strategies/resource.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- tests/static/test_strategies/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/static/test_strategies/resource.py b/tests/static/test_strategies/resource.py index 7180197b..7179d069 100644 --- a/tests/static/test_strategies/resource.py +++ b/tests/static/test_strategies/resource.py @@ -16,4 +16,4 @@ def initialize(self) -> dict: def get(self) -> dict: """resource distribution.""" - return dict(self.resource_config) + return self.resource_config.model_dump() From 5541e9393be75dea1c3c6cc329a57c7ce21177bc Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 26 Feb 2024 11:50:57 +0100 Subject: [PATCH 19/40] remove test ttl file and debug.py from requirements.txt --- requirements.txt | 1 - test.ttl | 40 ---------------------------------------- 2 files changed, 41 deletions(-) delete mode 100644 test.ttl diff --git a/requirements.txt b/requirements.txt index 8acb772e..b8a389f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ click-didyoumean==0.3.0 click-plugins==1.1.1 click-repl==0.3.0 cryptography==42.0.3 -debugpy>=1.8.0 diskcache==5.6.3 et-xmlfile==1.1.0 exceptiongroup==1.2.0 diff --git a/test.ttl b/test.ttl deleted file mode 100644 index 54a882d6..00000000 --- a/test.ttl +++ /dev/null @@ -1,40 +0,0 @@ -@prefix prov: . -@prefix rdfs: . -@prefix ex: . - -ex:InitialMethodDevelopmentOutcomes - a prov:Activity ; - rdfs:label "Initial method development outcomes at SINTEF" ; - prov:qualifiedAssociation [ - a prov:Association ; - prov:agent ex:SINTEFOcean ; - ] . - -ex:SINTEFOcean - a prov:Agent ; - rdfs:label "SINTEF Ocean" . - -ex:PolymerSolubility - a prov:Entity ; - rdfs:label "Solubility of polymers" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . - -ex:PyGCMSMethodTestingOutcomes - a prov:Entity ; - rdfs:label "Py-GCMS method testing and initial outcome and observations" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . - -ex:CalibrationCurvesAndLOD - a prov:Entity ; - rdfs:label "Calibration curves and LOD" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . - -ex:AnalysisMethods - a prov:Entity ; - rdfs:label "Analysis methods" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . - -ex:PSTestRecovery - a prov:Entity ; - rdfs:label "Test of PS NP recovery from MilliQ and Novachem" ; - prov:wasGeneratedBy ex:InitialMethodDevelopmentOutcomes . From aee3cf5c7076a5ee8cb19f7f60ec6e646ccf378e Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 26 Feb 2024 12:29:13 +0100 Subject: [PATCH 20/40] requested changes --- app/routers/datafilter.py | 30 ++++----------- app/routers/dataresource.py | 34 +++++------------ app/routers/function.py | 28 ++++---------- app/routers/mapping.py | 28 ++++---------- app/routers/parser.py | 48 +++++++----------------- app/routers/session.py | 53 ++++++++------------------- app/routers/transformation.py | 42 +++++---------------- tests/static/test_strategies/parse.py | 6 +-- 8 files changed, 73 insertions(+), 196 deletions(-) diff --git a/app/routers/datafilter.py b/app/routers/datafilter.py index 5a8d1d00..e054bd08 100644 --- a/app/routers/datafilter.py +++ b/app/routers/datafilter.py @@ -22,16 +22,14 @@ from oteapi.interfaces import IFilterStrategy -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["datafilter"]) +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["datafilter"], + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, +) -@ROUTER.post( - "/", - response_model=CreateFilterResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/", response_model=CreateFilterResponse) async def create_filter( cache: TRedisPlugin, config: FilterConfig, @@ -55,13 +53,7 @@ async def create_filter( return new_filter -@ROUTER.get( - "/{filter_id}", - response_model=GetFilterResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{filter_id}", response_model=GetFilterResponse) async def get_filter( cache: TRedisPlugin, filter_id: str, @@ -103,13 +95,7 @@ async def get_filter( return GetFilterResponse(**session_update) -@ROUTER.post( - "/{filter_id}/initialize", - response_model=InitializeFilterResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/{filter_id}/initialize", response_model=InitializeFilterResponse) async def initialize_filter( cache: TRedisPlugin, filter_id: str, diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 0e490011..c2ba0f1d 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -27,16 +27,17 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["dataresource"]) - - -@ROUTER.post( - "/", - response_model=CreateResourceResponse, +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["dataresource"], responses={ status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, + status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, }, ) + + +@ROUTER.post("/", response_model=CreateResourceResponse) async def create_dataresource( cache: TRedisPlugin, request: Request, @@ -77,13 +78,7 @@ async def create_dataresource( return new_resource -@ROUTER.get( - "/{resource_id}/info", - response_model=ResourceConfig, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{resource_id}/info", response_model=ResourceConfig) async def info_dataresource( cache: TRedisPlugin, resource_id: str, @@ -105,10 +100,6 @@ async def info_dataresource( "/{resource_id}", response_model=GetResourceResponse, response_model_exclude_unset=True, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, - }, ) async def read_dataresource( cache: TRedisPlugin, @@ -156,14 +147,7 @@ async def read_dataresource( return GetResourceResponse(**session_update) -@ROUTER.post( - "/{resource_id}/initialize", - response_model=InitializeResourceResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError}, - }, -) +@ROUTER.post("/{resource_id}/initialize", response_model=InitializeResourceResponse) async def initialize_dataresource( cache: TRedisPlugin, resource_id: str, diff --git a/app/routers/function.py b/app/routers/function.py index 0b87c9d3..9fb4f9cb 100644 --- a/app/routers/function.py +++ b/app/routers/function.py @@ -20,14 +20,14 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["function"]) - - -@ROUTER.post( - "/", - response_model=CreateFunctionResponse, +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["function"], responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, ) + + +@ROUTER.post("/", response_model=CreateFunctionResponse) async def create_function( cache: TRedisPlugin, config: FunctionConfig, @@ -56,13 +56,7 @@ async def create_function( return new_function -@ROUTER.get( - "/{function_id}", - response_model=GetFunctionResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{function_id}", response_model=GetFunctionResponse) async def get_function( cache: TRedisPlugin, function_id: str, @@ -104,13 +98,7 @@ async def get_function( return GetFunctionResponse(**session_update) -@ROUTER.post( - "/{function_id}/initialize", - response_model=InitializeFunctionResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/{function_id}/initialize", response_model=InitializeFunctionResponse) async def initialize_function( cache: TRedisPlugin, function_id: str, diff --git a/app/routers/mapping.py b/app/routers/mapping.py index 6ac8b621..3de8e58a 100644 --- a/app/routers/mapping.py +++ b/app/routers/mapping.py @@ -20,14 +20,14 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["mapping"]) - - -@ROUTER.post( - "/", - response_model=CreateMappingResponse, +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["mapping"], responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, ) + + +@ROUTER.post("/", response_model=CreateMappingResponse) async def create_mapping( cache: TRedisPlugin, config: MappingConfig, @@ -54,13 +54,7 @@ async def create_mapping( return new_mapping -@ROUTER.get( - "/{mapping_id}", - response_model=GetMappingResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{mapping_id}", response_model=GetMappingResponse) async def get_mapping( cache: TRedisPlugin, mapping_id: str, @@ -102,13 +96,7 @@ async def get_mapping( return GetMappingResponse(**session_update) -@ROUTER.post( - "/{mapping_id}/initialize", - response_model=InitializeMappingResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/{mapping_id}/initialize", response_model=InitializeMappingResponse) async def initialize_mapping( cache: TRedisPlugin, mapping_id: str, diff --git a/app/routers/parser.py b/app/routers/parser.py index 5c7bcdb5..7ae5f6c0 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -27,7 +27,11 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger("app.routers.parser") -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["parser"]) +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["parser"], + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, +) async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: @@ -44,11 +48,7 @@ async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> N # Create parser -@ROUTER.post( - "/", - response_model=CreateParserResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.post("/", response_model=CreateParserResponse) async def create_parser( cache: TRedisPlugin, config: ParserConfig, session_id: Optional[str] = None ) -> CreateParserResponse: @@ -69,11 +69,7 @@ async def create_parser( return new_parser -@ROUTER.delete( - "/", - response_model=DeleteAllParsersResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.delete("/", response_model=DeleteAllParsersResponse) async def delete_all_parsers( cache: TRedisPlugin, ) -> DeleteAllParsersResponse: @@ -89,11 +85,7 @@ async def delete_all_parsers( # List parsers -@ROUTER.get( - "/", - response_model=ListParsersResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.get("/", response_model=ListParsersResponse) async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: """Retrieve all parser IDs from cache.""" keylist = [ @@ -105,11 +97,7 @@ async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: # Get parser info -@ROUTER.get( - "/{parser_id}/info", - response_model=ParserConfig, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.get("/{parser_id}/info", response_model=ParserConfig) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" await _validate_cache_key(cache, parser_id, "parser_id") @@ -121,11 +109,7 @@ async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: # Run `get` on parser -@ROUTER.get( - "/{parser_id}", - response_model=GetParserResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.get("/{parser_id}", response_model=GetParserResponse) async def get_parser( cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None ) -> GetParserResponse: @@ -136,7 +120,7 @@ async def get_parser( raise ValueError("Cache value is None") config_dict = json.loads(cache_value) config = ParserConfig(**config_dict) - session_data: "Optional[dict[str, Any]]" = None + if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = await cache.get(session_id) @@ -157,13 +141,7 @@ async def get_parser( return GetParserResponse(**session_update) -@ROUTER.post( - "/{parser_id}/initialize", - response_model=InitializeParserResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/{parser_id}/initialize", response_model=InitializeParserResponse) async def initialize_parser( cache: TRedisPlugin, parser_id: str, @@ -176,7 +154,7 @@ async def initialize_parser( raise ValueError("Cache value is None") config_dict = json.loads(cache_value) config = ParserConfig(**config_dict) - session_data: "Optional[dict[str, Any]]" = None + if session_id: await _validate_cache_key(cache, session_id, "session_id") session_data = await cache.get(session_id) diff --git a/app/routers/session.py b/app/routers/session.py index b6e2708a..60c1b02d 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -21,14 +21,14 @@ from typing import Any, Union -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["session"]) - - -@ROUTER.post( - "/", - response_model=CreateSessionResponse, +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["session"], responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, ) + + +@ROUTER.post("/", response_model=CreateSessionResponse) async def create_session( cache: TRedisPlugin, session: Session, @@ -52,31 +52,26 @@ async def create_session( return new_session -@ROUTER.get( - "/", - response_model=ListSessionsResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.get("/", response_model=ListSessionsResponse) async def list_sessions( cache: TRedisPlugin, ) -> ListSessionsResponse: """Get all session keys""" - keylist: list = [] + keylist: list[Union[str, bytes]] = [] for key in await cache.keys(pattern=f"{IDPREFIX}*"): if not isinstance(key, (str, bytes)): raise TypeError( "Found a key that is not stored as bytes (stored as type " f"{type(key)!r})." ) + if isinstance(key, bytes): + key = key.decode(encoding="utf-8") keylist.append(key) + return ListSessionsResponse(keys=keylist) -@ROUTER.delete( - "/", - response_model=DeleteAllSessionsResponse, - responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, -) +@ROUTER.delete("/", response_model=DeleteAllSessionsResponse) async def delete_all_sessions( cache: TRedisPlugin, ) -> DeleteAllSessionsResponse: @@ -154,13 +149,7 @@ async def _update_session_list_item( return session -@ROUTER.put( - "/{session_id}", - response_model=Session, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.put("/{session_id}", response_model=Session) async def update_session( cache: TRedisPlugin, session_id: str, @@ -182,13 +171,7 @@ async def update_session( return session -@ROUTER.get( - "/{session_id}", - response_model=Session, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{session_id}", response_model=Session) async def get_session( cache: TRedisPlugin, session_id: str, @@ -207,13 +190,7 @@ async def get_session( return Session(**json.loads(cache_value)) -@ROUTER.delete( - "/{session_id}", - response_model=DeleteSessionResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.delete("/{session_id}", response_model=DeleteSessionResponse) async def delete_session( cache: TRedisPlugin, session_id: str, diff --git a/app/routers/transformation.py b/app/routers/transformation.py index ade87548..53cda7f7 100644 --- a/app/routers/transformation.py +++ b/app/routers/transformation.py @@ -23,16 +23,14 @@ from oteapi.interfaces import ITransformationStrategy -ROUTER = APIRouter(prefix=f"/{IDPREFIX}", tags=["transformation"]) +ROUTER = APIRouter( + prefix=f"/{IDPREFIX}", + tags=["transformation"], + responses={status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}}, +) -@ROUTER.post( - "/", - response_model=CreateTransformationResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.post("/", response_model=CreateTransformationResponse) async def create_transformation( cache: TRedisPlugin, config: TransformationConfig, @@ -61,13 +59,7 @@ async def create_transformation( return new_transformation -@ROUTER.get( - "/{transformation_id}/status", - response_model=TransformationStatus, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{transformation_id}/status", response_model=TransformationStatus) async def get_transformation_status( cache: TRedisPlugin, transformation_id: str, @@ -91,13 +83,7 @@ async def get_transformation_status( return strategy.status(task_id=task_id) -@ROUTER.get( - "/{transformation_id}", - response_model=GetTransformationResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, -) +@ROUTER.get("/{transformation_id}", response_model=GetTransformationResponse) async def get_transformation( cache: TRedisPlugin, transformation_id: str, @@ -142,11 +128,7 @@ async def get_transformation( @ROUTER.post( - "/{transformation_id}/execute", - response_model=ExecuteTransformationResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, + "/{transformation_id}/execute", response_model=ExecuteTransformationResponse ) async def execute_transformation( cache: TRedisPlugin, @@ -193,11 +175,7 @@ async def execute_transformation( @ROUTER.post( - "/{transformation_id}/initialize", - response_model=InitializeTransformationResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError}, - }, + "/{transformation_id}/initialize", response_model=InitializeTransformationResponse ) async def initialize_transformation( cache: TRedisPlugin, diff --git a/tests/static/test_strategies/parse.py b/tests/static/test_strategies/parse.py index cfeda32d..4e326275 100644 --- a/tests/static/test_strategies/parse.py +++ b/tests/static/test_strategies/parse.py @@ -2,17 +2,15 @@ # pylint: disable=unused-argument import json -from typing import TYPE_CHECKING, Annotated, Literal, Optional +from typing import TYPE_CHECKING, Literal, Optional from oteapi.datacache import DataCache from oteapi.datacache.datacache import DataCache -from oteapi.models import AttrDict, DataCacheConfig, ParserConfig +from oteapi.models import AttrDict, DataCacheConfig, HostlessAnyUrl, ParserConfig from oteapi.plugins import create_strategy from pydantic import Field from pydantic.dataclasses import dataclass -from pydantic.networks import Url, UrlConstraints -HostlessAnyUrl = Annotated[Url, UrlConstraints(host_required=False)] if TYPE_CHECKING: from typing import Any, Optional From e0786d73f23efc8ef4ad3a11a4dc3c2cbd961cf2 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 26 Feb 2024 13:33:43 +0100 Subject: [PATCH 21/40] move cache key validation --- app/redis_cache/_cache.py | 14 ++++++++++++++ app/routers/dataresource.py | 2 +- app/routers/parser.py | 14 +------------- 3 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 app/redis_cache/_cache.py diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py new file mode 100644 index 00000000..1c746e3c --- /dev/null +++ b/app/redis_cache/_cache.py @@ -0,0 +1,14 @@ +from app.models.error import httpexception_404_item_id_does_not_exist +from app.redis_cache import TRedisPlugin + +async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: + """Validate if a key exists in cache and is of expected type (str or bytes).""" + if not await cache.exists(key): + raise httpexception_404_item_id_does_not_exist(key, key_type) + + cache_value = await cache.get(key) + if not isinstance(cache_value, (str, bytes)): + raise TypeError( + f"Expected cache value of {key} to be a string or bytes, " + f"found it to be of type: `{type(cache_value)!r}`." + ) \ No newline at end of file diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index c2ba0f1d..3f2a2069 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -21,7 +21,7 @@ httpexception_422_resource_id_is_unprocessable, ) from app.redis_cache import TRedisPlugin -from app.routers.parser import _validate_cache_key +from app.redis_cache._cache import _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover diff --git a/app/routers/parser.py b/app/routers/parser.py index 7ae5f6c0..7f788761 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -19,6 +19,7 @@ ListParsersResponse, ) from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -34,19 +35,6 @@ ) -async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: - """Validate if a key exists in cache and is of expected type (str or bytes).""" - if not await cache.exists(key): - raise httpexception_404_item_id_does_not_exist(key, key_type) - - cache_value = await cache.get(key) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {key} to be a string or bytes, " - f"found it to be of type: `{type(cache_value)!r}`." - ) - - # Create parser @ROUTER.post("/", response_model=CreateParserResponse) async def create_parser( From 5a7bc1ebafd5dbda3fccf2643bd358bc7373dd21 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 26 Feb 2024 14:13:02 +0100 Subject: [PATCH 22/40] requested changes --- app/redis_cache/_cache.py | 3 ++- app/routers/parser.py | 27 ------------------------ entrypoint.sh | 2 +- tests/conftest.py | 7 ++---- tests/routers/test_parser.py | 15 +------------ tests/routers/test_resource.py | 2 +- tests/static/test_strategies/download.py | 7 +----- 7 files changed, 8 insertions(+), 55 deletions(-) diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index 1c746e3c..db28cb9d 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -1,6 +1,7 @@ from app.models.error import httpexception_404_item_id_does_not_exist from app.redis_cache import TRedisPlugin + async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: """Validate if a key exists in cache and is of expected type (str or bytes).""" if not await cache.exists(key): @@ -11,4 +12,4 @@ async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> N raise TypeError( f"Expected cache value of {key} to be a string or bytes, " f"found it to be of type: `{type(cache_value)!r}`." - ) \ No newline at end of file + ) diff --git a/app/routers/parser.py b/app/routers/parser.py index 7f788761..84e53aa3 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -57,33 +57,6 @@ async def create_parser( return new_parser -@ROUTER.delete("/", response_model=DeleteAllParsersResponse) -async def delete_all_parsers( - cache: TRedisPlugin, -) -> DeleteAllParsersResponse: - """ - Delete all parser configurations in the current memory database - """ - keylist = await cache.keys(pattern=f"{IDPREFIX}*") - if not keylist: - return DeleteAllParsersResponse(number_of_deleted_parsers=0) - return DeleteAllParsersResponse( - number_of_deleted_parsers=await cache.delete(*keylist) - ) - - -# List parsers -@ROUTER.get("/", response_model=ListParsersResponse) -async def list_parsers(cache: TRedisPlugin) -> ListParsersResponse: - """Retrieve all parser IDs from cache.""" - keylist = [ - key - for key in await cache.keys(pattern=f"{IDPREFIX}*") - if isinstance(key, (str, bytes)) - ] - return ListParsersResponse(keys=keylist) - - # Get parser info @ROUTER.get("/{parser_id}/info", response_model=ParserConfig) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: diff --git a/entrypoint.sh b/entrypoint.sh index d7a42b00..702dfe36 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -34,7 +34,7 @@ else fi if [ "$DEV_ENV" = "1" ]; then - python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@" + python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 asgi:app "$@" else hypercorn --bind 0.0.0.0:8080 asgi:app "$@" fi diff --git a/tests/conftest.py b/tests/conftest.py index c2e4dc75..ecd33cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,7 +83,7 @@ def test_data() -> "dict[str, str]": }, # dataresource "dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e": { - "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", + "downloadUrl": "https://filesamples.com/sample.json", "mediaType": "application/json", "resourceType": "resource/demo", }, @@ -108,10 +108,7 @@ def test_data() -> "dict[str, str]": "parserType": "parser/demo", "entity": "http://example.com/entity", "configuration": { - "downloadUrl": ( - "https://raw.githubusercontent.com/EMMC-ASBL/" - "oteapi-core/master/tests/static/sample2.json" - ), + "downloadUrl": ("https://example.org/sample2.json"), "mediaType": "application/json", }, }, diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py index 04da2ba3..aad1b588 100644 --- a/tests/routers/test_parser.py +++ b/tests/routers/test_parser.py @@ -14,7 +14,7 @@ def test_create_parser(client: "TestClient") -> None: "parserType": "parser/demo", "entity": "http://example.com/entity", "configuration": { - "downloadUrl": "https://filesamples.com/samples/code/json/sample2.json", + "downloadUrl": "https://filesamples.com/sample2.json", "mediaType": "application/json", }, }, @@ -23,19 +23,6 @@ def test_create_parser(client: "TestClient") -> None: assert response.status_code == 200 -def test_delete_all_parsers(client: "TestClient") -> None: - """Test deleting all parsers""" - response = client.delete("/parser/") - assert response.status_code == 200 - - -def test_list_parsers(client: "TestClient") -> None: - """Test listing parsers""" - response = client.get("/parser/") - assert response.status_code == 200 - assert "keys" in response.json() - - def test_info_parser(client: "TestClient") -> None: """Test getting information about a parser""" response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa/info") diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py index 0d112bce..5a66ad09 100644 --- a/tests/routers/test_resource.py +++ b/tests/routers/test_resource.py @@ -10,7 +10,7 @@ def test_create_dataresource(client: "TestClient"): response = client.post( "/dataresource/", json={ - "downloadUrl": "https://filesamples.com/samples/code/json/sample.json", + "downloadUrl": "https://filesamples.com/sample2.json", "mediaType": "application/json", "resourceType": "resource/demo", }, diff --git a/tests/static/test_strategies/download.py b/tests/static/test_strategies/download.py index 54c22cfd..19e9a2b9 100644 --- a/tests/static/test_strategies/download.py +++ b/tests/static/test_strategies/download.py @@ -119,11 +119,6 @@ def get(self) -> HTTPDownloadContent: if cache.config.accessKey and cache.config.accessKey in cache: key = cache.config.accessKey else: - req = requests.get( - str(self.download_config.downloadUrl), - allow_redirects=True, - timeout=(3, 27), # timeout: (connect, read) in seconds - ) - key = cache.add(req.content) + key = cache.add({"dummy_content": "dummycontent"}) return HTTPDownloadContent(key=key) From 759aebdf43dcca8e7c873cf013354aa574e04c51 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Mon, 26 Feb 2024 15:38:24 +0100 Subject: [PATCH 23/40] redo fetch cache key --- app/models/parser.py | 24 ------------------------ app/redis_cache/_cache.py | 11 +++++++++-- app/routers/dataresource.py | 6 +++--- app/routers/parser.py | 20 +++++++++----------- 4 files changed, 21 insertions(+), 40 deletions(-) diff --git a/app/models/parser.py b/app/models/parser.py index 5ca1cfac..7b55e7d4 100644 --- a/app/models/parser.py +++ b/app/models/parser.py @@ -42,27 +42,3 @@ class InitializeParserResponse(InitializeResponse): Router: `POST /parser/{parser_id}/initialize` """ - - -class DeleteAllParsersResponse(AttrDict): - """### Delete all sessions - - Router: `DELETE /parser` - """ - - number_of_deleted_parsers: Annotated[ - Optional[int], - Field(description="The number of deleted parsers in the Redis cache."), - ] - - -class ListParsersResponse(Session): - """List all parser ids - - Router: `GET /parser` - """ - - keys_: Annotated[ - list[str], - Field(description="List of all parser ids in the cache.", alias="keys"), - ] diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index db28cb9d..d391f48d 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -1,15 +1,22 @@ +"Cache operations" +from typing import Any + from app.models.error import httpexception_404_item_id_does_not_exist from app.redis_cache import TRedisPlugin -async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: +async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> Any: """Validate if a key exists in cache and is of expected type (str or bytes).""" if not await cache.exists(key): raise httpexception_404_item_id_does_not_exist(key, key_type) cache_value = await cache.get(key) - if not isinstance(cache_value, (str, bytes)): + if cache_value is None: + raise ValueError(f"Cache value for key '{key}' is None.") + elif not isinstance(cache_value, (str, bytes)): raise TypeError( f"Expected cache value of {key} to be a string or bytes, " f"found it to be of type: `{type(cache_value)!r}`." ) + else: + return cache_value diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 3f2a2069..507eea6e 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -21,7 +21,7 @@ httpexception_422_resource_id_is_unprocessable, ) from app.redis_cache import TRedisPlugin -from app.redis_cache._cache import _validate_cache_key +from app.redis_cache._cache import _fetch_cache_value from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -124,7 +124,7 @@ async def read_dataresource( config = ResourceConfig(**json.loads(cache_value)) if session_id: - await _validate_cache_key(cache, session_id, "session_id") + await _fetch_cache_value(cache, session_id, "session_id") session_data = await cache.get(session_id) if session_data is None: raise ValueError("Session data is None") @@ -168,7 +168,7 @@ async def initialize_dataresource( config = ResourceConfig(**json.loads(cache_value)) if session_id: - await _validate_cache_key(cache, session_id, "session_id") + await _fetch_cache_value(cache, session_id, "session_id") cache_value = await cache.get(session_id) session_data = await cache.get(session_id) if session_data is None: diff --git a/app/routers/parser.py b/app/routers/parser.py index 84e53aa3..a4636bc7 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -2,24 +2,22 @@ import json import logging -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Optional from fastapi import APIRouter, status from oteapi.models import ParserConfig from oteapi.plugins import create_strategy from oteapi.utils.config_updater import populate_config_from_session -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.models.parser import ( IDPREFIX, CreateParserResponse, - DeleteAllParsersResponse, GetParserResponse, InitializeParserResponse, - ListParsersResponse, ) from app.redis_cache import TRedisPlugin -from app.redis_cache._cache import _validate_cache_key +from app.redis_cache._cache import _fetch_cache_value from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -46,7 +44,7 @@ async def create_parser( await cache.set(new_parser.parser_id, config.model_dump_json()) if session_id: - await _validate_cache_key(cache, session_id, "session_id") + await _fetch_cache_value(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="resource_info", @@ -61,7 +59,7 @@ async def create_parser( @ROUTER.get("/{parser_id}/info", response_model=ParserConfig) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" - await _validate_cache_key(cache, parser_id, "parser_id") + await _fetch_cache_value(cache, parser_id, "parser_id") cache_value = await cache.get(parser_id) if cache_value is None: raise ValueError("Cache value is None") @@ -75,7 +73,7 @@ async def get_parser( cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None ) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" - await _validate_cache_key(cache, parser_id, "parser_id") + await _fetch_cache_value(cache, parser_id, "parser_id") cache_value = await cache.get(parser_id) if cache_value is None: raise ValueError("Cache value is None") @@ -83,7 +81,7 @@ async def get_parser( config = ParserConfig(**config_dict) if session_id: - await _validate_cache_key(cache, session_id, "session_id") + await _fetch_cache_value(cache, session_id, "session_id") session_data = await cache.get(session_id) if session_data is None: raise ValueError("Session data is None") @@ -109,7 +107,7 @@ async def initialize_parser( session_id: Optional[str] = None, ) -> InitializeParserResponse: """Initialize parser.""" - await _validate_cache_key(cache, parser_id, "parser_id") + await _fetch_cache_value(cache, parser_id, "parser_id") cache_value = await cache.get(parser_id) if cache_value is None: raise ValueError("Cache value is None") @@ -117,7 +115,7 @@ async def initialize_parser( config = ParserConfig(**config_dict) if session_id: - await _validate_cache_key(cache, session_id, "session_id") + await _fetch_cache_value(cache, session_id, "session_id") session_data = await cache.get(session_id) if session_data is None: raise ValueError("Session data is None") From f4aa22ab179de710b0f9aa8634803beb34da9e03 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Tue, 27 Feb 2024 09:30:04 +0100 Subject: [PATCH 24/40] requested changes --- app/redis_cache/_cache.py | 11 +++- app/routers/datafilter.py | 65 +++++------------- app/routers/dataresource.py | 51 ++------------- app/routers/function.py | 67 +++++-------------- app/routers/mapping.py | 67 +++++-------------- app/routers/parser.py | 29 ++------ app/routers/redisadmin.py | 1 + app/routers/session.py | 38 ++--------- app/routers/transformation.py | 120 +++++++++------------------------- 9 files changed, 112 insertions(+), 337 deletions(-) diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index d391f48d..9bfcfba3 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -6,9 +6,8 @@ async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> Any: - """Validate if a key exists in cache and is of expected type (str or bytes).""" - if not await cache.exists(key): - raise httpexception_404_item_id_does_not_exist(key, key_type) + """Fetch key value from Cache and check if its of expected type (str or bytes).""" + await _validate_cache_key(cache, key, key_type) cache_value = await cache.get(key) if cache_value is None: @@ -20,3 +19,9 @@ async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> An ) else: return cache_value + + +async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: + """Validate if a key exists in cache and is of expected type (str or bytes).""" + if not await cache.exists(key): + raise httpexception_404_item_id_does_not_exist(key, key_type) diff --git a/app/routers/datafilter.py b/app/routers/datafilter.py index e054bd08..dad99c63 100644 --- a/app/routers/datafilter.py +++ b/app/routers/datafilter.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, status from oteapi.models import FilterConfig from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session from app.models.datafilter import ( IDPREFIX, @@ -13,8 +14,9 @@ GetFilterResponse, InitializeFilterResponse, ) -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -41,8 +43,7 @@ async def create_filter( await cache.set(new_filter.filter_id, config.model_dump_json()) if session_id: - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="filter_info", @@ -60,32 +61,15 @@ async def get_filter( session_id: Optional[str] = None, ) -> GetFilterResponse: """Run and return data from a filter (data operation)""" - if not await cache.exists(filter_id): - raise httpexception_404_item_id_does_not_exist(filter_id, "filter_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(filter_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {filter_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, filter_id, "filter_id") config = FilterConfig(**json.loads(cache_value)) - strategy: "IFilterStrategy" = create_strategy("filter", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = strategy.get(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + strategy: "IFilterStrategy" = create_strategy("filter", config) + session_update = strategy.get() if session_update and session_id: await _update_session( @@ -102,32 +86,15 @@ async def initialize_filter( session_id: Optional[str] = None, ) -> InitializeFilterResponse: """Initialize and return data to update session.""" - if not await cache.exists(filter_id): - raise httpexception_404_item_id_does_not_exist(filter_id, "filter_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(filter_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {filter_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, filter_id, "filter_id") config = FilterConfig(**json.loads(cache_value)) - strategy: "IFilterStrategy" = create_strategy("filter", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = strategy.initialize(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + strategy: "IFilterStrategy" = create_strategy("filter", config) + session_update = strategy.initialize() if session_update and session_id: await _update_session( diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 507eea6e..f228d5b7 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -17,11 +17,10 @@ from app.models.error import ( HTTPNotFoundError, HTTPValidationError, - httpexception_404_item_id_does_not_exist, httpexception_422_resource_id_is_unprocessable, ) from app.redis_cache import TRedisPlugin -from app.redis_cache._cache import _fetch_cache_value +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -66,8 +65,7 @@ async def create_dataresource( await cache.set(new_resource.resource_id, resource_config) if session_id: - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="resource_info", @@ -84,15 +82,7 @@ async def info_dataresource( resource_id: str, ) -> ResourceConfig: """Get data resource info.""" - if not await cache.exists(resource_id): - raise httpexception_404_item_id_does_not_exist(resource_id, "resource_id") - - cache_value = await cache.get(resource_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {resource_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, resource_id, "resource_id") return ResourceConfig(**json.loads(cache_value)) @@ -110,24 +100,11 @@ async def read_dataresource( Parse data information using the appropriate parser. """ - if not await cache.exists(resource_id): - raise httpexception_404_item_id_does_not_exist(resource_id, "resource_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(resource_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {resource_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, resource_id, "resource_id") config = ResourceConfig(**json.loads(cache_value)) if session_id: - await _fetch_cache_value(cache, session_id, "session_id") - session_data = await cache.get(session_id) - if session_data is None: - raise ValueError("Session data is None") + session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) if not config.resourceType: @@ -154,25 +131,11 @@ async def initialize_dataresource( session_id: Optional[str] = None, ) -> InitializeResourceResponse: """Initialize data resource.""" - if not await cache.exists(resource_id): - raise httpexception_404_item_id_does_not_exist(resource_id, "resource_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(resource_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {resource_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, resource_id, "resource_id") config = ResourceConfig(**json.loads(cache_value)) if session_id: - await _fetch_cache_value(cache, session_id, "session_id") - cache_value = await cache.get(session_id) - session_data = await cache.get(session_id) - if session_data is None: - raise ValueError("Session data is None") + session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) if not config.resourceType: diff --git a/app/routers/function.py b/app/routers/function.py index 9fb4f9cb..bf8ef3ed 100644 --- a/app/routers/function.py +++ b/app/routers/function.py @@ -6,8 +6,9 @@ from fastapi import APIRouter, Request, status from oteapi.models import FunctionConfig from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.models.function import ( IDPREFIX, CreateFunctionResponse, @@ -15,11 +16,14 @@ InitializeFunctionResponse, ) from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover from typing import Any + from oteapi.interfaces import IFunctionStrategy + ROUTER = APIRouter( prefix=f"/{IDPREFIX}", tags=["function"], @@ -44,8 +48,7 @@ async def create_function( await cache.set(new_function.function_id, function_config) if session_id: - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="function_info", @@ -63,32 +66,15 @@ async def get_function( session_id: Optional[str] = None, ) -> GetFunctionResponse: """Get (execute) function.""" - if not await cache.exists(function_id): - raise httpexception_404_item_id_does_not_exist(function_id, "function_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(function_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {function_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, function_id, "function_id") config = FunctionConfig(**json.loads(cache_value)) - function_strategy = create_strategy("function", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = function_strategy.get(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + function_strategy: "IFunctionStrategy" = create_strategy("function", config) + session_update = function_strategy.get() if session_update and session_id: await _update_session( @@ -105,32 +91,15 @@ async def initialize_function( session_id: Optional[str] = None, ) -> InitializeFunctionResponse: """Initialize and update function.""" - if not await cache.exists(function_id): - raise httpexception_404_item_id_does_not_exist(function_id, "function_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(function_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {function_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, function_id, "function_id") config = FunctionConfig(**json.loads(cache_value)) - function_strategy = create_strategy("function", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = function_strategy.initialize(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + function_strategy: "IFunctionStrategy" = create_strategy("function", config) + session_update = function_strategy.initialize() if session_update and session_id: await _update_session( diff --git a/app/routers/mapping.py b/app/routers/mapping.py index 3de8e58a..a7fcea5d 100644 --- a/app/routers/mapping.py +++ b/app/routers/mapping.py @@ -6,8 +6,9 @@ from fastapi import APIRouter, status from oteapi.models import MappingConfig from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.models.mapping import ( IDPREFIX, CreateMappingResponse, @@ -15,11 +16,14 @@ InitializeMappingResponse, ) from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover from typing import Any + from oteapi.interfaces import IMappingStrategy + ROUTER = APIRouter( prefix=f"/{IDPREFIX}", tags=["mapping"], @@ -42,8 +46,7 @@ async def create_mapping( await cache.set(new_mapping.mapping_id, config.model_dump_json()) if session_id: - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="mapping_info", @@ -61,32 +64,15 @@ async def get_mapping( session_id: Optional[str] = None, ) -> GetMappingResponse: """Run and return data""" - if not await cache.exists(mapping_id): - raise httpexception_404_item_id_does_not_exist(mapping_id, "mapping_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(mapping_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {mapping_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, mapping_id, "mapping_id") config = MappingConfig(**json.loads(cache_value)) - mapping_strategy = create_strategy("mapping", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = mapping_strategy.get(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + mapping_strategy: "IMappingStrategy" = create_strategy("mapping", config) + session_update = mapping_strategy.get() if session_update and session_id: await _update_session( @@ -108,32 +94,15 @@ async def initialize_mapping( - **mapping_id**: Unique identifier of a mapping configuration - **session_id**: Optional reference to a session object """ - if not await cache.exists(mapping_id): - raise httpexception_404_item_id_does_not_exist(mapping_id, "mapping_id") - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(mapping_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {mapping_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, mapping_id, "mapping_id") config = MappingConfig(**json.loads(cache_value)) - mapping_strategy = create_strategy("mapping", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = mapping_strategy.initialize(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + mapping_strategy: "IMappingStrategy" = create_strategy("mapping", config) + session_update = mapping_strategy.initialize() if session_update and session_id: await _update_session( diff --git a/app/routers/parser.py b/app/routers/parser.py index a4636bc7..4b1f7b72 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -17,7 +17,7 @@ InitializeParserResponse, ) from app.redis_cache import TRedisPlugin -from app.redis_cache._cache import _fetch_cache_value +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -44,7 +44,7 @@ async def create_parser( await cache.set(new_parser.parser_id, config.model_dump_json()) if session_id: - await _fetch_cache_value(cache, session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="resource_info", @@ -59,10 +59,7 @@ async def create_parser( @ROUTER.get("/{parser_id}/info", response_model=ParserConfig) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" - await _fetch_cache_value(cache, parser_id, "parser_id") - cache_value = await cache.get(parser_id) - if cache_value is None: - raise ValueError("Cache value is None") + cache_value = await _fetch_cache_value(cache, parser_id, "parser_id") config_dict = json.loads(cache_value) return ParserConfig(**config_dict) @@ -73,18 +70,12 @@ async def get_parser( cache: TRedisPlugin, parser_id: str, session_id: Optional[str] = None ) -> GetParserResponse: """Retrieve and parse data using a specified parser.""" - await _fetch_cache_value(cache, parser_id, "parser_id") - cache_value = await cache.get(parser_id) - if cache_value is None: - raise ValueError("Cache value is None") + cache_value = await _fetch_cache_value(cache, parser_id, "parser_id") config_dict = json.loads(cache_value) config = ParserConfig(**config_dict) if session_id: - await _fetch_cache_value(cache, session_id, "session_id") - session_data = await cache.get(session_id) - if session_data is None: - raise ValueError("Session data is None") + session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) strategy: "IParseStrategy" = create_strategy("parse", config) @@ -107,18 +98,12 @@ async def initialize_parser( session_id: Optional[str] = None, ) -> InitializeParserResponse: """Initialize parser.""" - await _fetch_cache_value(cache, parser_id, "parser_id") - cache_value = await cache.get(parser_id) - if cache_value is None: - raise ValueError("Cache value is None") + cache_value = await _fetch_cache_value(cache, parser_id, "parser_id") config_dict = json.loads(cache_value) config = ParserConfig(**config_dict) if session_id: - await _fetch_cache_value(cache, session_id, "session_id") - session_data = await cache.get(session_id) - if session_data is None: - raise ValueError("Session data is None") + session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) strategy: "IParseStrategy" = create_strategy("parse", config) diff --git a/app/routers/redisadmin.py b/app/routers/redisadmin.py index 62ea4d2b..f0071a7e 100644 --- a/app/routers/redisadmin.py +++ b/app/routers/redisadmin.py @@ -7,6 +7,7 @@ from app.models.error import httpexception_404_item_id_does_not_exist from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value ROUTER = APIRouter(prefix="/redis", tags=["admin"]) diff --git a/app/routers/session.py b/app/routers/session.py index 60c1b02d..19e49850 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, status from oteapi.models import AttrDict -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.models.response import Session from app.models.session import ( IDPREFIX, @@ -16,6 +16,7 @@ ListSessionsResponse, ) from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key if TYPE_CHECKING: # pragma: no cover from typing import Any, Union @@ -98,14 +99,7 @@ async def _get_session( redis: TRedisPlugin, ) -> Session: """Return the session contents given a `session_id`.""" - if not await redis.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - cache_value = await redis.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, found it " - f"to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(redis, session_id, "session_id") return Session(**json.loads(cache_value)) @@ -156,15 +150,7 @@ async def update_session( updated_session: AttrDict, ) -> Session: """Update session object.""" - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, found it " - f"to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, session_id, "session_id") session = Session(**json.loads(cache_value)) session.update(updated_session) await cache.set(session_id, session.model_dump_json().encode("utf-8")) @@ -177,16 +163,7 @@ async def get_session( session_id: str, ) -> Session: """Fetch the entire session object.""" - cache_exists: int = await cache.exists(session_id) - if not cache_exists: - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, found it " - f"to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value(cache, session_id, "session_id") return Session(**json.loads(cache_value)) @@ -196,9 +173,6 @@ async def delete_session( session_id: str, ) -> DeleteSessionResponse: """Delete a session object.""" - cache_exists: int = await cache.exists(session_id) - if not cache_exists: - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - + await _validate_cache_key(cache, session_id, "session_id") await cache.delete(session_id) return DeleteSessionResponse(success=True) diff --git a/app/routers/transformation.py b/app/routers/transformation.py index 53cda7f7..c3371102 100644 --- a/app/routers/transformation.py +++ b/app/routers/transformation.py @@ -6,8 +6,9 @@ from fastapi import APIRouter, Request, status from oteapi.models import TransformationConfig, TransformationStatus from oteapi.plugins import create_strategy +from oteapi.utils.config_updater import populate_config_from_session -from app.models.error import HTTPNotFoundError, httpexception_404_item_id_does_not_exist +from app.models.error import HTTPNotFoundError from app.models.transformation import ( IDPREFIX, CreateTransformationResponse, @@ -16,6 +17,7 @@ InitializeTransformationResponse, ) from app.redis_cache import TRedisPlugin +from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover @@ -47,8 +49,7 @@ async def create_transformation( await cache.set(new_transformation.transformation_id, transformation_config) if session_id: - if not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") + await _validate_cache_key(cache, session_id, "session_id") await _update_session_list_item( session_id=session_id, list_key="transformation_info", @@ -66,17 +67,9 @@ async def get_transformation_status( task_id: str, ) -> TransformationStatus: """Get the current status of a defined transformation.""" - if not await cache.exists(transformation_id): - raise httpexception_404_item_id_does_not_exist( - transformation_id, "transformation_id" - ) - - cache_value = await cache.get(transformation_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {transformation_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value( + cache, transformation_id, "transformation_id" + ) config = TransformationConfig(**json.loads(cache_value)) strategy: "ITransformationStrategy" = create_strategy("transformation", config) @@ -90,34 +83,17 @@ async def get_transformation( session_id: Optional[str] = None, ) -> GetTransformationResponse: """Get transformation.""" - if not await cache.exists(transformation_id): - raise httpexception_404_item_id_does_not_exist( - transformation_id, "transformation_id" - ) - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(transformation_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {transformation_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value( + cache, transformation_id, "transformation_id" + ) config = TransformationConfig(**json.loads(cache_value)) - strategy: "ITransformationStrategy" = create_strategy("transformation", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = strategy.get(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + strategy: "ITransformationStrategy" = create_strategy("transformation", config) + session_update = strategy.get() if session_update and session_id: await _update_session( @@ -137,34 +113,17 @@ async def execute_transformation( ) -> ExecuteTransformationResponse: """Execute (run) a transformation.""" # Fetch transformation info from cache - if not await cache.exists(transformation_id): - raise httpexception_404_item_id_does_not_exist( - transformation_id, "transformation_id" - ) - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(transformation_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {transformation_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value( + cache, transformation_id, "transformation_id" + ) config = TransformationConfig(**json.loads(cache_value)) - strategy: "ITransformationStrategy" = create_strategy("transformation", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = strategy.get(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + strategy: "ITransformationStrategy" = create_strategy("transformation", config) + session_update = strategy.get() if session_update and session_id: await _update_session( @@ -184,34 +143,17 @@ async def initialize_transformation( ) -> InitializeTransformationResponse: """Initialize a transformation.""" # Fetch transformation info from cache - if not await cache.exists(transformation_id): - raise httpexception_404_item_id_does_not_exist( - transformation_id, "transformation_id" - ) - if session_id and not await cache.exists(session_id): - raise httpexception_404_item_id_does_not_exist(session_id, "session_id") - - cache_value = await cache.get(transformation_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {transformation_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) + cache_value = await _fetch_cache_value( + cache, transformation_id, "transformation_id" + ) config = TransformationConfig(**json.loads(cache_value)) - strategy: "ITransformationStrategy" = create_strategy("transformation", config) - if session_id: - cache_value = await cache.get(session_id) - if not isinstance(cache_value, (str, bytes)): - raise TypeError( - f"Expected cache value of {session_id} to be a string or bytes, " - f"found it to be of type {type(cache_value)!r}." - ) - session_data: "Optional[dict[str, Any]]" = ( - None if not session_id else json.loads(cache_value) - ) - session_update = strategy.initialize(session=session_data) + session_data = await _fetch_cache_value(cache, session_id, "session_id") + populate_config_from_session(json.loads(session_data), config) + + strategy: "ITransformationStrategy" = create_strategy("transformation", config) + session_update = strategy.initialize() if session_update and session_id: await _update_session( From 24d023711bc02cc83f1225ae796bed3d01380231 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Tue, 27 Feb 2024 09:37:56 +0100 Subject: [PATCH 25/40] fix pylint errors --- app/models/parser.py | 5 ++--- app/redis_cache/_cache.py | 3 +-- app/routers/redisadmin.py | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/models/parser.py b/app/models/parser.py index 7b55e7d4..96308c55 100644 --- a/app/models/parser.py +++ b/app/models/parser.py @@ -1,12 +1,11 @@ """Parser specific pydantic response models.""" -from typing import Annotated, Optional +from typing import Annotated from uuid import uuid4 -from oteapi.models import AttrDict from pydantic import Field -from app.models.response import CreateResponse, GetResponse, InitializeResponse, Session +from app.models.response import CreateResponse, GetResponse, InitializeResponse IDPREFIX = "parser" diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index 9bfcfba3..eb140475 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -17,8 +17,7 @@ async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> An f"Expected cache value of {key} to be a string or bytes, " f"found it to be of type: `{type(cache_value)!r}`." ) - else: - return cache_value + return cache_value async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None: diff --git a/app/routers/redisadmin.py b/app/routers/redisadmin.py index f0071a7e..62ea4d2b 100644 --- a/app/routers/redisadmin.py +++ b/app/routers/redisadmin.py @@ -7,7 +7,6 @@ from app.models.error import httpexception_404_item_id_does_not_exist from app.redis_cache import TRedisPlugin -from app.redis_cache._cache import _fetch_cache_value ROUTER = APIRouter(prefix="/redis", tags=["admin"]) From 94cf49c88c7038278ffb14dd4b6d39ce824c424f Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Tue, 27 Feb 2024 09:47:02 +0100 Subject: [PATCH 26/40] fix pylint errors --- app/redis_cache/_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index eb140475..2b7ef5b0 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -12,7 +12,7 @@ async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> An cache_value = await cache.get(key) if cache_value is None: raise ValueError(f"Cache value for key '{key}' is None.") - elif not isinstance(cache_value, (str, bytes)): + if not isinstance(cache_value, (str, bytes)): raise TypeError( f"Expected cache value of {key} to be a string or bytes, " f"found it to be of type: `{type(cache_value)!r}`." From 968b0f3d2eed15002a9c5ae6eb11529dccec4fc5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 27 Feb 2024 14:29:37 +0100 Subject: [PATCH 27/40] Remove unused imports --- app/redis_cache/_cache.py | 3 ++- app/routers/datafilter.py | 2 -- app/routers/dataresource.py | 5 +---- app/routers/function.py | 2 -- app/routers/mapping.py | 2 -- app/routers/transformation.py | 2 -- 6 files changed, 3 insertions(+), 13 deletions(-) diff --git a/app/redis_cache/_cache.py b/app/redis_cache/_cache.py index 2b7ef5b0..1226bb5d 100644 --- a/app/redis_cache/_cache.py +++ b/app/redis_cache/_cache.py @@ -1,4 +1,5 @@ -"Cache operations" +"""Cache operations.""" + from typing import Any from app.models.error import httpexception_404_item_id_does_not_exist diff --git a/app/routers/datafilter.py b/app/routers/datafilter.py index dad99c63..01057e10 100644 --- a/app/routers/datafilter.py +++ b/app/routers/datafilter.py @@ -20,8 +20,6 @@ from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover - from typing import Any - from oteapi.interfaces import IFilterStrategy ROUTER = APIRouter( diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index f228d5b7..4ca72d80 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -1,7 +1,7 @@ """Data Resource.""" import json -from typing import TYPE_CHECKING, Optional +from typing import Optional from fastapi import APIRouter, Request, status from oteapi.models import ResourceConfig @@ -23,9 +23,6 @@ from app.redis_cache._cache import _fetch_cache_value, _validate_cache_key from app.routers.session import _update_session, _update_session_list_item -if TYPE_CHECKING: # pragma: no cover - from typing import Any - ROUTER = APIRouter( prefix=f"/{IDPREFIX}", tags=["dataresource"], diff --git a/app/routers/function.py b/app/routers/function.py index bf8ef3ed..e5048252 100644 --- a/app/routers/function.py +++ b/app/routers/function.py @@ -20,8 +20,6 @@ from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover - from typing import Any - from oteapi.interfaces import IFunctionStrategy ROUTER = APIRouter( diff --git a/app/routers/mapping.py b/app/routers/mapping.py index a7fcea5d..d0bdb6a6 100644 --- a/app/routers/mapping.py +++ b/app/routers/mapping.py @@ -20,8 +20,6 @@ from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover - from typing import Any - from oteapi.interfaces import IMappingStrategy ROUTER = APIRouter( diff --git a/app/routers/transformation.py b/app/routers/transformation.py index c3371102..ecd69e2e 100644 --- a/app/routers/transformation.py +++ b/app/routers/transformation.py @@ -21,8 +21,6 @@ from app.routers.session import _update_session, _update_session_list_item if TYPE_CHECKING: # pragma: no cover - from typing import Any - from oteapi.interfaces import ITransformationStrategy ROUTER = APIRouter( From 2ae97b01f898e1e041569c8630237f059e4a98e5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 27 Feb 2024 14:33:19 +0100 Subject: [PATCH 28/40] Satisfy pre-commit hooks --- .pre-commit-config.yaml | 2 +- tests/routers/test_resource.py | 4 ++++ tests/static/test_strategies/download.py | 1 - tests/static/test_strategies/parse.py | 3 +-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c291286..dba810a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,7 +66,7 @@ repos: - id: pylint-tests name: pylint - tests entry: pylint - args: ["--rcfile=pyproject.toml", "--extension-pkg-whitelist='pydantic'", "--disable=C0415,W0621"] + args: ["--rcfile=pyproject.toml", "--extension-pkg-whitelist='pydantic'", "--disable=C0415,W0621,E1101"] language: python types: [python] require_serial: true diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py index 5a66ad09..26ffe4fb 100644 --- a/tests/routers/test_resource.py +++ b/tests/routers/test_resource.py @@ -7,6 +7,7 @@ def test_create_dataresource(client: "TestClient"): + """Test create dataresource.""" response = client.post( "/dataresource/", json={ @@ -19,6 +20,7 @@ def test_create_dataresource(client: "TestClient"): def test_get_dataresource_info(client: "TestClient"): + """Test get dataresource info.""" response = client.get( "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/info" ) @@ -26,6 +28,7 @@ def test_get_dataresource_info(client: "TestClient"): def test_read_dataresource(client: "TestClient"): + """Test read dataresource.""" response = client.get( "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e" ) @@ -33,6 +36,7 @@ def test_read_dataresource(client: "TestClient"): def test_initialize_dataresource(client: "TestClient"): + """Test initialize dataresource.""" response = client.post( "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/initialize" ) diff --git a/tests/static/test_strategies/download.py b/tests/static/test_strategies/download.py index 19e9a2b9..1bf8acf1 100644 --- a/tests/static/test_strategies/download.py +++ b/tests/static/test_strategies/download.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Annotated, Optional -import requests from oteapi.datacache.datacache import DataCache from oteapi.models import AttrDict, DataCacheConfig, ResourceConfig from pydantic import AnyHttpUrl, BaseModel, Field diff --git a/tests/static/test_strategies/parse.py b/tests/static/test_strategies/parse.py index 4e326275..5b5ae861 100644 --- a/tests/static/test_strategies/parse.py +++ b/tests/static/test_strategies/parse.py @@ -5,14 +5,13 @@ from typing import TYPE_CHECKING, Literal, Optional from oteapi.datacache import DataCache -from oteapi.datacache.datacache import DataCache from oteapi.models import AttrDict, DataCacheConfig, HostlessAnyUrl, ParserConfig from oteapi.plugins import create_strategy from pydantic import Field from pydantic.dataclasses import dataclass if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any class DEMOConfig(AttrDict): From bc04d822f807d5d9b4e640908a6bf3b055aa8012 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 28 Feb 2024 13:55:47 +0100 Subject: [PATCH 29/40] Temporarily use the latest developments in oteapi-core --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b8a389f8..ef6b0e67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,8 @@ idna==3.6 iso8601==2.1.0 kombu==5.3.5 openpyxl==3.1.2 -oteapi-core==0.6.1 +# oteapi-core==0.6.1 +oteapi-core @ git+https://github.com/EMMC-ASBL/oteapi-core@master paramiko==3.4.0 pillow==10.2.0 priority==2.0.0 From 25e8deb74487cfd765756b3de83a03c01a435a43 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 28 Feb 2024 13:56:12 +0100 Subject: [PATCH 30/40] Do not show coverage on failed test runs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1df8f038..9ba88d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ allow_redefinition = true plugins = ["pydantic.mypy"] [tool.pytest.ini_options] -addopts = "-rs --cov=app --cov-report=term-missing" +addopts = "-rs --cov=app --cov-report=term-missing:skip-covered --no-cov-on-fail" filterwarnings = [ "ignore:.*imp module.*:DeprecationWarning", From d3eb4c83e7fc39a0c7b48d2d64c3657f8199c0af Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 28 Feb 2024 13:59:30 +0100 Subject: [PATCH 31/40] Add session merging tests to all router tests --- tests/conftest.py | 27 +++-- tests/routers/test_filter.py | 99 +++++++++++++++-- tests/routers/test_function.py | 87 ++++++++++++++- tests/routers/test_mapping.py | 82 +++++++++++++- tests/routers/test_parser.py | 95 +++++++++++++++- tests/routers/test_resource.py | 94 ++++++++++++++-- tests/routers/test_transformation.py | 105 ++++++++++++++++-- tests/static/test_strategies/download.py | 27 +---- tests/static/test_strategies/filter.py | 38 +++---- tests/static/test_strategies/function.py | 22 +--- tests/static/test_strategies/mapping.py | 23 +--- tests/static/test_strategies/parse.py | 18 ++- .../static/test_strategies/transformation.py | 24 ++-- 13 files changed, 587 insertions(+), 154 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ecd33cb4..fd1a24a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,34 +1,47 @@ """Fixtures and configuration for PyTest.""" # pylint: disable=invalid-name,redefined-builtin,unused-argument,comparison-with-callable +import logging from typing import TYPE_CHECKING import pytest if TYPE_CHECKING: from pathlib import Path + from typing import Union from fastapi.testclient import TestClient +logging.getLogger("test_strategies").setLevel(logging.DEBUG) + class DummyCache: """Mock cache for RedisCache.""" - obj = {} + obj: dict[str, "Union[str, bytes]"] = {} def __init__(self, o=None): self.obj = o or {} async def set(self, id, data) -> None: """Mock `set()` method.""" + import json + if data: - self.obj[id] = data + if isinstance(data, (str, bytes)): + self.obj[id] = data + else: + self.obj[id] = json.dumps(data) async def get(self, id) -> dict: """Mock `get()` method.""" import json + from copy import deepcopy - return json.loads(json.dumps(self.obj[id])) + if isinstance(self.obj[id], (str, bytes)): + return deepcopy(self.obj[id]) + + return json.dumps(self.obj[id]) async def keys(self, pattern: str) -> "list[bytes]": """Mock `keys()` method.""" @@ -63,7 +76,7 @@ def top_dir() -> "Path": return Path(__file__).resolve().parent.parent.resolve() -@pytest.fixture(scope="session") +@pytest.fixture() def test_data() -> "dict[str, str]": """Test data stored in DummyCache.""" import json @@ -95,8 +108,8 @@ def test_data() -> "dict[str, str]": "configuration": {}, }, # sessions - "1": {"foo": "bar"}, - "2": {"foo": "bar"}, + "session-f752c613-fde0-4d43-a7f6-c50f68642daa": {"foo": "bar"}, + "session-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e": {"foo": ["bar", "baz"]}, # transformation "transformation-f752c613-fde0-4d43-a7f6-c50f68642daa": { "transformationType": "script/demo", @@ -174,7 +187,7 @@ def load_test_strategies() -> None: } -@pytest.fixture(scope="session") +@pytest.fixture() def client(test_data: "dict[str, dict]") -> "TestClient": """Return a test client.""" from fastapi.testclient import TestClient diff --git a/tests/routers/test_filter.py b/tests/routers/test_filter.py index b634f695..9ddaab33 100644 --- a/tests/routers/test_filter.py +++ b/tests/routers/test_filter.py @@ -1,14 +1,19 @@ """Test data filter.""" -import json from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient def test_create_filter(client: "TestClient") -> None: """Test creating a filter.""" + import json + response = client.post( "/filter/", json={ @@ -18,23 +23,101 @@ def test_create_filter(client: "TestClient") -> None: "limit": 1, "configuration": {}, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) # Ensure that session returns with a session id assert "filter_id" in response.json() - assert response.status_code == 200, f"Response:\n{json.dumps(response.json())}" + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}" -def test_get_filter(client: "TestClient") -> None: +def test_get_filter(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting a filter.""" - response = client.get("/filter/filter-961f5314-9e8e-411e-a216-ba0eb8e8bc6e") - assert response.status_code == 200, f"Response:\n{json.dumps(response.json())}" + import json + + filter_id = next(_ for _ in test_data if _.startswith("filter-")) + response = client.get( + f"/filter/{filter_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}" -def test_initialize_filter(client: "TestClient") -> None: +def test_initialize_filter(client: "TestClient", test_data: dict[str, str]) -> None: """Test initializing a filter.""" + import json + + filter_id = next(_ for _ in test_data if _.startswith("filter-")) response = client.post( - "/filter/filter-961f5314-9e8e-411e-a216-ba0eb8e8bc6e/initialize", json={} + f"/filter/{filter_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert ( response.status_code == 200 - ), f"Response:\n{json.dumps(response.json())}\nResponse URL: {response.url}" + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import FilterConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + filter_id = next(_ for _ in test_data if _.startswith("filter-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[filter_id]) + expected_merged_config["configuration"].update(json.loads(test_data[session_id])) + + def create_strategy_middleware( + strategy_type: 'Literal["filter"]', config: FilterConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "filter" + assert isinstance(config, FilterConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.datafilter.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/filter/{filter_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/filter/{filter_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/routers/test_function.py b/tests/routers/test_function.py index 158e89cb..3e2c5954 100644 --- a/tests/routers/test_function.py +++ b/tests/routers/test_function.py @@ -2,7 +2,11 @@ from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient @@ -14,6 +18,8 @@ def test_create_function(client: "TestClient") -> None: "functionType": "function/demo", "configuration": {}, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert "function_id" in response.json() assert response.status_code == 200 @@ -29,6 +35,8 @@ def test_create_function_auth_model(client: "TestClient") -> None: "configuration": {}, "token": dummy_secret, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 @@ -46,7 +54,8 @@ def test_create_function_auth_headers(client: "TestClient") -> None: "functionType": "function/demo", "configuration": {}, }, - headers={"Authorization": dummy_secret}, + headers={"Authorization": dummy_secret, "Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 @@ -55,15 +64,83 @@ def test_create_function_auth_headers(client: "TestClient") -> None: assert response.json().get("token") == dummy_secret -def test_get_function(client: "TestClient") -> None: +def test_get_function(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting a function.""" - response = client.get("/function/function-a647012a-7ab9-4f2c-9c13-2564aa6d95a1") + function_id = next(_ for _ in test_data if _.startswith("function-")) + response = client.get( + f"/function/{function_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) assert response.status_code == 200 -def test_initialize_function(client: "TestClient") -> None: +def test_initialize_function(client: "TestClient", test_data: dict[str, str]) -> None: """Test initializing a function.""" + function_id = next(_ for _ in test_data if _.startswith("function-")) response = client.post( - "/function/function-a647012a-7ab9-4f2c-9c13-2564aa6d95a1/initialize", json={} + f"/function/{function_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import FunctionConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + function_id = next(_ for _ in test_data if _.startswith("function-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[function_id]) + expected_merged_config["configuration"].update(json.loads(test_data[session_id])) + + def create_strategy_middleware( + strategy_type: 'Literal["function"]', config: FunctionConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "function" + assert isinstance(config, FunctionConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.function.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/function/{function_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/function/{function_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/routers/test_mapping.py b/tests/routers/test_mapping.py index b695daae..81e246a6 100644 --- a/tests/routers/test_mapping.py +++ b/tests/routers/test_mapping.py @@ -2,7 +2,11 @@ from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient @@ -16,20 +20,90 @@ def test_create_mapping(client: "TestClient") -> None: "triples": [["a", "b", "c"]], "configuration": {}, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert "mapping_id" in response.json() assert response.status_code == 200 -def test_get_mapping(client: "TestClient") -> None: +def test_get_mapping(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting a mapping.""" - response = client.get("/mapping/mapping-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e") + mapping_id = next(_ for _ in test_data if _.startswith("mapping-")) + response = client.get( + f"/mapping/{mapping_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) assert response.status_code == 200 -def test_initialize_mapping(client: "TestClient") -> None: +def test_initialize_mapping(client: "TestClient", test_data: dict[str, str]) -> None: """Test initializing a mapping.""" + mapping_id = next(_ for _ in test_data if _.startswith("mapping-")) response = client.post( - "/mapping/mapping-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e/initialize", json={} + f"/mapping/{mapping_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import MappingConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + mapping_id = next(_ for _ in test_data if _.startswith("mapping-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[mapping_id]) + expected_merged_config["configuration"].update(json.loads(test_data[session_id])) + + def create_strategy_middleware( + strategy_type: 'Literal["mapping"]', config: MappingConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "mapping" + assert isinstance(config, MappingConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.mapping.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/mapping/{mapping_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/mapping/{mapping_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/routers/test_parser.py b/tests/routers/test_parser.py index aad1b588..f7465e03 100644 --- a/tests/routers/test_parser.py +++ b/tests/routers/test_parser.py @@ -2,7 +2,11 @@ from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient @@ -18,20 +22,103 @@ def test_create_parser(client: "TestClient") -> None: "mediaType": "application/json", }, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert "parser_id" in response.json() assert response.status_code == 200 -def test_info_parser(client: "TestClient") -> None: +def test_info_parser(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting information about a parser""" - response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa/info") + parser_id = next(_ for _ in test_data if _.startswith("parser-")) + response = client.get( + f"/parser/{parser_id}/info", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) assert response.status_code == 200 assert "parserType" in response.json() assert "configuration" in response.json() -def test_get_parser(client: "TestClient") -> None: +def test_get_parser(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting and parsing data using a specified parser""" - response = client.get("/parser/parser-f752c613-fde0-4d43-a7f6-c50f68642daa") + parser_id = next(_ for _ in test_data if _.startswith("parser-")) + response = client.get( + f"/parser/{parser_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) assert response.status_code == 200 + + +def test_initialize_parser(client: "TestClient", test_data: dict[str, str]) -> None: + """Test initializing a parser.""" + parser_id = next(_ for _ in test_data if _.startswith("parser-")) + response = client.post( + f"/parser/{parser_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + assert response.status_code == 200 + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import ParserConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + parser_id = next(_ for _ in test_data if _.startswith("parser-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[parser_id]) + expected_merged_config["configuration"].update(json.loads(test_data[session_id])) + + def create_strategy_middleware( + strategy_type: 'Literal["parse"]', config: ParserConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "parse" + assert isinstance(config, ParserConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.parser.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/parser/{parser_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/parser/{parser_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py index 26ffe4fb..ca6524f6 100644 --- a/tests/routers/test_resource.py +++ b/tests/routers/test_resource.py @@ -1,12 +1,16 @@ -""" Test parser """ +""" Test dataresource """ from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient -def test_create_dataresource(client: "TestClient"): +def test_create_dataresource(client: "TestClient") -> None: """Test create dataresource.""" response = client.post( "/dataresource/", @@ -15,29 +19,103 @@ def test_create_dataresource(client: "TestClient"): "mediaType": "application/json", "resourceType": "resource/demo", }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 -def test_get_dataresource_info(client: "TestClient"): +def test_get_dataresource_info(client: "TestClient", test_data: dict[str, str]) -> None: """Test get dataresource info.""" + dataresource_id = next(_ for _ in test_data if _.startswith("dataresource-")) response = client.get( - "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/info" + f"/dataresource/{dataresource_id}/info", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 -def test_read_dataresource(client: "TestClient"): +def test_read_dataresource(client: "TestClient", test_data: dict[str, str]) -> None: """Test read dataresource.""" + dataresource_id = next(_ for _ in test_data if _.startswith("dataresource-")) response = client.get( - "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e" + f"/dataresource/{dataresource_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 -def test_initialize_dataresource(client: "TestClient"): +def test_initialize_dataresource( + client: "TestClient", test_data: dict[str, str] +) -> None: """Test initialize dataresource.""" + dataresource_id = next(_ for _ in test_data if _.startswith("dataresource-")) response = client.post( - "/dataresource/dataresource-910c9965-a318-4ac4-9123-9c55d5b86f2e/initialize" + f"/dataresource/{dataresource_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import ResourceConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + dataresource_id = next(_ for _ in test_data if _.startswith("dataresource-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[dataresource_id]) + assert "configuration" not in expected_merged_config + expected_merged_config["configuration"] = json.loads(test_data[session_id]) + + def create_strategy_middleware( + strategy_type: 'Literal["resource"]', config: ResourceConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "resource" + assert isinstance(config, ResourceConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.dataresource.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/dataresource/{dataresource_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/dataresource/{dataresource_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/routers/test_transformation.py b/tests/routers/test_transformation.py index 48162f80..f5600126 100644 --- a/tests/routers/test_transformation.py +++ b/tests/routers/test_transformation.py @@ -5,6 +5,8 @@ import pytest if TYPE_CHECKING: + from typing import Literal + from fastapi.testclient import TestClient @@ -17,6 +19,8 @@ def test_create_transformation(client: "TestClient") -> None: "name": "script/dummy", "configuration": {}, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 @@ -32,6 +36,8 @@ def test_create_transformation_auth_model(client: "TestClient") -> None: "configuration": {}, "token": dummy_secret, }, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 @@ -50,7 +56,8 @@ def test_create_transformation_auth_headers(client: "TestClient") -> None: "name": "script/dummy", "configuration": {}, }, - headers={"Authorization": dummy_secret}, + headers={"Authorization": dummy_secret, "Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 @@ -59,30 +66,42 @@ def test_create_transformation_auth_headers(client: "TestClient") -> None: assert response.json().get("token") == dummy_secret -def test_get_transformation(client: "TestClient") -> None: +def test_get_transformation(client: "TestClient", test_data: dict[str, str]) -> None: """Test getting a transformation.""" + transformation_id = next(_ for _ in test_data if _.startswith("transformation-")) response = client.get( - "/transformation/transformation-f752c613-fde0-4d43-a7f6-c50f68642daa" + f"/transformation/{transformation_id}", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 -def test_initialize_transformation(client: "TestClient") -> None: +def test_initialize_transformation( + client: "TestClient", test_data: dict[str, str] +) -> None: """Test initializing a transformation.""" + transformation_id = next(_ for _ in test_data if _.startswith("transformation-")) response = client.post( - "/transformation/transformation-f752c613-fde0-4d43-a7f6-c50f68642daa/execute", - json={}, + f"/transformation/{transformation_id}/initialize", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 -def test_get_transformation_status(client: "TestClient") -> None: +def test_get_transformation_status( + client: "TestClient", test_data: dict[str, str] +) -> None: """Test getting a transformation status.""" from oteapi.models.transformationconfig import TransformationStatus from pydantic import ValidationError + transformation_id = next(_ for _ in test_data if _.startswith("transformation-")) response = client.get( - "/transformation/transformation-f752c613-fde0-4d43-a7f6-c50f68642daa/status?task_id=" + f"/transformation/{transformation_id}/status?task_id=", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) try: TransformationStatus(**response.json()) @@ -91,10 +110,74 @@ def test_get_transformation_status(client: "TestClient") -> None: assert response.status_code == 200 -def test_execute_transformation(client: "TestClient") -> None: +def test_execute_transformation( + client: "TestClient", test_data: dict[str, str] +) -> None: """Test executing a transformation.""" + transformation_id = next(_ for _ in test_data if _.startswith("transformation-")) response = client.post( - "/transformation/transformation-f752c613-fde0-4d43-a7f6-c50f68642daa/execute", - json={}, + f"/transformation/{transformation_id}/execute", + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), ) assert response.status_code == 200 + + +@pytest.mark.parametrize("method", ["initialize", "get"]) +def test_session_config_merge( + client: "TestClient", + test_data: dict[str, str], + method: 'Literal["initialize", "get"]', + monkeypatch: "pytest.MonkeyPatch", +) -> None: + """Test the current session is merged into the strategy configuration.""" + import json + + from oteapi.models import TransformationConfig + from oteapi.plugins import create_strategy + + original_create_strategy = create_strategy + + transformation_id = next(_ for _ in test_data if _.startswith("transformation-")) + session_id = next(_ for _ in test_data if _.startswith("session-")) + + expected_merged_config = json.loads(test_data[transformation_id]) + expected_merged_config["configuration"].update(json.loads(test_data[session_id])) + + def create_strategy_middleware( + strategy_type: 'Literal["transformation"]', config: TransformationConfig + ): + """Create a strategy middleware - do some testing.""" + assert strategy_type == "transformation" + assert isinstance(config, TransformationConfig) + + # THIS is where we test the session has been properly merged into the strategy + # configuration ! + assert ( + config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + ) + + return original_create_strategy(strategy_type, config) + + monkeypatch.setattr( + "app.routers.transformation.create_strategy", create_strategy_middleware + ) + + if method == "initialize": + response = client.post( + f"/transformation/{transformation_id}/initialize", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + else: # method == "get" + response = client.get( + f"/transformation/{transformation_id}", + params={"session_id": session_id}, + headers={"Content-Type": "application/json"}, + timeout=(3.0, 27.0), + ) + + assert ( + response.status_code == 200 + ), f"Response:\n{json.dumps(response.json(), indent=2)}\nResponse URL: {response.url}" diff --git a/tests/static/test_strategies/download.py b/tests/static/test_strategies/download.py index 1bf8acf1..1ea118b5 100644 --- a/tests/static/test_strategies/download.py +++ b/tests/static/test_strategies/download.py @@ -1,15 +1,12 @@ """Demo download strategy class for file.""" -from typing import TYPE_CHECKING, Annotated, Optional +from typing import Annotated, Optional from oteapi.datacache.datacache import DataCache from oteapi.models import AttrDict, DataCacheConfig, ResourceConfig from pydantic import AnyHttpUrl, BaseModel, Field from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any - class FileConfig(BaseModel): """File Specific Configuration""" @@ -33,17 +30,12 @@ class FileStrategy: resource_config: ResourceConfig - def initialize( - self, session: "Optional[dict[str, Any]]" = None - ) -> "dict[str, Any]": + def initialize(self) -> "AttrDict": """Initialize""" - del session - del self.resource_config - return {} + return AttrDict() - def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def get(self) -> "AttrDict": """Read local file.""" - del session # fix ignore-unused-argument assert self.resource_config.downloadUrl assert ( self.resource_config.downloadUrl.scheme # pylint: disable=no-member @@ -63,7 +55,7 @@ def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": with open(filename, mode, encoding=config.encoding) as handle: key = cache.add(handle.read()) - return {"key": key} + return AttrDict(key=key) class HTTPSConfig(AttrDict): @@ -97,14 +89,7 @@ class HTTPDownloadContent(AttrDict): @dataclass class HTTPSStrategy: - """Strategy for retrieving data via http. - - **Registers strategies**: - - - `("scheme", "http")` - - `("scheme", "https")` - - """ + """Strategy for retrieving data via http.""" download_config: HTTPSDemoConfig diff --git a/tests/static/test_strategies/filter.py b/tests/static/test_strategies/filter.py index bd5d4b01..bf8c3a36 100644 --- a/tests/static/test_strategies/filter.py +++ b/tests/static/test_strategies/filter.py @@ -1,41 +1,37 @@ """Demo filter strategy.""" -from typing import TYPE_CHECKING, Annotated +from typing import Annotated +from oteapi.models import AttrDict from oteapi.models.filterconfig import FilterConfig -from pydantic import BaseModel, Field +from pydantic import Field from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any, Optional - -class DemoDataModel(BaseModel): +class DemoDataConfiguration(AttrDict): """Demo filter data model.""" demo_data: Annotated[list[int], Field(description="List of demo data.")] = [] +class DemoFilterConfig(FilterConfig): + """Demo filter configuration.""" + + configuration: DemoDataConfiguration = Field( + ..., description=FilterConfig.model_fields["configuration"].description + ) + + @dataclass class DemoFilter: """Filter Strategy.""" - filter_config: FilterConfig + filter_config: DemoFilterConfig - def initialize( - self, session: "Optional[dict[str, Any]]" = None - ) -> "dict[str, Any]": + def initialize(self) -> "AttrDict": """Initialize strategy and return a dictionary""" - del session # unused - del self.filter_config # unused - - return {"result": "collectionid"} + return AttrDict(result="collectionid") - def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def get(self) -> "AttrDict": """Execute strategy and return a dictionary""" - del session # unused - model = DemoDataModel( - **self.filter_config.configuration # pylint: disable=not-a-mapping - ) - - return {"key": model.demo_data} + return AttrDict(key=self.filter_config.configuration.demo_data) diff --git a/tests/static/test_strategies/function.py b/tests/static/test_strategies/function.py index 259a53bc..ce1de37e 100644 --- a/tests/static/test_strategies/function.py +++ b/tests/static/test_strategies/function.py @@ -1,27 +1,16 @@ """Demo function strategy class.""" -from typing import TYPE_CHECKING - from oteapi.models import AttrDict, FunctionConfig from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any, Optional - @dataclass class DemoFunctionStrategy: - """Function Strategy. - - **Registers strategies**: - - - `("functionType", "function/DEMO")` - - """ + """Function Strategy.""" function_config: FunctionConfig - def initialize(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: + def initialize(self) -> AttrDict: """Initialize strategy. This method will be called through the `/initialize` endpoint of the OTEAPI @@ -35,12 +24,9 @@ def initialize(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: session-specific context from services. """ - del session # unused - del self.function_config # unused - return AttrDict() - def get(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: + def get(self) -> AttrDict: """Execute the strategy. This method will be called through the strategy-specific endpoint of the @@ -54,6 +40,4 @@ def get(self, session: "Optional[dict[str, Any]]" = None) -> AttrDict: session-specific context from services. """ - del session # unused - del self.function_config # unused return AttrDict() diff --git a/tests/static/test_strategies/mapping.py b/tests/static/test_strategies/mapping.py index b0a70ca8..2fd90cb8 100644 --- a/tests/static/test_strategies/mapping.py +++ b/tests/static/test_strategies/mapping.py @@ -1,13 +1,9 @@ """Demo mapping strategy class.""" -from typing import TYPE_CHECKING - +from oteapi.models import AttrDict from oteapi.models.mappingconfig import MappingConfig from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any, Optional - @dataclass class DemoMappingStrategy: @@ -15,19 +11,10 @@ class DemoMappingStrategy: mapping_config: MappingConfig - def initialize( - self, session: "Optional[dict[str, Any]]" = None - ) -> "dict[str, Any]": + def initialize(self) -> "AttrDict": """Initialize mapping""" + return AttrDict() - del session # unused - del self.mapping_config # unused - return {} - - def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def get(self) -> "AttrDict": """Manage mapping and return shared map""" - - del session # unused - del self.mapping_config # unused - - return {} + return AttrDict() diff --git a/tests/static/test_strategies/parse.py b/tests/static/test_strategies/parse.py index 5b5ae861..84c80fa6 100644 --- a/tests/static/test_strategies/parse.py +++ b/tests/static/test_strategies/parse.py @@ -1,8 +1,7 @@ """Demo strategy class for text/json.""" -# pylint: disable=unused-argument import json -from typing import TYPE_CHECKING, Literal, Optional +from typing import Literal, Optional from oteapi.datacache import DataCache from oteapi.models import AttrDict, DataCacheConfig, HostlessAnyUrl, ParserConfig @@ -10,9 +9,6 @@ from pydantic import Field from pydantic.dataclasses import dataclass -if TYPE_CHECKING: - from typing import Any - class DEMOConfig(AttrDict): """JSON parse-specific Configuration Data Model.""" @@ -20,7 +16,7 @@ class DEMOConfig(AttrDict): downloadUrl: Optional[HostlessAnyUrl] = Field( None, description="The HTTP(S) URL, which will be downloaded." ) - mediaType: Optional[str] = Field( + mediaType: Optional[Literal["application/json"]] = Field( "application/json", description=("The media type"), ) @@ -51,11 +47,11 @@ class DemoJSONDataParseStrategy: parse_config: DEMOParserConfig - def initialize(self) -> "dict[str, Any]": + def initialize(self) -> "AttrDict": """Initialize""" - return {} + return AttrDict() - def get(self) -> "dict[str, Any]": + def get(self) -> "AttrDict": """Parse json.""" downloader = create_strategy( "download", self.parse_config.configuration.model_dump() @@ -65,5 +61,5 @@ def get(self) -> "dict[str, Any]": content = cache.get(output["key"]) if isinstance(content, dict): - return content - return json.loads(content) + return AttrDict(**content) + return AttrDict(**json.loads(content)) diff --git a/tests/static/test_strategies/transformation.py b/tests/static/test_strategies/transformation.py index eb505906..c84c9d38 100644 --- a/tests/static/test_strategies/transformation.py +++ b/tests/static/test_strategies/transformation.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import TYPE_CHECKING +from oteapi.models import AttrDict from oteapi.models.transformationconfig import ( TransformationConfig, TransformationStatus, @@ -10,7 +11,7 @@ from pydantic.dataclasses import dataclass if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any @dataclass @@ -19,25 +20,16 @@ class DummyTransformationStrategy: transformation_config: TransformationConfig - def run(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def run(self) -> "dict[str, Any]": """Run a job, return a jobid""" - del session - del self.transformation_config return {"result": "a01d"} - def initialize( - self, session: "Optional[dict[str, Any]]" = None - ) -> "dict[str, Any]": + def initialize(self) -> "AttrDict": """Initialize a job""" - del session - del self.transformation_config - - return {"result": "collection id"} + return AttrDict(result="collection id") def status(self, task_id: str) -> TransformationStatus: """Get job status""" - del self.transformation_config # unused - return TransformationStatus( id=task_id, status="wip", @@ -47,8 +39,6 @@ def status(self, task_id: str) -> TransformationStatus: finishTime=datetime.utcnow(), ) - def get(self, session: "Optional[dict[str, Any]]" = None) -> "dict[str, Any]": + def get(self) -> "AttrDict": """get transformation""" - del session # unused - del self.transformation_config # unused - return {} + return AttrDict() From 7aceaa72229f5afac0f08536f476ef389175f1bd Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 28 Feb 2024 14:05:42 +0100 Subject: [PATCH 32/40] Update dumping dataresouce config in test --- tests/routers/test_resource.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/routers/test_resource.py b/tests/routers/test_resource.py index ca6524f6..60cf0e8e 100644 --- a/tests/routers/test_resource.py +++ b/tests/routers/test_resource.py @@ -91,8 +91,14 @@ def create_strategy_middleware( # THIS is where we test the session has been properly merged into the strategy # configuration ! + # Cannot use exclude_unset because the 'configuration' is not set originally. + # When 'configuration' is updated, this does not change the original pydantic + # model understanding that the 'configuration' is not set. + # So instead, we use exclude_none to include the 'configuration' key in the + # dumped dict, but exclude the non-overwritten/snon-set 'description' value. assert ( - config.model_dump(mode="json", exclude_unset=True) == expected_merged_config + config.model_dump(mode="json", exclude_none=True, exclude=["description"]) + == expected_merged_config ) return original_create_strategy(strategy_type, config) From 7181f95c7b853ea0d458eec7f2b4b2b8581b3166 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 29 Feb 2024 15:37:09 +0100 Subject: [PATCH 33/40] Use v0.7.0.dev0 of oteapi-core Note: When installing locally the requirements.txt via 'pip install' remember to use the '--pre' option. Fix the regex in 'entrypoint.sh' to allow for "weird" versions of oteapi-core in requirements.txt. --- .github/utils/direct_requirements.txt | 2 +- entrypoint.sh | 2 +- requirements.txt | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/utils/direct_requirements.txt b/.github/utils/direct_requirements.txt index 89c52394..708276e4 100644 --- a/.github/utils/direct_requirements.txt +++ b/.github/utils/direct_requirements.txt @@ -1,6 +1,6 @@ fastapi>=0.110.0 hypercorn>=0.14.4 -oteapi-core>=0.6.1 +oteapi-core>=0.7.0.dev0 pydantic>=2.6.2 pydantic-settings>=2.2.1 redis>=5.0.1 diff --git a/entrypoint.sh b/entrypoint.sh index 702dfe36..0aab2248 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -15,7 +15,7 @@ set -e # constraint when installing the plugin packages. I.e., the plugin packages cannot change the versions of the # already installed packages. oteapi_core_requirement="$(pip list --format=freeze --include-editable | grep 'oteapi-core')" -sed -E -e "s|oteapi-core==[0-9.]+|${oteapi_core_requirement}|" requirements.txt > constraints.txt +sed -E -e "s|oteapi-core==.*|${oteapi_core_requirement}|" requirements.txt > constraints.txt if [ -n "${OTEAPI_PLUGIN_PACKAGES}" ]; then echo "Installing plugins:" diff --git a/requirements.txt b/requirements.txt index abaa4d0d..4a1ab941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,8 +28,7 @@ idna==3.6 iso8601==2.1.0 kombu==5.3.5 openpyxl==3.1.2 -# oteapi-core==0.6.1 -oteapi-core @ git+https://github.com/EMMC-ASBL/oteapi-core@master +oteapi-core==0.7.0.dev0 paramiko==3.4.0 pillow==10.2.0 priority==2.0.0 From 3cef5603787bde01ba9446ce878e5c274b3759cf Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 29 Feb 2024 19:52:20 +0100 Subject: [PATCH 34/40] Update docker-compose_ci_plugin.yml Expose debug port in development Dockerfile target. Hopefully this fixes the docker-compose development CI. --- .github/utils/docker-compose_ci_plugins.yml | 8 ++++++-- .github/workflows/ci_tests.yml | 2 +- .vscode/launch.json | 2 +- Dockerfile | 2 +- entrypoint.sh | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/utils/docker-compose_ci_plugins.yml b/.github/utils/docker-compose_ci_plugins.yml index 214bb2df..70b32885 100644 --- a/.github/utils/docker-compose_ci_plugins.yml +++ b/.github/utils/docker-compose_ci_plugins.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.3" services: oteapi: @@ -7,13 +7,17 @@ services: target: "${DOCKER_OTEAPI_TARGET:-development}" ports: - "${PORT:-8080}:8080" + - "5678:5678" # Debug port environment: OTEAPI_REDIS_TYPE: redis OTEAPI_REDIS_HOST: redis OTEAPI_REDIS_PORT: 6379 OTEAPI_prefix: "${OTEAPI_prefix:-/api/v1}" - OTEAPI_PLUGIN_PACKAGES: + OTEAPI_INCLUDE_REDISADMIN: "${OTEAPI_INCLUDE_REDISADMIN:-True}" + OTEAPI_EXPOSE_SECRETS: "${OTEAPI_EXPOSE_SECRETS:-True}" PATH_TO_OTEAPI_CORE: "${PATH_TO_OTEAPI_CORE:-/dev/null}" + OTEAPI_PLUGIN_PACKAGES: + OTEAPI_AUTHENTICATION_DEPENDENCIES: depends_on: - redis networks: diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index e98ca030..0915b95b 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -150,7 +150,7 @@ jobs: run: | docker compose -f docker-compose_dev.yml up & .github/utils/wait_for_it.sh localhost:${{ env.PORT }} -t 120 - sleep 15 + sleep 20 curl -X "GET" -i "http://localhost:${{ env.PORT }}${{ env.OTEAPI_prefix }}/session" - name: Clone current requirement version of `oteapi-core` locally diff --git a/.vscode/launch.json b/.vscode/launch.json index 1b5a08c0..70fad952 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,7 +3,7 @@ "configurations": [ { "name": "Python: Remote Attach", - "type": "python", + "type": "debugpy", "request": "attach", "connect": { "host": "localhost", diff --git a/Dockerfile b/Dockerfile index 8e973a9a..af10cfd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ COPY .dev/requirements_dev.txt .pre-commit-config.yaml pyproject.toml ./ RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt ENV DEV_ENV=1 # Run app with reload option -EXPOSE 8080 +EXPOSE 8080 5678 CMD if [ "${PATH_TO_OTEAPI_CORE}" != "/dev/null" ] && [ -n "${PATH_TO_OTEAPI_CORE}" ]; then \ pip install -U --force-reinstall -e /oteapi_core; fi \ && ./entrypoint.sh --reload --debug --log-level debug diff --git a/entrypoint.sh b/entrypoint.sh index 0aab2248..65f46b4a 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,7 +33,7 @@ else echo "No extra plugin packages provided. Specify 'OTEAPI_PLUGIN_PACKAGES' to specify plugin packages." fi -if [ "$DEV_ENV" = "1" ]; then +if [ "${DEV_ENV}" = "1" ]; then python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 asgi:app "$@" else hypercorn --bind 0.0.0.0:8080 asgi:app "$@" From bfb6ff0ebe536471e8bb2321a2d32fe1c39e025d Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 29 Feb 2024 20:13:33 +0100 Subject: [PATCH 35/40] Try avoiding running debugpy --- .github/utils/docker-compose_ci_plugins.yml | 1 + docker-compose_dev.yml | 1 + entrypoint.sh | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/utils/docker-compose_ci_plugins.yml b/.github/utils/docker-compose_ci_plugins.yml index 70b32885..77b30910 100644 --- a/.github/utils/docker-compose_ci_plugins.yml +++ b/.github/utils/docker-compose_ci_plugins.yml @@ -18,6 +18,7 @@ services: PATH_TO_OTEAPI_CORE: "${PATH_TO_OTEAPI_CORE:-/dev/null}" OTEAPI_PLUGIN_PACKAGES: OTEAPI_AUTHENTICATION_DEPENDENCIES: + CI: depends_on: - redis networks: diff --git a/docker-compose_dev.yml b/docker-compose_dev.yml index 2a0ad68d..c314b57e 100644 --- a/docker-compose_dev.yml +++ b/docker-compose_dev.yml @@ -18,6 +18,7 @@ services: PATH_TO_OTEAPI_CORE: OTEAPI_PLUGIN_PACKAGES: OTEAPI_AUTHENTICATION_DEPENDENCIES: + CI: depends_on: - redis networks: diff --git a/entrypoint.sh b/entrypoint.sh index 65f46b4a..b1cd2144 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,6 +8,10 @@ # This is relevant for, e.g., the `development` target. set -e +if [ "${DEV_ENV}" == "1" ]; then + set -x +fi + # If a custom `oteapi-core` version is installed, the version for it may differ from # that given in `requirements.txt`. To work around this, we copy the `requirements.txt` # file into a `constraints.txt` file and replace the `oteapi-core` version there with @@ -33,7 +37,7 @@ else echo "No extra plugin packages provided. Specify 'OTEAPI_PLUGIN_PACKAGES' to specify plugin packages." fi -if [ "${DEV_ENV}" = "1" ]; then +if [ "${DEV_ENV}" == "1" ] && [ "${CI}" == "" ]; then python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 asgi:app "$@" else hypercorn --bind 0.0.0.0:8080 asgi:app "$@" From 6b57b1d9e7628c917d2a3b83e99d8b825eb3cd74 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 29 Feb 2024 20:16:53 +0100 Subject: [PATCH 36/40] Revert changed sleep in CI --- .github/workflows/ci_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 0915b95b..e98ca030 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -150,7 +150,7 @@ jobs: run: | docker compose -f docker-compose_dev.yml up & .github/utils/wait_for_it.sh localhost:${{ env.PORT }} -t 120 - sleep 20 + sleep 15 curl -X "GET" -i "http://localhost:${{ env.PORT }}${{ env.OTEAPI_prefix }}/session" - name: Clone current requirement version of `oteapi-core` locally From 306c0ed92bb8eb65119e8010b71ef009ac51462e Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 29 Feb 2024 20:41:17 +0100 Subject: [PATCH 37/40] Remove logging setup in tests --- tests/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fd1a24a1..bf7185aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,6 @@ """Fixtures and configuration for PyTest.""" # pylint: disable=invalid-name,redefined-builtin,unused-argument,comparison-with-callable -import logging from typing import TYPE_CHECKING import pytest @@ -12,8 +11,6 @@ from fastapi.testclient import TestClient -logging.getLogger("test_strategies").setLevel(logging.DEBUG) - class DummyCache: """Mock cache for RedisCache.""" From 0e1213a92a07bf0cf184ef0fdb55e6167ac58387 Mon Sep 17 00:00:00 2001 From: Treesa Joseph <66374813+Treesarj@users.noreply.github.com> Date: Fri, 1 Mar 2024 08:43:04 +0100 Subject: [PATCH 38/40] Update app/routers/session.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- app/routers/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/session.py b/app/routers/session.py index 19e49850..50b5e2e1 100644 --- a/app/routers/session.py +++ b/app/routers/session.py @@ -58,7 +58,7 @@ async def list_sessions( cache: TRedisPlugin, ) -> ListSessionsResponse: """Get all session keys""" - keylist: list[Union[str, bytes]] = [] + keylist: list[str] = [] for key in await cache.keys(pattern=f"{IDPREFIX}*"): if not isinstance(key, (str, bytes)): raise TypeError( From 0ad16f0050bc7271d9ddf7ca150172bf6d016aa2 Mon Sep 17 00:00:00 2001 From: TreesaJ Date: Fri, 1 Mar 2024 09:29:24 +0100 Subject: [PATCH 39/40] requested changes --- app/routers/dataresource.py | 57 +++++++++++++++++++------------------ app/routers/parser.py | 2 +- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/routers/dataresource.py b/app/routers/dataresource.py index 4ca72d80..15e1fe64 100644 --- a/app/routers/dataresource.py +++ b/app/routers/dataresource.py @@ -73,7 +73,9 @@ async def create_dataresource( return new_resource -@ROUTER.get("/{resource_id}/info", response_model=ResourceConfig) +@ROUTER.get( + "/{resource_id}/info", response_model=ResourceConfig, include_in_schema=False +) async def info_dataresource( cache: TRedisPlugin, resource_id: str, @@ -100,23 +102,23 @@ async def read_dataresource( cache_value = await _fetch_cache_value(cache, resource_id, "resource_id") config = ResourceConfig(**json.loads(cache_value)) + if (not config.resourceType) or ( + not ( + (config.downloadUrl and config.mediaType) + or (config.accessUrl and config.accessService) + ) + ): + raise httpexception_422_resource_id_is_unprocessable(resource_id) + if session_id: session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) - if not config.resourceType: - raise httpexception_422_resource_id_is_unprocessable(resource_id) - - if (config.downloadUrl and config.mediaType) or ( - config.accessUrl and config.accessService - ): - session_update = create_strategy("resource", config).get() - if session_update and session_id: - await _update_session( - session_id=session_id, updated_session=session_update, redis=cache - ) - else: - raise httpexception_422_resource_id_is_unprocessable(resource_id) + session_update = create_strategy("resource", config).get() + if session_update and session_id: + await _update_session( + session_id=session_id, updated_session=session_update, redis=cache + ) return GetResourceResponse(**session_update) @@ -131,23 +133,22 @@ async def initialize_dataresource( cache_value = await _fetch_cache_value(cache, resource_id, "resource_id") config = ResourceConfig(**json.loads(cache_value)) + if (not config.resourceType) or ( + not ( + (config.downloadUrl and config.mediaType) + or (config.accessUrl and config.accessService) + ) + ): + raise httpexception_422_resource_id_is_unprocessable(resource_id) + if session_id: session_data = await _fetch_cache_value(cache, session_id, "session_id") populate_config_from_session(json.loads(session_data), config) - if not config.resourceType: - raise httpexception_422_resource_id_is_unprocessable(resource_id) - - if (config.downloadUrl and config.mediaType) or ( - config.accessUrl and config.accessService - ): - # Download strategy - session_update = create_strategy("resource", config).initialize() - if session_update and session_id: - await _update_session( - session_id=session_id, updated_session=session_update, redis=cache - ) - else: - raise httpexception_422_resource_id_is_unprocessable(resource_id) + session_update = create_strategy("resource", config).initialize() + if session_update and session_id: + await _update_session( + session_id=session_id, updated_session=session_update, redis=cache + ) return InitializeResourceResponse(**session_update) diff --git a/app/routers/parser.py b/app/routers/parser.py index 4b1f7b72..ac4b2f07 100644 --- a/app/routers/parser.py +++ b/app/routers/parser.py @@ -56,7 +56,7 @@ async def create_parser( # Get parser info -@ROUTER.get("/{parser_id}/info", response_model=ParserConfig) +@ROUTER.get("/{parser_id}/info", response_model=ParserConfig, include_in_schema=False) async def info_parser(cache: TRedisPlugin, parser_id: str) -> ParserConfig: """Get information about a specific parser.""" cache_value = await _fetch_cache_value(cache, parser_id, "parser_id") From 257fe9fdc6ebd5e9948f4237d0c29555847b6fc1 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 1 Mar 2024 10:06:57 +0100 Subject: [PATCH 40/40] Update README with important notice Mention the current use of a development version of oteapi-core. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 63a662f0..5016b76f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Open Translation Environment (OTE) API +**Important**: The latest [`oteapi` Docker image](https://github.com/EMMC-ASBL/oteapi-services/pkgs/container/oteapi) is using a development version of [`oteapi-core`](https://github.com/EMMC-ASBL/oteapi-core). +To use a version of the `oteapi` Docker image that runs only on the latest stable version of `oteapi-core`, use the version tag `1.20240228.345` or earlier. +Example: `ghcr.io/emmc-asbl/oteapi:1.20240228.345`. + ## Run in Docker ### Development target